:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use chrono::Utc;
use tokio_util::sync::CancellationToken;
use kafka_deduplicator::checkpoint::{
hash_prefix_for_partition, CheckpointConfig, CheckpointExporter, CheckpointMetadata,
CheckpointPlan, CheckpointUploader, CheckpointWorker, UploadCancelledError,
};
use kafka_deduplicator::kafka::types::Partition;
use kafka_deduplicator::store::TimestampMetadata;
use kafka_deduplicator::test_utils::test_helpers::{create_test_dedup_store, TestRawEventBuilder};
use anyhow::{Context, Result};
use tempfile::TempDir;
use tracing::info;
/// Mock uploader for testing that stores files in local filesystem.
/// Holds ownership of TempDir via Arc to ensure the directory persists for the uploader's lifetime
/// and survives cloning (multiple references share the same temp directory).
#[derive(Debug, Clone)]
pub struct MockUploader {
/// TempDir that owns the upload directory (shared across clones, dropped when all refs gone)
_temp_dir: Arc<TempDir>,
/// Local directory where uploaded files are stored
upload_dir: PathBuf,
/// Whether the uploader should simulate being available
available: bool,
}
impl MockUploader {
pub fn new() -> Result<Self> {
let temp_dir = TempDir::new()?;
let upload_dir = temp_dir.path().to_path_buf();
Ok(Self {
_temp_dir: Arc::new(temp_dir),
upload_dir,
available: true,
})
}
pub fn new_unavailable() -> Result<Self> {
let temp_dir = TempDir::new()?;
let upload_dir = temp_dir.path().to_path_buf();
Ok(Self {
_temp_dir: Arc::new(temp_dir),
upload_dir,
available: false,
})
}
/// Get the stored files for testing
pub async fn get_stored_files(&self) -> Result<HashMap<String, Vec<u8>>> {
let mut files = HashMap::new();
if !self.upload_dir.exists() {
return Ok(files);
}
let mut stack = vec![self.upload_dir.clone()];
while let Some(current_path) = stack.pop() {
let entries = std::fs::read_dir(¤t_path)?;
for entry in entries {
let entry = entry?;
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else {
let relative_path = path.strip_prefix(&self.upload_dir)?;
let key = relative_path.to_string_lossy().replace('\\', "/");
let data = tokio::fs::read(&path).await?;
files.insert(key, data);
}
}
}
Ok(files)
}
/// Clear all stored files
pub async fn clear(&self) -> Result<()> {
if self.upload_dir.exists() {
tokio::fs::remove_dir_all(&self.upload_dir).await?;
tokio::fs::create_dir_all(&self.upload_dir).await?;
}
Ok(())
}
/// Get count of stored files
pub async fn file_count(&self) -> Result<usize> {
if !self.upload_dir.exists() {
return Ok(0);
}
let files = self.get_stored_files().await?;
Ok(files.len())
}
}
impl Default for MockUploader {
fn default() -> Self {
Self::new().unwrap()
}
}
#[async_trait]
impl CheckpointUploader for MockUploader {
async fn upload_checkpoint_with_plan_cancellable(
&self,
plan: &CheckpointPlan,
cancel_token: Option<&CancellationToken>,
) -> Result<Vec<String>> {
if let Some(token) = cancel_token {
if token.is_cancelled() {
info!("Mock uploader: cancelled before starting");
return Err(UploadCancelledError {
reason: "before starting".to_string(),
}
.into());
}
}
info!(
"Mock uploading checkpoint with plan: {} files to upload to remote path: {}",
plan.files_to_upload.len(),
plan.info.get_remote_attempt_path(),
);
let remote_parent_path = self.upload_dir.join(plan.info.get_remote_attempt_path());
tokio::fs::create_dir_all(remote_parent_path).await?;
let mut files_to_upload = Vec::new();
for local_file in &plan.files_to_upload {
files_to_upload.push((
local_file.local_path.clone(),
plan.info.get_file_key(&local_file.filename),
));
}
let mut uploaded_keys = Vec::new();
for (local_file_path, remote_file_path_str) in files_to_upload {
if let Some(token) = cancel_token {
if token.is_cancelled() {
info!("Mock uploader: cancelled during file upload");
return Err(UploadCancelledError {
reason: "during file upload".to_string(),
}
.into());
}
}
let remote_file_path = self.upload_dir.join(&remote_file_path_str);
if let Some(parent) = remote_file_path.parent() {
tokio::fs::create_dir_all(parent).await?;
}
tokio::fs::copy(&local_file_path, &remote_file_path).await?;
uploaded_keys.push(remote_file_path_str);
}
if let Some(token) = cancel_token {
if token.is_cancelled() {
info!("Mock uploader: cancelled before metadata upload");
return Err(UploadCancelledError {
reason: "before metadata upload".to_string(),
}
.into());
}
}
let metadata_key = self.upload_dir.join(plan.info.get_metadata_key());
let metadata_json = &plan
.info
.metadata
.to_json()
.with_context(|| format!("Failed to serialize checkpoint metadata: {plan:?}"))?;
tokio::fs::write(&metadata_key, metadata_json).await?;
uploaded_keys.push(metadata_key.to_string_lossy().to_string());
info!("Mock uploaded {} files with plan", uploaded_keys.len());
Ok(uploaded_keys)
}
async fn is_available(&self) -> bool {
self.available
}
}
#[tokio::test]
async fn test_checkpoint_exporter_creation() {
let uploader = MockUploader::new().unwrap();
let exporter = CheckpointExporter::new(Box::new(uploader));
assert!(exporter.is_available().await);
}
#[tokio::test]
async fn test_unavailable_uploader() {
let tmp_store_dir = TempDir::new().unwrap();
let test_topic = "test_unavailable_uploader";
let test_partition = 0;
let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);
// Add some test data
let events = vec![
TestRawEventBuilder::new()
.distinct_id("user1")
.token("token1")
.event("event1")
.current_timestamp()
.build(),
TestRawEventBuilder::new()
.distinct_id("user2")
.token("token1")
.event("event2")
.current_timestamp()
.build(),
];
for event in &events {
let key = event.into();
let metadata = TimestampMetadata::new(event);
store.put_timestamp_record(&key, &metadata).unwrap();
}
let tmp_checkpoint_dir = TempDir::new().unwrap();
let config = CheckpointConfig {
checkpoint_interval: Duration::from_secs(60),
local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
s3_bucket: "test-bucket".to_string(),
s3_key_prefix: "test-prefix".to_string(),
aws_region: Some("us-east-1".to_string()),
..Default::default()
};
let uploader = Box::new(MockUploader::new_unavailable().unwrap());
let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));
let local_base_dir = Path::new(&config.local_checkpoint_dir);
let remote_namespace = config.s3_key_prefix.clone();
let attempt_timestamp = Utc::now();
let partition = Partition::new(test_topic.to_string(), test_partition);
let worker = CheckpointWorker::new(
1,
local_base_dir,
remote_namespace,
partition.clone(),
attempt_timestamp,
exporter.clone(),
None,
);
// The wrapper thread closure that is spawned to run this in
// production will catch and log/stat these errors
let result = worker.checkpoint_partition(&store, None).await;
assert!(result.is_err());
// No files should be uploaded when the remote storage is unavailable
let file_count = uploader.file_count().await.unwrap();
assert_eq!(
file_count, 0,
"No files should be uploaded when uploader is unavailable"
);
}
#[tokio::test]
async fn test_unpopulated_exporter() {
let tmp_store_dir = TempDir::new().unwrap();
let test_topic = "test_unpopulated_exporter";
let test_partition = 0;
let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);
// Add some test data
let events = vec![
TestRawEventBuilder::new()
.distinct_id("user1")
.token("token1")
.event("event1")
.current_timestamp()
.build(),
TestRawEventBuilder::new()
.distinct_id("user2")
.token("token1")
.event("event2")
.current_timestamp()
.build(),
];
for event in &events {
let key = event.into();
let metadata = TimestampMetadata::new(event);
store.put_timestamp_record(&key, &metadata).unwrap();
}
let tmp_checkpoint_dir = TempDir::new().unwrap();
let config = CheckpointConfig {
checkpoint_interval: Duration::from_secs(60),
local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
s3_bucket: "test-bucket".to_string(),
s3_key_prefix: "test-prefix".to_string(),
aws_region: Some("us-east-1".to_string()),
..Default::default()
};
let local_base_dir = Path::new(&config.local_checkpoint_dir);
let remote_namespace = config.s3_key_prefix.clone();
let attempt_timestamp = Utc::now();
let partition = Partition::new(test_topic.to_string(), test_partition);
// without an exporter supplied to the worker, the checkpoint will
// succeed and be created locally but never uploaded to remote storage
let worker = CheckpointWorker::new(
1,
local_base_dir,
remote_namespace,
partition.clone(),
attempt_timestamp,
None,
None,
);
// Checkpoint should still succeed even if uploader is unavailable
let result = worker.checkpoint_partition(&store, None).await;
assert!(result.is_ok());
assert!(result.unwrap().is_none());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_from_plan_with_no_previous_metadata() {
let test_topic = "manual_cp_incremental";
let test_partition = 0;
let tmp_store_dir = TempDir::new().unwrap();
let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);
// Add some test data
let events = vec![
TestRawEventBuilder::new()
.distinct_id("user1")
.token("token1")
.event("event1")
.current_timestamp()
.build(),
TestRawEventBuilder::new()
.distinct_id("user2")
.token("token1")
.event("event2")
.current_timestamp()
.build(),
];
for event in &events {
let key = event.into();
let metadata = TimestampMetadata::new(event);
store.put_timestamp_record(&key, &metadata).unwrap();
}
let tmp_checkpoint_dir = TempDir::new().unwrap();
let config = CheckpointConfig {
checkpoint_interval: Duration::from_secs(60),
local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
s3_bucket: "test-bucket".to_string(),
s3_key_prefix: "test-prefix".to_string(),
aws_region: Some("us-east-1".to_string()),
..Default::default()
};
let uploader = Box::new(MockUploader::new().unwrap());
let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));
let local_base_dir = Path::new(&config.local_checkpoint_dir);
let remote_namespace = config.s3_key_prefix.clone();
let partition = Partition::new(test_topic.to_string(), test_partition);
let attempt_timestamp = Utc::now();
let worker = CheckpointWorker::new(
1,
local_base_dir,
remote_namespace,
partition.clone(),
attempt_timestamp,
exporter.clone(),
None,
);
// Perform checkpoint without previous metadata
let result = worker.checkpoint_partition(&store, None).await;
assert!(result.is_ok());
let result = result.unwrap();
assert!(result.is_some());
let info = result.unwrap();
// All remote paths (metadata and objects) now include hash prefix
let hash = hash_prefix_for_partition(partition.topic(), partition.partition_number());
let expected_remote_path = format!(
"{}/{}/{}/{}/{}",
hash,
config.s3_key_prefix,
partition.topic(),
partition.partition_number(),
CheckpointMetadata::generate_id(attempt_timestamp),
);
assert_eq!(info.get_remote_attempt_path(), expected_remote_path);
let remote_checkpoint_files = uploader.get_stored_files().await.unwrap();
assert!(!remote_checkpoint_files.is_empty());
for k in remote_checkpoint_files.keys() {
assert!(
k.starts_with(&expected_remote_path),
"all keys should be under hashed path, got: {k}"
);
}
// Verify exported files contain expected RocksDB checkpoint files
assert!(remote_checkpoint_files
.keys()
.any(|k| k.ends_with("CURRENT")));
assert!(remote_checkpoint_files
.keys()
.any(|k| k.contains("MANIFEST-")));
assert!(remote_checkpoint_files
.keys()
.any(|k| k.contains("OPTIONS-")));
assert!(remote_checkpoint_files.keys().any(|k| k.ends_with(".sst")));
assert!(remote_checkpoint_files.keys().any(|k| k.ends_with(".log")));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_from_plan_with_previous_metadata() {
// Note: detailed planner diffs are exercised in the planner test suite
let test_topic = "test_cp_from_plan_with_prev_metadata";
let test_partition = 0;
let tmp_store_dir = TempDir::new().unwrap();
let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);
// Add some test data
let events = vec![TestRawEventBuilder::new()
.distinct_id("user1")
.token("token1")
.event("event1")
.current_timestamp()
.build()];
for event in &events {
let key = event.into();
let metadata = TimestampMetadata::new(event);
store.put_timestamp_record(&key, &metadata).unwrap();
}
let tmp_checkpoint_dir = TempDir::new().unwrap();
let config = CheckpointConfig {
checkpoint_interval: Duration::from_secs(60),
local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
s3_bucket: "test-bucket".to_string(),
s3_key_prefix: "test-prefix".to_string(),
aws_region: Some("us-east-1".to_string()),
..Default::default()
};
let uploader = Box::new(MockUploader::new().unwrap());
let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));
let local_base_dir = Path::new(&config.local_checkpoint_dir);
let remote_namespace = config.s3_key_prefix.clone();
let partition = Partition::new(test_topic.to_string(), test_partition);
let attempt_timestamp = Utc::now();
let worker = CheckpointWorker::new(
1,
local_base_dir,
remote_namespace.clone(),
partition.clone(),
attempt_timestamp,
exporter.clone(),
None,
);
// Perform checkpoint without previous metadata
let result = worker.checkpoint_partition(&store, None).await;
assert!(result.is_ok());
let result = result.unwrap();
assert!(result.is_some());
let orig_info = result.unwrap();
// All remote paths (metadata and objects) now include hash prefix
let hash = hash_prefix_for_partition(partition.topic(), partition.partition_number());
let orig_expected_remote_path = format!(
"{}/{}/{}/{}/{}",
hash,
config.s3_key_prefix,
partition.topic(),
partition.partition_number(),
CheckpointMetadata::generate_id(attempt_timestamp),
);
assert_eq!(
orig_info.get_remote_attempt_path(),
orig_expected_remote_path
);
let orig_remote_checkpoint_files = uploader.get_stored_files().await.unwrap();
assert!(!orig_remote_checkpoint_files.is_empty());
for k in orig_remote_checkpoint_files.keys() {
assert!(
k.starts_with(&orig_expected_remote_path),
"all keys should be under hashed path, got: {k}"
);
}
// Verify exported files contain expected RocksDB checkpoint files, including SSTs
assert!(orig_remote_checkpoint_files
.keys()
.any(|k| k.ends_with(".sst")));
assert!(orig_remote_checkpoint_files
.keys()
.any(|k| k.ends_with("CURRENT")));
assert!(orig_remote_checkpoint_files
.keys()
.any(|k| k.contains("MANIFEST-")));
assert!(orig_remote_checkpoint_files
.keys()
.any(|k| k.contains("OPTIONS-")));
assert!(orig_remote_checkpoint_files
.keys()
.any(|k| k.ends_with(".log")));
// Do not add more test data so SST files don't change across checkpoints.
// Await > 1 second so timestamp attempt directory changes between checkpoints
tokio::time::sleep(Duration::from_millis(1100)).await;
let next_attempt_timestamp = Utc::now();
let next_checkpoint_id = CheckpointMetadata::generate_id(next_attempt_timestamp);
let next_expected_remote_path = format!(
"{}/{}/{}/{}/{}",
hash,
config.s3_key_prefix,
partition.topic(),
partition.partition_number(),
next_checkpoint_id,
);
assert!(uploader.clear().await.is_ok());
let worker_next = CheckpointWorker::new(
2,
local_base_dir,
remote_namespace,
partition.clone(),
next_attempt_timestamp,
exporter.clone(),
None,
);
let result = worker_next
.checkpoint_partition(&store, Some(&orig_info.metadata))
.await;
assert!(result.is_ok());
let result = result.unwrap();
assert!(result.is_some());
// The new checkpoint metadata should contain a mix of new and reused files
let info = result.unwrap();
assert_eq!(info.get_remote_attempt_path(), next_expected_remote_path);
let next_remote_checkpoint_files = uploader.get_stored_files().await.unwrap();
assert!(!next_remote_checkpoint_files.is_empty());
for k in next_remote_checkpoint_files.keys() {
assert!(
k.starts_with(&next_expected_remote_path),
"all keys should be under hashed path, got: {k}"
);
}
// there should be no new SST files uploaded in this checkpoint
// because the original checkpoint uploaded them already
assert!(next_remote_checkpoint_files
.keys()
.all(|k| !k.ends_with(".sst")));
// Verify exported files contain expected RocksDB non-SST files
// For now, we always upload the latest CURRENT, but
// but this attempt, we don't upload new MANIFEST, OPTIONS,
// or .log (WAL) files because the checksums will not be different
assert!(next_remote_checkpoint_files
.keys()
.any(|k| k.ends_with("CURRENT")));
assert!(next_remote_checkpoint_files
.keys()
.all(|k| !k.contains("MANIFEST-")));
assert!(next_remote_checkpoint_files
.keys()
.all(|k| !k.contains("OPTIONS-")));
assert!(next_remote_checkpoint_files
.keys()
.all(|k| !k.ends_with(".log")));
}
// ============================================================
// Cancellation Tests
// ============================================================
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_with_pre_cancelled_token_skips() {
let test_topic = "test_pre_cancelled";
let test_partition = 0;
let tmp_store_dir = TempDir::new().unwrap();
let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);
// Add test data
let event = TestRawEventBuilder::new()
.distinct_id("user1")
.token("token1")
.event("event1")
.current_timestamp()
.build();
let key = (&event).into();
let metadata = TimestampMetadata::new(&event);
store.put_timestamp_record(&key, &metadata).unwrap();
let tmp_checkpoint_dir = TempDir::new().unwrap();
let config = CheckpointConfig {
checkpoint_interval: Duration::from_secs(60),
local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
s3_bucket: "test-bucket".to_string(),
s3_key_prefix: "test-prefix".to_string(),
aws_region: Some("us-east-1".to_string()),
..Default::default()
};
let uploader = Box::new(MockUploader::new().unwrap());
let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));
let partition = Partition::new(test_topic.to_string(), test_partition);
let attempt_timestamp = Utc::now();
let worker = CheckpointWorker::new(
1,
Path::new(&config.local_checkpoint_dir),
config.s3_key_prefix.clone(),
partition.clone(),
attempt_timestamp,
exporter.clone(),
None,
);
// Create pre-cancelled token
let cancel_token = CancellationToken::new();
cancel_token.cancel();
// Pre-cancelled checkpoint should be cooperatively skipped
let result = worker
.checkpoint_partition_cancellable(&store, None, Some(&cancel_token), Some("test"))
.await;
assert!(
matches!(result, Ok(None)),
"Checkpoint should be skipped when pre-cancelled, got: {result:?}"
);
// Verify no files were uploaded
let file_count = uploader.file_count().await.unwrap();
assert_eq!(
file_count, 0,
"No files should be uploaded when pre-cancelled"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_with_active_token_succeeds() {
let test_topic = "test_active_token";
let test_partition = 0;
let tmp_store_dir = TempDir::new().unwrap();
let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);
// Add test data
let event = TestRawEventBuilder::new()
.distinct_id("user1")
.token("token1")
.event("event1")
.current_timestamp()
.build();
let key = (&event).into();
let metadata = TimestampMetadata::new(&event);
store.put_timestamp_record(&key, &metadata).unwrap();
let tmp_checkpoint_dir = TempDir::new().unwrap();
let config = CheckpointConfig {
checkpoint_interval: Duration::from_secs(60),
local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
s3_bucket: "test-bucket".to_string(),
s3_key_prefix: "test-prefix".to_string(),
aws_region: Some("us-east-1".to_string()),
..Default::default()
};
let uploader = Box::new(MockUploader::new().unwrap());
let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));
let partition = Partition::new(test_topic.to_string(), test_partition);
let attempt_timestamp = Utc::now();
let worker = CheckpointWorker::new(
1,
Path::new(&config.local_checkpoint_dir),
config.s3_key_prefix.clone(),
partition.clone(),
attempt_timestamp,
exporter.clone(),
None,
);
// Create active (non-cancelled) token
let cancel_token = CancellationToken::new();
// Checkpoint should succeed with active token
let result = worker
.checkpoint_partition_cancellable(&store, None, Some(&cancel_token), None)
.await;
assert!(
result.is_ok(),
"Checkpoint should succeed with active token"
);
let info = result.unwrap();
assert!(info.is_some(), "Should return CheckpointInfo");
// Verify files were uploaded
let file_count = uploader.file_count().await.unwrap();
assert!(
file_count > 0,
"Files should be uploaded when token is active"
);
}
#[tokio::test]
async fn test_checkpoint_without_token_succeeds() {
// Verify the legacy non-cancellable path still works
let test_topic = "test_no_token";
let test_partition = 0;
let tmp_store_dir = TempDir::new().unwrap();
let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);
// Add test data
let event = TestRawEventBuilder::new()
.distinct_id("user1")
.token("token1")
.event("event1")
.current_timestamp()
.build();
let key = (&event).into();
let metadata = TimestampMetadata::new(&event);
store.put_timestamp_record(&key, &metadata).unwrap();
let tmp_checkpoint_dir = TempDir::new().unwrap();
let config = CheckpointConfig {
checkpoint_interval: Duration::from_secs(60),
local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
s3_bucket: "test-bucket".to_string(),
s3_key_prefix: "test-prefix".to_string(),
aws_region: Some("us-east-1".to_string()),
..Default::default()
};
let uploader = Box::new(MockUploader::new().unwrap());
let exporter = Some(Arc::new(CheckpointExporter::new(uploader.clone())));
let partition = Partition::new(test_topic.to_string(), test_partition);
let attempt_timestamp = Utc::now();
let worker = CheckpointWorker::new(
1,
Path::new(&config.local_checkpoint_dir),
config.s3_key_prefix.clone(),
partition.clone(),
attempt_timestamp,
exporter.clone(),
None,
);
// Use legacy non-cancellable method (no token)
let result = worker.checkpoint_partition(&store, None).await;
assert!(result.is_ok(), "Legacy checkpoint should succeed");
let info = result.unwrap();
assert!(info.is_some(), "Should return CheckpointInfo");
// Verify files were uploaded
let file_count = uploader.file_count().await.unwrap();
assert!(file_count > 0, "Files should be uploaded");
}
// ============================================================
// spawn_blocking Tests
// ============================================================
// These tests verify that spawn_blocking correctly moves blocking RocksDB
// operations off the tokio worker threads to prevent runtime starvation.
/// Test that spawn_blocking doesn't starve the tokio runtime.
/// This is the core issue that spawn_blocking was introduced to solve.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_doesnt_starve_runtime() {
use std::sync::atomic::{AtomicUsize, Ordering};
let heartbeat_count = Arc::new(AtomicUsize::new(0));
let count_clone = heartbeat_count.clone();
// Simulate liveness check running every 50ms
let heartbeat_task = tokio::spawn(async move {
for _ in 0..10 {
tokio::time::sleep(Duration::from_millis(50)).await;
count_clone.fetch_add(1, Ordering::SeqCst);
}
});
// Create a store with some data that will require actual RocksDB work
let test_topic = "test_starve";
let test_partition = 0;
let tmp_store_dir = TempDir::new().unwrap();
let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);
// Write some test data to make checkpoint creation do real work
for i in 0..100 {
let event = TestRawEventBuilder::new()
.distinct_id(&format!("user-{}", i))
.token("test-token")
.event(&format!("event-{}", i))
.current_timestamp()
.build();
let key = (&event).into();
let metadata = TimestampMetadata::new(&event);
store.put_timestamp_record(&key, &metadata).unwrap();
}
// Create checkpoint worker
let tmp_checkpoint_dir = TempDir::new().unwrap();
let config = CheckpointConfig {
checkpoint_interval: Duration::from_secs(60),
local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
s3_bucket: "test-bucket".to_string(),
s3_key_prefix: "test-prefix".to_string(),
aws_region: Some("us-east-1".to_string()),
..Default::default()
};
let worker = CheckpointWorker::new(
1,
Path::new(&config.local_checkpoint_dir),
config.s3_key_prefix.clone(),
Partition::new(test_topic.to_string(), test_partition),
Utc::now(),
None,
None,
);
// Run checkpoint (which uses spawn_blocking for RocksDB operations)
let checkpoint_task = tokio::spawn(async move {
worker
.checkpoint_partition_cancellable(
&store,
None,
Some(&CancellationToken::new()),
Some("test"),
)
.await
});
let (heartbeat_result, checkpoint_result) = tokio::join!(heartbeat_task, checkpoint_task);
// With spawn_blocking: heartbeats continue during checkpoint
// Without spawn_blocking: heartbeats would stall until checkpoint completes
assert!(heartbeat_result.is_ok());
let final_count = heartbeat_count.load(Ordering::SeqCst);
assert!(
final_count >= 8,
"Heartbeats should continue during checkpoint (got {})",
final_count
);
assert!(checkpoint_result.unwrap().is_ok());
}
/// Test that concurrent checkpoint operations complete successfully
/// when running on the blocking thread pool.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_concurrent_checkpoint_operations_complete() {
let tmp_base_dir = TempDir::new().unwrap();
let mut handles = vec![];
// Create multiple workers for different partitions
for partition_num in 0..5 {
let tmp_checkpoint_dir = tmp_base_dir
.path()
.join(format!("checkpoints-{}", partition_num));
let tmp_store_dir = tmp_base_dir.path().join(format!("store-{}", partition_num));
let handle = tokio::spawn(async move {
let test_topic = "test_concurrent";
let store = create_test_dedup_store(&tmp_store_dir, test_topic, partition_num);
// Write some test data
for i in 0..20 {
let event = TestRawEventBuilder::new()
.distinct_id(&format!("user-p{}-{}", partition_num, i))
.token("test-token")
.event(&format!("event-{}", i))
.current_timestamp()
.build();
let key = (&event).into();
let metadata = TimestampMetadata::new(&event);
store.put_timestamp_record(&key, &metadata).unwrap();
}
let worker = CheckpointWorker::new(
partition_num as u32,
&tmp_checkpoint_dir,
"test-namespace".to_string(),
Partition::new(test_topic.to_string(), partition_num),
Utc::now(),
None,
None,
);
worker
.checkpoint_partition_cancellable(
&store,
None,
Some(&CancellationToken::new()),
Some("test"),
)
.await
});
handles.push(handle);
}
// All checkpoints should complete successfully despite limited tokio workers
for handle in handles {
let result = handle.await.unwrap();
assert!(
result.is_ok(),
"All concurrent checkpoints should complete successfully"
);
}
}
/// Test that checkpoint operations produce correct results after thread migration.
/// Verifies that Arc cloning and spawn_blocking don't corrupt data.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_checkpoint_metadata_consistency_across_threads() {
let test_topic = "test_consistency";
let test_partition = 0;
let tmp_store_dir = TempDir::new().unwrap();
let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);
// Use a fixed timestamp to ensure consistent keys across writes and reads
let fixed_timestamp = "1234567890";
// Write known data to store
let test_events = vec![
("user-1", "token-1", "event-1"),
("user-2", "token-2", "event-2"),
("user-3", "token-3", "event-3"),
];
for (user, token, event_name) in &test_events {
let event = TestRawEventBuilder::new()
.distinct_id(user)
.token(token)
.event(event_name)
.timestamp(fixed_timestamp)
.build();
let key = (&event).into();
let metadata = TimestampMetadata::new(&event);
store.put_timestamp_record(&key, &metadata).unwrap();
}
// Create checkpoint (which clones store Arc into spawn_blocking)
let tmp_checkpoint_dir = TempDir::new().unwrap();
let config = CheckpointConfig {
checkpoint_interval: Duration::from_secs(60),
local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
s3_bucket: "test-bucket".to_string(),
s3_key_prefix: "test-prefix".to_string(),
aws_region: Some("us-east-1".to_string()),
..Default::default()
};
let worker = CheckpointWorker::new(
1,
Path::new(&config.local_checkpoint_dir),
config.s3_key_prefix.clone(),
Partition::new(test_topic.to_string(), test_partition),
Utc::now(),
None,
None,
);
let result = worker
.checkpoint_partition_cancellable(
&store,
None,
Some(&CancellationToken::new()),
Some("test"),
)
.await;
assert!(result.is_ok(), "Checkpoint should succeed");
// Verify original store still has correct data (Arc wasn't corrupted)
for (user, token, event_name) in &test_events {
let event = TestRawEventBuilder::new()
.distinct_id(user)
.token(token)
.event(event_name)
.timestamp(fixed_timestamp)
.build();
let key = (&event).into();
let metadata = store.get_timestamp_record(&key).unwrap();
assert!(metadata.is_some(), "Original store should still have data");
assert_eq!(
metadata.unwrap().original_event.token.as_deref(),
Some(*token),
"Metadata should survive spawn_blocking clone"
);
}
}
/// Test that liveness handlers aren't starved during long checkpoint operations.
/// This simulates the production scenario that spawn_blocking was added to fix.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn test_liveness_check_during_checkpoint() {
use std::sync::atomic::{AtomicBool, Ordering};
let liveness_flag = Arc::new(AtomicBool::new(false));
let flag_clone = liveness_flag.clone();
let test_topic = "test_liveness";
let test_partition = 0;
let tmp_store_dir = TempDir::new().unwrap();
let store = create_test_dedup_store(tmp_store_dir.path(), test_topic, test_partition);
// Write data to make checkpoint take some time
for i in 0..200 {
let event = TestRawEventBuilder::new()
.distinct_id(&format!("user-{}", i))
.token("test-token")
.event(&format!("event-{}", i))
.current_timestamp()
.build();
let key = (&event).into();
let metadata = TimestampMetadata::new(&event);
store.put_timestamp_record(&key, &metadata).unwrap();
}
// Spawn liveness check on tokio runtime
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(20)).await;
flag_clone.store(true, Ordering::SeqCst);
});
// Start checkpoint operation
let tmp_checkpoint_dir = TempDir::new().unwrap();
let config = CheckpointConfig {
checkpoint_interval: Duration::from_secs(60),
local_checkpoint_dir: tmp_checkpoint_dir.path().to_string_lossy().to_string(),
s3_bucket: "test-bucket".to_string(),
s3_key_prefix: "test-prefix".to_string(),
aws_region: Some("us-east-1".to_string()),
..Default::default()
};
let worker = CheckpointWorker::new(
1,
Path::new(&config.local_checkpoint_dir),
config.s3_key_prefix.clone(),
Partition::new(test_topic.to_string(), test_partition),
Utc::now(),
None,
None,
);
let _result = worker
.checkpoint_partition_cancellable(
&store,
None,
Some(&CancellationToken::new()),
Some("test"),
)
.await;
// Verify liveness check completed despite checkpoint running
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(
liveness_flag.load(Ordering::SeqCst),
"Liveness check should complete while checkpoint blocks"
);
}