Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

28.0 KB
Raw Blame
use std::collections::HashMap;
use std::path::{Path, PathBuf};

use super::{CheckpointFile, CheckpointInfo, CheckpointMetadata, PlanningCancelledError};
use crate::observability::metrics::CHECKPOINT_PLAN_FILES_TOTAL;

use anyhow::{Context, Result};
use sha2::{Digest, Sha256};
use tokio_util::sync::CancellationToken;
use tracing::{debug, info};

#[derive(Debug)]
pub struct CheckpointPlan {
    pub info: CheckpointInfo,
    pub files_to_upload: Vec<LocalCheckpointFile>,
}

/// Build a checkpoint plan: `metadata` with its files tracked, plus the files to upload.
///
/// With no previous attempt, every file uploads and the plan starts a new chain. Otherwise dedup is
/// keyed on filename: SST files (immutable) are reused from the previous attempt; mutable files are
/// re-uploaded only on checksum change, and the plan stays in the previous attempt's chain. The
/// caller must pass a previous attempt of the same DB, because SST names are unique only within one
/// DB. The cancellation token, if given, is checked during the directory walk and before hashing
/// non-SST files.
pub fn plan_checkpoint(
    local_checkpoint_attempt_dir: &Path,
    metadata: CheckpointMetadata,
    remote_bucket_namespace: String,
    previous_metadata: Option<&CheckpointMetadata>,
    cancel_token: Option<&CancellationToken>,
) -> Result<CheckpointPlan> {
    ensure_not_cancelled(cancel_token, "before planning start")?;
    let mut info = CheckpointInfo::new(metadata, remote_bucket_namespace);
    let mut files_to_upload: Vec<LocalCheckpointFile> = Vec::new();

    let local_files = collect_local_files(local_checkpoint_attempt_dir, cancel_token)?;
    info!(
        "Found {} files in local checkpoint directory",
        local_files.len()
    );

    let Some(prev_meta) = previous_metadata else {
        info!("No previous checkpoint metadata - tracking all files for upload");
        for candidate in local_files {
            let remote_filepath = info.get_file_key(&candidate.filename);
            info.metadata
                .track_file(remote_filepath, candidate.checksum.clone());
            files_to_upload.push(candidate);
        }
        return Ok(CheckpointPlan {
            info,
            files_to_upload,
        });
    };

    info.metadata.full_upload_at = prev_meta.full_upload_at;
    let mut prev_file_map: HashMap<String, CheckpointFile> = HashMap::new();
    for prev_cp_file in &prev_meta.files {
        let filename = prev_cp_file
            .remote_filepath
            .rsplit('/')
            .next()
            .unwrap_or(&prev_cp_file.remote_filepath)
            .to_string();
        prev_file_map.insert(filename, prev_cp_file.clone());
    }

    info!(
        "Built file map with {} files from previous checkpoint {}",
        prev_file_map.len(),
        prev_meta.id
    );

    for candidate in local_files {
        let filename = &candidate.filename;
        if let Some(prev_file) = prev_file_map.get(filename) {
            if retain_or_replace_duplicate(&candidate, prev_file) {
                debug!("Duplicate file {} - new file will be uploaded", filename);
                metrics::counter!(CHECKPOINT_PLAN_FILES_TOTAL, "action" => "replaced").increment(1);
                let remote_filepath = info.get_file_key(filename);
                info.metadata
                    .track_file(remote_filepath, candidate.checksum.clone());
                files_to_upload.push(candidate);
            } else {
                debug!(
                    "Duplicate file {} - retaining original, skipping upload",
                    filename
                );
                metrics::counter!(CHECKPOINT_PLAN_FILES_TOTAL, "action" => "retained").increment(1);
                info.metadata.track_file(
                    prev_file.remote_filepath.clone(),
                    prev_file.checksum.clone(),
                );
            }
        } else {
            debug!("New file {} will be uploaded", filename);
            metrics::counter!(CHECKPOINT_PLAN_FILES_TOTAL, "action" => "added").increment(1);
            let remote_filepath = info.get_file_key(filename);
            info.metadata
                .track_file(remote_filepath, candidate.checksum.clone());
            files_to_upload.push(candidate);
        }
    }

    info!(
        "Checkpoint plan: {} new files, {} reused files",
        files_to_upload.len(),
        info.metadata.files.len() - files_to_upload.len(),
    );

    Ok(CheckpointPlan {
        info,
        files_to_upload,
    })
}

fn retain_or_replace_duplicate(
    candidate: &LocalCheckpointFile,
    prev_file: &CheckpointFile,
) -> bool {
    match &candidate.filename {
        f if f == "CURRENT" => true,       // always re-uploaded
        f if f.ends_with(".sst") => false, // immutable, always reused

        // RocksDB mutates these in place; re-upload only on checksum change.
        f if f.starts_with("OPTIONS-") || f.starts_with("MANIFEST-") || f.ends_with(".log") => {
            candidate.checksum != prev_file.checksum
        }
        _ => true, // bias to keeping latest files for safety
    }
}

fn collect_local_files(
    base_attempt_path: &Path,
    cancel_token: Option<&CancellationToken>,
) -> Result<Vec<LocalCheckpointFile>> {
    let mut files = Vec::new();
    let mut stack = vec![base_attempt_path.to_path_buf()];

    while let Some(current_path) = stack.pop() {
        ensure_not_cancelled(cancel_token, "during directory walk")?;
        let entries = std::fs::read_dir(&current_path)
            .with_context(|| format!("Failed to read directory: {current_path:?}"))?;

        for entry in entries {
            ensure_not_cancelled(cancel_token, "during directory walk")?;
            let entry = entry.context(format!(
                "Failed to read directory entry from {base_attempt_path:?}"
            ))?;
            let path = entry.path();

            if path.is_dir() {
                stack.push(path);
            } else {
                let candidate =
                    build_candidate_file(&path, cancel_token).context("In build_candidate_file")?;
                files.push(candidate);
            }
        }
    }

    Ok(files)
}

fn build_candidate_file(
    file_path: &Path,
    cancel_token: Option<&CancellationToken>,
) -> Result<LocalCheckpointFile> {
    let local_file_path = file_path.to_path_buf();
    let filename = file_path
        .file_name()
        .context(format!("Failed to get filename for: {file_path:?}"))?
        .to_string_lossy()
        .to_string();

    let checksum = if filename.ends_with(".sst") {
        String::default()
    } else {
        ensure_not_cancelled(cancel_token, "before hashing non-SST file")?;
        load_and_hash_file(file_path).context("In load_and_hash_file")?
    };

    Ok(LocalCheckpointFile::new(
        filename,
        checksum,
        local_file_path,
    ))
}

fn load_and_hash_file(file_path: &Path) -> Result<String> {
    let mut file = std::fs::File::open(file_path)
        .with_context(|| format!("Failed to open file for hashing: {file_path:?}"))?;

    let mut hasher = Sha256::new();
    std::io::copy(&mut file, &mut hasher)
        .with_context(|| format!("Failed to read and hash file: {file_path:?}"))?;
    let hash = hasher.finalize();
    let checksum = format!("{hash:x}");

    Ok(checksum.to_string())
}

fn ensure_not_cancelled(
    cancel_token: Option<&CancellationToken>,
    reason: &'static str,
) -> Result<()> {
    if cancel_token.is_some_and(CancellationToken::is_cancelled) {
        return Err(PlanningCancelledError {
            reason: reason.to_string(),
        }
        .into());
    }

    Ok(())
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LocalCheckpointFile {
    pub filename: String,
    pub checksum: String,
    pub local_path: PathBuf,
}

impl LocalCheckpointFile {
    pub fn new(filename: String, checksum: String, local_path: PathBuf) -> Self {
        Self {
            filename,
            checksum,
            local_path,
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use chrono::{DateTime, Duration, Utc};
    use tempfile::TempDir;
    use tokio_util::sync::CancellationToken;

    use crate::store::durability::{CheckpointLineage, PodOrdinal};
    use crate::store::DbIdentity;

    fn attempt_dir(base: &Path, checkpoint_id: &str) -> PathBuf {
        CheckpointLineage::new(PodOrdinal::STANDALONE)
            .local_dir(base)
            .join(checkpoint_id)
    }

    fn metadata_at(attempt_timestamp: DateTime<Utc>) -> CheckpointMetadata {
        CheckpointMetadata::new(
            PodOrdinal::STANDALONE,
            attempt_timestamp,
            DbIdentity::for_test("db"),
        )
    }

    fn remote_attempt_dir(namespace: &str, checkpoint_id: &str) -> String {
        format!(
            "{}{checkpoint_id}",
            CheckpointLineage::new(PodOrdinal::STANDALONE).remote_dir(namespace)
        )
    }

    #[test]
    fn plan_checkpoint_no_previous() {
        let temp_dir = TempDir::new().unwrap();
        let remote_bucket_namespace = "checkpoints";
        let attempt_timestamp = Utc::now();
        let checkpoint_id = CheckpointMetadata::generate_id(attempt_timestamp);
        let local_checkpoint_attempt_dir = attempt_dir(temp_dir.path(), &checkpoint_id);

        std::fs::create_dir_all(&local_checkpoint_attempt_dir).unwrap();
        std::fs::write(local_checkpoint_attempt_dir.join("file1.sst"), b"data1").unwrap();
        std::fs::write(local_checkpoint_attempt_dir.join("file2.sst"), b"data2").unwrap();

        let plan = plan_checkpoint(
            &local_checkpoint_attempt_dir,
            metadata_at(attempt_timestamp),
            remote_bucket_namespace.to_string(),
            None,
            None,
        )
        .unwrap();

        let expected_sst1 =
            build_candidate_file(&local_checkpoint_attempt_dir.join("file1.sst"), None).unwrap();
        let expected_sst2 =
            build_candidate_file(&local_checkpoint_attempt_dir.join("file2.sst"), None).unwrap();

        assert_eq!(plan.files_to_upload.len(), 2);
        let got_sst1 = plan
            .files_to_upload
            .iter()
            .find(|f| f.filename == "file1.sst")
            .unwrap();
        let got_sst2 = plan
            .files_to_upload
            .iter()
            .find(|f| f.filename == "file2.sst")
            .unwrap();
        assert_eq!(got_sst1, &expected_sst1);
        assert_eq!(got_sst2, &expected_sst2);

        assert_eq!(plan.info.metadata.files.len(), 2);
        let expected_remote_path = remote_attempt_dir(remote_bucket_namespace, &checkpoint_id);
        assert!(plan
            .info
            .metadata
            .files
            .iter()
            .all(|f| f.remote_filepath.starts_with(&expected_remote_path)));
        assert!(plan
            .info
            .metadata
            .files
            .iter()
            .any(|f| f.remote_filepath.ends_with("file1.sst")));
        assert!(plan
            .info
            .metadata
            .files
            .iter()
            .any(|f| f.remote_filepath.ends_with("file2.sst")));
        assert_eq!(
            plan.info.get_metadata_key(),
            format!("{expected_remote_path}/metadata.json"),
        );
        assert_eq!(plan.info.metadata.full_upload_at, attempt_timestamp);
    }

    #[test]
    fn plan_checkpoint_with_previous() {
        let temp_dir = TempDir::new().unwrap();
        let remote_bucket_namespace = "checkpoints";
        let prev_attempt_timestamp = Utc::now() - Duration::hours(1);
        let prev_checkpoint_id = CheckpointMetadata::generate_id(prev_attempt_timestamp);

        let mut prev_metadata = metadata_at(prev_attempt_timestamp);
        let chain_started = prev_attempt_timestamp - Duration::hours(5);
        prev_metadata.full_upload_at = chain_started;

        let prev_local_attempt_dir = attempt_dir(temp_dir.path(), &prev_checkpoint_id);

        std::fs::create_dir_all(&prev_local_attempt_dir).unwrap();
        std::fs::write(prev_local_attempt_dir.join("00001.sst"), b"data1").unwrap();
        std::fs::write(prev_local_attempt_dir.join("00002.sst"), b"data2").unwrap();

        let expected_prev_sst1 =
            build_candidate_file(&prev_local_attempt_dir.join("00001.sst"), None).unwrap();
        let expected_prev_sst2 =
            build_candidate_file(&prev_local_attempt_dir.join("00002.sst"), None).unwrap();

        let prev_remote_path =
            format!("{remote_bucket_namespace}/cohort_stream_state/0/{prev_checkpoint_id}");

        let prev_sst1_remote_path = format!("{prev_remote_path}/00001.sst");
        prev_metadata.track_file(
            prev_sst1_remote_path.clone(),
            expected_prev_sst1.checksum.clone(),
        );

        let prev_sst2_remote_path = format!("{prev_remote_path}/00002.sst");
        prev_metadata.track_file(
            prev_sst2_remote_path.clone(),
            expected_prev_sst2.checksum.clone(),
        );

        let attempt_timestamp = Utc::now();
        let checkpoint_id = CheckpointMetadata::generate_id(attempt_timestamp);
        let local_checkpoint_attempt_dir = attempt_dir(temp_dir.path(), &checkpoint_id);

        std::fs::create_dir_all(&local_checkpoint_attempt_dir).unwrap();
        std::fs::write(local_checkpoint_attempt_dir.join("00001.sst"), b"data1").unwrap();
        std::fs::write(local_checkpoint_attempt_dir.join("00002.sst"), b"data2").unwrap();
        std::fs::write(local_checkpoint_attempt_dir.join("00003.sst"), b"data3").unwrap();
        let expected_sst3 =
            build_candidate_file(&local_checkpoint_attempt_dir.join("00003.sst"), None).unwrap();

        let plan = plan_checkpoint(
            &local_checkpoint_attempt_dir,
            metadata_at(attempt_timestamp),
            remote_bucket_namespace.to_string(),
            Some(&prev_metadata),
            None,
        )
        .unwrap();

        assert_eq!(plan.files_to_upload.len(), 1);
        assert_eq!(
            plan.info.metadata.full_upload_at, chain_started,
            "an incremental plan stays in its baseline's chain",
        );
        let got_sst3 = plan
            .files_to_upload
            .iter()
            .find(|f| f.filename == "00003.sst")
            .unwrap();
        assert_eq!(got_sst3, &expected_sst3);

        assert_eq!(plan.info.metadata.files.len(), 3);

        let current_attempt_remote_path =
            remote_attempt_dir(remote_bucket_namespace, &checkpoint_id);

        let sst1_remote_path = format!("{prev_remote_path}/00001.sst");
        let sst2_remote_path = format!("{prev_remote_path}/00002.sst");
        let sst3_remote_path = format!("{current_attempt_remote_path}/00003.sst");

        let sst1_file_meta = plan
            .info
            .metadata
            .files
            .iter()
            .find(|f| f.remote_filepath == sst1_remote_path)
            .unwrap();

        let sst2_file_meta = plan
            .info
            .metadata
            .files
            .iter()
            .find(|f| f.remote_filepath == sst2_remote_path)
            .unwrap();

        let sst3_file_meta = plan
            .info
            .metadata
            .files
            .iter()
            .find(|f| f.remote_filepath == sst3_remote_path)
            .unwrap();

        assert_eq!(&sst1_file_meta.checksum, &expected_prev_sst1.checksum);
        assert_eq!(&sst2_file_meta.checksum, &expected_prev_sst2.checksum);
        assert_eq!(sst3_file_meta.checksum, expected_sst3.checksum);
    }

    #[test]
    fn plan_checkpoint_with_only_prev_files() {
        let temp_dir = TempDir::new().unwrap();
        let remote_bucket_namespace = "checkpoints";
        let prev_attempt_timestamp = Utc::now() - Duration::hours(1);
        let prev_checkpoint_id = CheckpointMetadata::generate_id(prev_attempt_timestamp);

        let mut prev_metadata = metadata_at(prev_attempt_timestamp);

        let prev_local_attempt_dir = attempt_dir(temp_dir.path(), &prev_checkpoint_id);

        std::fs::create_dir_all(&prev_local_attempt_dir).unwrap();
        std::fs::write(prev_local_attempt_dir.join("00001.sst"), b"data1").unwrap();
        std::fs::write(prev_local_attempt_dir.join("00002.sst"), b"data2").unwrap();
        std::fs::write(prev_local_attempt_dir.join("00003.sst"), b"data3").unwrap();

        let expected_prev_sst1 =
            build_candidate_file(&prev_local_attempt_dir.join("00001.sst"), None).unwrap();
        let expected_prev_sst2 =
            build_candidate_file(&prev_local_attempt_dir.join("00002.sst"), None).unwrap();
        let expected_prev_sst3 =
            build_candidate_file(&prev_local_attempt_dir.join("00003.sst"), None).unwrap();

        let prev_remote_path =
            format!("{remote_bucket_namespace}/cohort_stream_state/0/{prev_checkpoint_id}");
        let prev_sst1_remote_path = format!("{prev_remote_path}/00001.sst");
        let prev_sst2_remote_path = format!("{prev_remote_path}/00002.sst");
        let prev_sst3_remote_path = format!("{prev_remote_path}/00003.sst");

        prev_metadata.track_file(
            prev_sst1_remote_path.clone(),
            expected_prev_sst1.checksum.clone(),
        );
        prev_metadata.track_file(
            prev_sst2_remote_path.clone(),
            expected_prev_sst2.checksum.clone(),
        );
        prev_metadata.track_file(
            prev_sst3_remote_path.clone(),
            expected_prev_sst3.checksum.clone(),
        );

        let attempt_timestamp = Utc::now();
        let checkpoint_id = CheckpointMetadata::generate_id(attempt_timestamp);
        let local_checkpoint_attempt_dir = attempt_dir(temp_dir.path(), &checkpoint_id);

        std::fs::create_dir_all(&local_checkpoint_attempt_dir).unwrap();
        std::fs::write(local_checkpoint_attempt_dir.join("00001.sst"), b"data1").unwrap();
        std::fs::write(local_checkpoint_attempt_dir.join("00002.sst"), b"data2").unwrap();
        std::fs::write(local_checkpoint_attempt_dir.join("00003.sst"), b"data3").unwrap();

        let plan = plan_checkpoint(
            &local_checkpoint_attempt_dir,
            metadata_at(attempt_timestamp),
            remote_bucket_namespace.to_string(),
            Some(&prev_metadata),
            None,
        )
        .unwrap();

        assert!(plan.files_to_upload.is_empty());
        assert_eq!(plan.info.metadata.files.len(), 3);

        let sst1_file_meta = plan
            .info
            .metadata
            .files
            .iter()
            .find(|f| f.remote_filepath == prev_sst1_remote_path)
            .unwrap();
        let sst2_file_meta = plan
            .info
            .metadata
            .files
            .iter()
            .find(|f| f.remote_filepath == prev_sst2_remote_path)
            .unwrap();
        let sst3_file_meta = plan
            .info
            .metadata
            .files
            .iter()
            .find(|f| f.remote_filepath == prev_sst3_remote_path)
            .unwrap();

        assert_eq!(&sst1_file_meta.checksum, &expected_prev_sst1.checksum);
        assert_eq!(&sst2_file_meta.checksum, &expected_prev_sst2.checksum);
        assert_eq!(&sst3_file_meta.checksum, &expected_prev_sst3.checksum);
    }

    #[test]
    fn plan_checkpoint_with_previous_and_non_sst_types() {
        let temp_dir = TempDir::new().unwrap();
        let remote_bucket_namespace = "checkpoints";
        let prev_attempt_timestamp = Utc::now() - Duration::hours(1);
        let prev_checkpoint_id = CheckpointMetadata::generate_id(prev_attempt_timestamp);

        let mut prev_metadata = metadata_at(prev_attempt_timestamp);

        let prev_local_attempt_dir = attempt_dir(temp_dir.path(), &prev_checkpoint_id);

        std::fs::create_dir_all(&prev_local_attempt_dir).unwrap();
        std::fs::write(prev_local_attempt_dir.join("00001.sst"), b"data1").unwrap();
        std::fs::write(prev_local_attempt_dir.join("00002.sst"), b"data2").unwrap();
        std::fs::write(prev_local_attempt_dir.join("MANIFEST-000000"), b"data3").unwrap();
        std::fs::write(prev_local_attempt_dir.join("OPTIONS-000000"), b"data4").unwrap();
        std::fs::write(prev_local_attempt_dir.join("CURRENT"), b"data5").unwrap();
        std::fs::write(prev_local_attempt_dir.join("00001.log"), b"data6").unwrap();

        let expected_prev_sst1 =
            build_candidate_file(&prev_local_attempt_dir.join("00001.sst"), None).unwrap();
        let expected_prev_sst2 =
            build_candidate_file(&prev_local_attempt_dir.join("00002.sst"), None).unwrap();
        let expected_prev_manifest =
            build_candidate_file(&prev_local_attempt_dir.join("MANIFEST-000000"), None).unwrap();
        let expected_prev_options =
            build_candidate_file(&prev_local_attempt_dir.join("OPTIONS-000000"), None).unwrap();
        let expected_prev_current =
            build_candidate_file(&prev_local_attempt_dir.join("CURRENT"), None).unwrap();
        let expected_prev_log =
            build_candidate_file(&prev_local_attempt_dir.join("00001.log"), None).unwrap();

        let prev_remote_path =
            format!("{remote_bucket_namespace}/cohort_stream_state/0/{prev_checkpoint_id}");
        let prev_sst1_remote_path = format!("{prev_remote_path}/00001.sst");
        let prev_sst2_remote_path = format!("{prev_remote_path}/00002.sst");
        let prev_manifest_remote_path = format!("{prev_remote_path}/MANIFEST-000000");
        let prev_options_remote_path = format!("{prev_remote_path}/OPTIONS-000000");
        let prev_current_remote_path = format!("{prev_remote_path}/CURRENT");
        let prev_log_remote_path = format!("{prev_remote_path}/00001.log");

        prev_metadata.track_file(
            prev_sst1_remote_path.clone(),
            expected_prev_sst1.checksum.clone(),
        );
        prev_metadata.track_file(
            prev_sst2_remote_path.clone(),
            expected_prev_sst2.checksum.clone(),
        );
        prev_metadata.track_file(
            prev_manifest_remote_path.clone(),
            expected_prev_manifest.checksum.clone(),
        );
        prev_metadata.track_file(
            prev_options_remote_path.clone(),
            expected_prev_options.checksum.clone(),
        );
        prev_metadata.track_file(
            prev_current_remote_path.clone(),
            expected_prev_current.checksum.clone(),
        );
        prev_metadata.track_file(
            prev_log_remote_path.clone(),
            expected_prev_log.checksum.clone(),
        );

        let attempt_timestamp = Utc::now();
        let checkpoint_id = CheckpointMetadata::generate_id(attempt_timestamp);
        let local_checkpoint_attempt_dir = attempt_dir(temp_dir.path(), &checkpoint_id);

        std::fs::create_dir_all(&local_checkpoint_attempt_dir).unwrap();
        std::fs::write(local_checkpoint_attempt_dir.join("00001.sst"), b"data1").unwrap(); // no change, always retain
        std::fs::write(local_checkpoint_attempt_dir.join("00002.sst"), b"data2").unwrap(); // no change, always retain
        std::fs::write(local_checkpoint_attempt_dir.join("00003.sst"), b"data7").unwrap(); // new file, should upload
        std::fs::write(
            local_checkpoint_attempt_dir.join("MANIFEST-000000"),
            b"data3",
        )
        .unwrap(); // unchanged, should retain
        std::fs::write(
            local_checkpoint_attempt_dir.join("OPTIONS-000000"),
            b"data4",
        )
        .unwrap(); // unchanged, should retain
        std::fs::write(local_checkpoint_attempt_dir.join("CURRENT"), b"data5").unwrap(); // unchanged, but always uploaded
        std::fs::write(
            local_checkpoint_attempt_dir.join("00001.log"),
            b"data6_APPENDED",
        )
        .unwrap(); // mutated, should upload

        let plan = plan_checkpoint(
            &local_checkpoint_attempt_dir,
            metadata_at(attempt_timestamp),
            remote_bucket_namespace.to_string(),
            Some(&prev_metadata),
            None,
        )
        .unwrap();

        assert_eq!(plan.files_to_upload.len(), 3);
        assert!(plan
            .files_to_upload
            .iter()
            .any(|f| f.filename == "00003.sst"));
        assert!(plan
            .files_to_upload
            .iter()
            .any(|f| f.filename == "00001.log"));
        assert!(plan.files_to_upload.iter().any(|f| f.filename == "CURRENT"));

        assert_eq!(plan.info.metadata.files.len(), 7);

        // New files (sst3, CURRENT, log) use the hashed path; retained files keep the previous path.
        let current_attempt_remote_path =
            remote_attempt_dir(remote_bucket_namespace, &checkpoint_id);
        let sst1_remote_path = format!("{prev_remote_path}/00001.sst");
        let sst2_remote_path = format!("{prev_remote_path}/00002.sst");
        let sst3_remote_path = format!("{current_attempt_remote_path}/00003.sst");
        let manifest_remote_path = format!("{prev_remote_path}/MANIFEST-000000");
        let options_remote_path = format!("{prev_remote_path}/OPTIONS-000000");
        let current_remote_path = format!("{current_attempt_remote_path}/CURRENT");
        let log_remote_path = format!("{current_attempt_remote_path}/00001.log");

        assert!(plan
            .info
            .metadata
            .files
            .iter()
            .any(|f| f.remote_filepath == sst1_remote_path));
        assert!(plan
            .info
            .metadata
            .files
            .iter()
            .any(|f| f.remote_filepath == sst2_remote_path));
        assert!(plan
            .info
            .metadata
            .files
            .iter()
            .any(|f| f.remote_filepath == sst3_remote_path));
        assert!(plan
            .info
            .metadata
            .files
            .iter()
            .any(|f| f.remote_filepath == manifest_remote_path));
        assert!(plan
            .info
            .metadata
            .files
            .iter()
            .any(|f| f.remote_filepath == options_remote_path));
        assert!(plan
            .info
            .metadata
            .files
            .iter()
            .any(|f| f.remote_filepath == current_remote_path));
        assert!(plan
            .info
            .metadata
            .files
            .iter()
            .any(|f| f.remote_filepath == log_remote_path));
    }

    #[test]
    fn plan_checkpoint_cancelled_before_start() {
        let temp_dir = TempDir::new().unwrap();
        let attempt_timestamp = Utc::now();
        let checkpoint_id = CheckpointMetadata::generate_id(attempt_timestamp);
        let local_checkpoint_attempt_dir = attempt_dir(temp_dir.path(), &checkpoint_id);

        std::fs::create_dir_all(&local_checkpoint_attempt_dir).unwrap();
        std::fs::write(local_checkpoint_attempt_dir.join("00001.sst"), b"data1").unwrap();

        let cancel_token = CancellationToken::new();
        cancel_token.cancel();

        let err = plan_checkpoint(
            &local_checkpoint_attempt_dir,
            metadata_at(attempt_timestamp),
            "checkpoints".to_string(),
            None,
            Some(&cancel_token),
        )
        .unwrap_err();

        assert!(
            err.downcast_ref::<PlanningCancelledError>().is_some(),
            "error should be PlanningCancelledError: {err}"
        );
    }

    #[test]
    fn collect_local_files_cancelled_during_directory_walk() {
        let temp_dir = TempDir::new().unwrap();
        let nested_dir = temp_dir.path().join("nested");
        std::fs::create_dir_all(&nested_dir).unwrap();
        std::fs::write(nested_dir.join("00001.sst"), b"data1").unwrap();

        let cancel_token = CancellationToken::new();
        cancel_token.cancel();

        let err = collect_local_files(temp_dir.path(), Some(&cancel_token)).unwrap_err();
        assert!(
            err.downcast_ref::<PlanningCancelledError>().is_some(),
            "error should be PlanningCancelledError: {err}"
        );
    }

    #[test]
    fn build_candidate_file_cancelled_before_hashing_non_sst() {
        let temp_dir = TempDir::new().unwrap();
        let file_path = temp_dir.path().join("MANIFEST-000000");
        std::fs::write(&file_path, b"manifest-data").unwrap();

        let cancel_token = CancellationToken::new();
        cancel_token.cancel();

        let err = build_candidate_file(&file_path, Some(&cancel_token)).unwrap_err();
        assert!(
            err.downcast_ref::<PlanningCancelledError>().is_some(),
            "error should be PlanningCancelledError: {err}"
        );
    }
}