Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.2 KB
Raw Blame
//! Reads checkpoint candidates from S3. Deciding which candidate to restore is
//! [`recovery`](super::recovery)'s job; this module only downloads and decodes.

use std::path::Path;
use std::time::{Duration, Instant};

use chrono::{DateTime, Utc};
use tracing::info;

use super::manifest::{ManifestError, OffsetManifest, MANIFEST_FILENAME};
use super::metadata::{CheckpointMetadata, MetadataError, METADATA_FILENAME};
use super::CheckpointDownloader;
use crate::observability::metrics::CHECKPOINT_IMPORT_DURATION_SECONDS;

#[derive(Debug, thiserror::Error)]
pub enum ImportError {
    /// The attempt has no `metadata.json`: its upload failed or was cancelled before the end, since
    /// `metadata.json` is written last.
    #[error("the attempt has no {METADATA_FILENAME}, so its upload never finished")]
    Unfinished,
    #[error(transparent)]
    Download(anyhow::Error),
    #[error(transparent)]
    Metadata(#[from] MetadataError),
    #[error(transparent)]
    Manifest(#[from] ManifestError),
    #[error("the checkpoint tracks no {MANIFEST_FILENAME}")]
    NoManifest,
    #[error("downloading the checkpoint files took longer than {0:?}")]
    Timeout(Duration),
}

#[derive(Debug)]
pub struct CheckpointImporter {
    downloader: Box<dyn CheckpointDownloader>,
    import_timeout: Duration,
}

impl CheckpointImporter {
    pub fn new(downloader: Box<dyn CheckpointDownloader>, import_timeout: Duration) -> Self {
        Self {
            downloader,
            import_timeout,
        }
    }

    /// Metadata keys of the lineage's attempts inside the listing window that ends at `now`, newest
    /// first.
    pub async fn candidates(&self, now: DateTime<Utc>) -> anyhow::Result<Vec<String>> {
        self.downloader.list_recent_checkpoints(now).await
    }

    pub async fn metadata(&self, metadata_key: &str) -> Result<CheckpointMetadata, ImportError> {
        let bytes = self
            .downloader
            .download_file(metadata_key)
            .await
            .map_err(|err| match err.downcast_ref::<object_store::Error>() {
                Some(object_store::Error::NotFound { .. }) => ImportError::Unfinished,
                _ => ImportError::Download(err),
            })?;
        Ok(CheckpointMetadata::from_json_bytes(&bytes)?)
    }

    /// The candidate's `offsets.json`, downloaded alone so a candidate is judged before its bulk
    /// download.
    pub async fn manifest(
        &self,
        metadata: &CheckpointMetadata,
    ) -> Result<OffsetManifest, ImportError> {
        let suffix = format!("/{MANIFEST_FILENAME}");
        let key = metadata
            .files
            .iter()
            .find(|file| file.remote_filepath.ends_with(&suffix))
            .ok_or(ImportError::NoManifest)?;
        let bytes = self
            .downloader
            .download_file(&key.remote_filepath)
            .await
            .map_err(ImportError::Download)?;
        Ok(OffsetManifest::decode(&bytes)?)
    }

    /// Downloads every file the candidate tracks into `stage`, which the caller owns and removes on
    /// failure.
    pub async fn fetch_files(
        &self,
        metadata: &CheckpointMetadata,
        stage: &Path,
    ) -> Result<(), ImportError> {
        let started = Instant::now();
        let keys: Vec<String> = metadata
            .files
            .iter()
            .map(|file| file.remote_filepath.clone())
            .collect();
        let download = self.downloader.download_files(&keys, stage);
        let result = match tokio::time::timeout(self.import_timeout, download).await {
            Ok(Ok(())) => Ok(()),
            Ok(Err(err)) => Err(ImportError::Download(err)),
            Err(_elapsed) => Err(ImportError::Timeout(self.import_timeout)),
        };
        let label = if result.is_ok() { "success" } else { "failed" };
        metrics::histogram!(CHECKPOINT_IMPORT_DURATION_SECONDS, "result" => label)
            .record(started.elapsed().as_secs_f64());
        if result.is_ok() {
            info!(
                checkpoint = %metadata.id,
                files = keys.len(),
                elapsed_secs = started.elapsed().as_secs_f64(),
                "downloaded checkpoint files into the stage",
            );
        }
        result
    }
}