Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

23.6 KB
Raw Blame
use std::sync::Arc;
use std::time::{Duration, Instant};

use anyhow::Context;
use async_trait::async_trait;
use aws_sdk_s3::error::ProvideErrorMetadata;
use aws_sdk_s3::primitives::ByteStream;
use etcd_client::EventType;
use personhog_coordination::store::PersonhogStore;
use serde::{Deserialize, Serialize};
use uuid::Uuid;

use crate::changelog;
use crate::etcd::EtcdHandle;

const RETRY_INITIAL: Duration = Duration::from_secs(1);
const RETRY_MAX: Duration = Duration::from_secs(60);

#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct BackupVersion {
    pub version_id: String,
    pub last_modified: Option<String>,
    pub is_latest: bool,
    pub size: i64,
}

#[derive(Debug, Clone, PartialEq)]
pub struct StoredObject {
    pub body: Vec<u8>,
    pub etag: String,
}

#[derive(Debug, Clone, PartialEq)]
pub enum Precondition {
    Absent,
    Matches(String),
}

#[derive(Debug, Clone, Copy, PartialEq)]
pub enum PutOutcome {
    Written,
    Conflict,
}

#[async_trait]
pub trait BackupStore: Send + Sync {
    /// The current version, or the named one; `None` when it does not exist.
    async fn get(
        &self,
        key: &str,
        version_id: Option<&str>,
    ) -> anyhow::Result<Option<StoredObject>>;
    async fn put(
        &self,
        key: &str,
        value: &[u8],
        precondition: Precondition,
    ) -> anyhow::Result<PutOutcome>;
    /// Newest first.
    async fn versions(&self, key: &str) -> anyhow::Result<Vec<BackupVersion>>;
}

pub struct S3BackupStore {
    client: aws_sdk_s3::Client,
    bucket: String,
}

impl S3BackupStore {
    pub fn new(client: aws_sdk_s3::Client, bucket: String) -> Self {
        Self { client, bucket }
    }

    pub async fn from_env(bucket: String) -> Self {
        let config = aws_config::load_defaults(aws_config::BehaviorVersion::latest()).await;
        // Local S3 (an `AWS_ENDPOINT_URL` such as the dev stack's SeaweedFS)
        // serves buckets by path, not by virtual host.
        let s3_config = aws_sdk_s3::config::Builder::from(&config)
            .force_path_style(config.endpoint_url().is_some())
            .build();
        Self::new(aws_sdk_s3::Client::from_conf(s3_config), bucket)
    }
}

#[async_trait]
impl BackupStore for S3BackupStore {
    async fn get(
        &self,
        key: &str,
        version_id: Option<&str>,
    ) -> anyhow::Result<Option<StoredObject>> {
        let result = self
            .client
            .get_object()
            .bucket(&self.bucket)
            .key(key)
            .set_version_id(version_id.map(str::to_string))
            .send()
            .await;
        let object = match result {
            Ok(object) => object,
            Err(e) if matches!(e.code(), Some("NoSuchKey" | "NoSuchVersion")) => return Ok(None),
            Err(e) => {
                return Err(anyhow::Error::from(e.into_service_error()))
                    .with_context(|| format!("get s3://{}/{key}", self.bucket))
            }
        };
        let etag = object
            .e_tag()
            .with_context(|| format!("s3://{}/{key} has no ETag", self.bucket))?
            .to_string();
        let body = object.body.collect().await.context("read backup body")?;
        Ok(Some(StoredObject {
            body: body.into_bytes().to_vec(),
            etag,
        }))
    }

    async fn put(
        &self,
        key: &str,
        value: &[u8],
        precondition: Precondition,
    ) -> anyhow::Result<PutOutcome> {
        let request = self
            .client
            .put_object()
            .bucket(&self.bucket)
            .key(key)
            .content_type("application/json")
            .body(ByteStream::from(value.to_vec()));
        let request = match precondition {
            Precondition::Absent => request.if_none_match("*"),
            Precondition::Matches(etag) => request.if_match(etag),
        };
        match request.send().await {
            Ok(_) => Ok(PutOutcome::Written),
            // S3 answers 409 instead of 412 when another conditional upload
            // of the key is in flight.
            Err(e)
                if matches!(
                    e.code(),
                    Some("PreconditionFailed" | "ConditionalRequestConflict")
                ) =>
            {
                Ok(PutOutcome::Conflict)
            }
            Err(e) => Err(anyhow::Error::from(e.into_service_error()))
                .with_context(|| format!("put s3://{}/{key}", self.bucket)),
        }
    }

    async fn versions(&self, key: &str) -> anyhow::Result<Vec<BackupVersion>> {
        let resp = self
            .client
            .list_object_versions()
            .bucket(&self.bucket)
            .prefix(key)
            .send()
            .await
            .map_err(|e| anyhow::Error::from(e.into_service_error()))
            .with_context(|| format!("list versions of s3://{}/{key}", self.bucket))?;
        let mut versions: Vec<BackupVersion> = resp
            .versions()
            .iter()
            .filter(|v| v.key() == Some(key))
            .filter_map(|v| {
                Some(BackupVersion {
                    version_id: v.version_id()?.to_string(),
                    last_modified: v.last_modified().map(|t| t.to_string()),
                    is_latest: v.is_latest().unwrap_or(false),
                    size: v.size().unwrap_or(0),
                })
            })
            .collect();
        // Two writes can share a timestamp, so the current version leads explicitly.
        versions.sort_by(|a, b| {
            b.is_latest
                .cmp(&a.is_latest)
                .then_with(|| b.last_modified.cmp(&a.last_modified))
        });
        Ok(versions)
    }
}

/// The stored form of a config. The nonce gives every upload distinct bytes,
/// and so a distinct ETag: the config alone can return to earlier bytes, and
/// an ETag condition would then pass for an object written since it was read.
#[derive(Serialize, Deserialize)]
struct Envelope {
    nonce: String,
    config: String,
}

fn wrap_config(config: &[u8]) -> anyhow::Result<Vec<u8>> {
    let envelope = Envelope {
        nonce: Uuid::new_v4().to_string(),
        config: String::from_utf8(config.to_vec()).context("config/changelog is not UTF-8")?,
    };
    Ok(serde_json::to_vec(&envelope)?)
}

pub fn unwrap_config(body: &[u8]) -> anyhow::Result<Vec<u8>> {
    let envelope: Envelope =
        serde_json::from_slice(body).context("the backup is not a backup envelope")?;
    Ok(envelope.config.into_bytes())
}

fn record(outcome: &str) {
    common_metrics::inc(
        "ingestion_control_plane_changelog_backups_total",
        &[("outcome".to_string(), outcome.to_string())],
        1,
    );
}

fn set_in_sync(in_sync: bool) {
    common_metrics::gauge(
        "ingestion_control_plane_changelog_backup_in_sync",
        &[],
        if in_sync { 1.0 } else { 0.0 },
    );
}

pub fn object_key(prefix: &str) -> String {
    format!("{}/config-changelog.json", prefix.trim_matches('/'))
}

/// The backup is read before etcd, and the upload requires that backup to
/// still be current, so an upload never replaces a newer config.
async fn sync(store: &PersonhogStore, backups: &dyn BackupStore, key: &str) -> anyhow::Result<()> {
    let backup = backups.get(key, None).await?;
    let (config, _) = store.get_changelog_config_raw_with_revision().await?;
    let Some(config) = config else {
        set_in_sync(true);
        return Ok(());
    };
    let precondition = match backup {
        Some(object) if unwrap_config(&object.body).ok().as_deref() == Some(config.as_slice()) => {
            record("unchanged");
            set_in_sync(true);
            return Ok(());
        }
        Some(object) => Precondition::Matches(object.etag),
        None => Precondition::Absent,
    };
    match backups
        .put(key, &wrap_config(&config)?, precondition)
        .await?
    {
        PutOutcome::Written => {
            record("uploaded");
            set_in_sync(true);
            tracing::info!(key, "backed up config/changelog");
            Ok(())
        }
        PutOutcome::Conflict => {
            record("conflict");
            anyhow::bail!("the backup changed after it was read")
        }
    }
}

/// A deletion is never backed up, because the last good copy is what a
/// rebuild restores.
async fn watch_once(
    store: &PersonhogStore,
    backups: &dyn BackupStore,
    key: &str,
) -> anyhow::Result<()> {
    let (_, revision) = store.get_changelog_config_raw_with_revision().await?;
    sync(store, backups, key).await?;
    let mut stream = store.watch_changelog_config_from(revision + 1).await?;
    while let Some(response) = stream.message().await? {
        if response.canceled() {
            anyhow::bail!(
                "config/changelog watch cancelled by etcd: {}",
                response.cancel_reason()
            );
        }
        for event in response.events() {
            match event.event_type() {
                EventType::Put => sync(store, backups, key).await?,
                EventType::Delete => {
                    record("deleted_skipped");
                    tracing::warn!(key, "config/changelog was deleted; keeping the last backup");
                }
            }
        }
    }
    anyhow::bail!("config/changelog watch ended")
}

pub async fn run(etcd: Arc<EtcdHandle>, prefix: String, backups: Arc<dyn BackupStore>) {
    let key = object_key(&prefix);
    let mut delay = RETRY_INITIAL;
    set_in_sync(false);
    loop {
        let started = Instant::now();
        let attempt = async {
            let client = etcd.client().await?;
            watch_once(&changelog::store(&client, &prefix), backups.as_ref(), &key).await
        };
        if let Err(e) = attempt.await {
            record("error");
            set_in_sync(false);
            tracing::warn!(key, error = %format!("{e:#}"), "config/changelog backup failed; retrying");
        }
        if started.elapsed() > RETRY_MAX {
            delay = RETRY_INITIAL;
        }
        tokio::time::sleep(delay).await;
        delay = (delay * 2).min(RETRY_MAX);
    }
}

#[cfg(test)]
pub(crate) mod tests {
    use std::collections::HashMap;
    use std::hash::{DefaultHasher, Hash, Hasher};
    use std::sync::atomic::{AtomicBool, Ordering};
    use std::sync::Mutex;

    use envconfig::Envconfig;
    use tokio::sync::Notify;

    use super::*;
    use crate::config::Config;

    /// Versions in insertion order, the last one current. Like S3, the ETag
    /// is a hash of the body, so identical bodies share an ETag.
    #[derive(Default)]
    pub(crate) struct MemoryBackups {
        pub(crate) versions: Mutex<Vec<Vec<u8>>>,
    }

    fn etag(body: &[u8]) -> String {
        let mut hasher = DefaultHasher::new();
        body.hash(&mut hasher);
        format!("{:x}", hasher.finish())
    }

    impl MemoryBackups {
        fn configs(&self) -> Vec<Vec<u8>> {
            let versions = self.versions.lock().unwrap();
            versions.iter().map(|v| unwrap_config(v).unwrap()).collect()
        }
    }

    #[async_trait]
    impl BackupStore for MemoryBackups {
        async fn get(
            &self,
            _key: &str,
            version_id: Option<&str>,
        ) -> anyhow::Result<Option<StoredObject>> {
            let versions = self.versions.lock().unwrap();
            let body = match version_id {
                None => versions.last(),
                Some(id) => id.parse::<usize>().ok().and_then(|i| versions.get(i)),
            };
            Ok(body.map(|body| StoredObject {
                body: body.clone(),
                etag: etag(body),
            }))
        }

        async fn put(
            &self,
            _key: &str,
            value: &[u8],
            precondition: Precondition,
        ) -> anyhow::Result<PutOutcome> {
            let mut versions = self.versions.lock().unwrap();
            let current = versions.last().map(|body| etag(body));
            let holds = match precondition {
                Precondition::Absent => current.is_none(),
                Precondition::Matches(expected) => current == Some(expected),
            };
            if !holds {
                return Ok(PutOutcome::Conflict);
            }
            versions.push(value.to_vec());
            Ok(PutOutcome::Written)
        }

        async fn versions(&self, _key: &str) -> anyhow::Result<Vec<BackupVersion>> {
            let versions = self.versions.lock().unwrap();
            Ok((0..versions.len())
                .rev()
                .map(|i| BackupVersion {
                    version_id: i.to_string(),
                    last_modified: None,
                    is_latest: i + 1 == versions.len(),
                    size: versions[i].len() as i64,
                })
                .collect())
        }
    }

    #[derive(Clone, Copy, Debug, PartialEq)]
    enum Stall {
        BeforeRead,
        BeforeUpload,
    }

    struct Stalled {
        inner: Arc<MemoryBackups>,
        at: Stall,
        stalled: AtomicBool,
        arrived: Notify,
        release: Notify,
    }

    impl Stalled {
        fn new(inner: Arc<MemoryBackups>, at: Stall) -> Arc<Self> {
            Arc::new(Self {
                inner,
                at,
                stalled: AtomicBool::new(false),
                arrived: Notify::new(),
                release: Notify::new(),
            })
        }

        async fn stall(&self, at: Stall) {
            if at == self.at && !self.stalled.swap(true, Ordering::SeqCst) {
                self.arrived.notify_one();
                self.release.notified().await;
            }
        }
    }

    #[async_trait]
    impl BackupStore for Stalled {
        async fn get(
            &self,
            key: &str,
            version_id: Option<&str>,
        ) -> anyhow::Result<Option<StoredObject>> {
            self.stall(Stall::BeforeRead).await;
            self.inner.get(key, version_id).await
        }

        async fn put(
            &self,
            key: &str,
            value: &[u8],
            precondition: Precondition,
        ) -> anyhow::Result<PutOutcome> {
            self.stall(Stall::BeforeUpload).await;
            self.inner.put(key, value, precondition).await
        }

        async fn versions(&self, key: &str) -> anyhow::Result<Vec<BackupVersion>> {
            self.inner.versions(key).await
        }
    }

    pub(crate) async fn test_etcd(name: &str) -> (Arc<EtcdHandle>, String) {
        let nanos = std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap()
            .as_nanos();
        let env = HashMap::from([(
            "ETCD_ENDPOINTS".to_string(),
            "http://localhost:2379".to_string(),
        )]);
        let config = Config::init_from_hashmap(&env).unwrap();
        let etcd = Arc::new(EtcdHandle::from_config(&config).unwrap());
        (etcd, format!("/test-control-plane-{name}-{nanos}/"))
    }

    async fn local_s3() -> S3BackupStore {
        use aws_sdk_s3::config::{Credentials, Region};
        use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};

        const BUCKET: &str = "control-plane-backup-test";
        let config = aws_sdk_s3::Config::builder()
            .behavior_version(aws_config::BehaviorVersion::latest())
            .region(Region::new("us-east-1"))
            .endpoint_url("http://localhost:19000")
            .credentials_provider(Credentials::new(
                "object_storage_root_user",
                "object_storage_root_password",
                None,
                None,
                "test",
            ))
            .force_path_style(true)
            .build();
        let client = aws_sdk_s3::Client::from_conf(config);
        // Creating an existing bucket fails; the versioning call below is what must succeed.
        drop(client.create_bucket().bucket(BUCKET).send().await);
        client
            .put_bucket_versioning()
            .bucket(BUCKET)
            .versioning_configuration(
                VersioningConfiguration::builder()
                    .status(BucketVersioningStatus::Enabled)
                    .build(),
            )
            .send()
            .await
            .expect("local S3 at localhost:19000");
        S3BackupStore::new(client, BUCKET.to_string())
    }

    #[tokio::test]
    async fn s3_store_reads_versions_and_enforces_upload_conditions() {
        let store = local_s3().await;
        let nanos = std::time::SystemTime::now()
            .duration_since(std::time::UNIX_EPOCH)
            .unwrap()
            .as_nanos();
        let key = format!("test-{nanos}/config-changelog.json");
        assert_eq!(store.get(&key, None).await.unwrap(), None);

        let put = |value: &'static [u8], precondition| store.put(&key, value, precondition);
        assert_eq!(
            put(b"first", Precondition::Absent).await.unwrap(),
            PutOutcome::Written
        );
        assert_eq!(
            put(b"again", Precondition::Absent).await.unwrap(),
            PutOutcome::Conflict
        );
        let first = store.get(&key, None).await.unwrap().unwrap();
        assert_eq!(
            put(b"second", Precondition::Matches(first.etag.clone()))
                .await
                .unwrap(),
            PutOutcome::Written
        );
        assert_eq!(
            put(b"stale", Precondition::Matches(first.etag))
                .await
                .unwrap(),
            PutOutcome::Conflict
        );

        assert_eq!(
            store.get(&key, None).await.unwrap().unwrap().body,
            b"second"
        );
        let versions = store.versions(&key).await.unwrap();
        assert_eq!(versions.len(), 2);
        assert!(versions[0].is_latest);
        let earlier = store
            .get(&key, Some(&versions[1].version_id))
            .await
            .unwrap();
        assert_eq!(earlier.unwrap().body, b"first");
    }

    async fn wait_for_versions(backups: &MemoryBackups, count: usize) {
        let deadline = Instant::now() + Duration::from_secs(10);
        while backups.versions.lock().unwrap().len() < count {
            assert!(Instant::now() < deadline, "expected {count} backups");
            tokio::time::sleep(Duration::from_millis(20)).await;
        }
    }

    #[tokio::test]
    async fn every_change_is_backed_up_and_a_deletion_never_is() {
        let (etcd, prefix) = test_etcd("backup-watch").await;
        let store = changelog::store(&etcd.client().await.unwrap(), &prefix);
        assert!(store
            .seed_changelog_config("personhog_updates_64", 64, "personhog-writer")
            .await
            .unwrap());
        let (seeded, _) = store
            .get_changelog_config_raw_with_revision()
            .await
            .unwrap();
        let seeded = seeded.unwrap();
        let backups = Arc::new(MemoryBackups::default());
        let watcher = tokio::spawn(run(etcd.clone(), prefix.clone(), backups.clone()));
        wait_for_versions(&backups, 1).await;

        let inner = store.inner();
        let key = format!("{}config/changelog", inner.prefix());
        inner.put_raw(&key, seeded.clone()).await.unwrap();
        let moved = br#"{"topic":"personhog_updates_128","partitions":128,"epoch":2}"#.to_vec();
        inner.put_raw(&key, moved.clone()).await.unwrap();
        wait_for_versions(&backups, 2).await;
        inner.delete(&key).await.unwrap();
        // Events arrive in order, so this one being backed up proves the
        // deletion before it was handled.
        let reseeded = br#"{"topic":"personhog_updates_128","partitions":128,"epoch":3}"#.to_vec();
        inner.put_raw(&key, reseeded.clone()).await.unwrap();
        wait_for_versions(&backups, 3).await;
        watcher.abort();

        assert_eq!(backups.configs(), vec![seeded, moved, reseeded]);
    }

    fn spawn_sync(
        client: &etcd_client::Client,
        prefix: &str,
        backups: Arc<Stalled>,
    ) -> tokio::task::JoinHandle<anyhow::Result<()>> {
        let store = changelog::store(client, prefix);
        let key = object_key(prefix);
        tokio::spawn(async move { sync(&store, backups.as_ref(), &key).await })
    }

    #[tokio::test]
    async fn a_stalled_pod_never_replaces_a_newer_backup() {
        for at in [Stall::BeforeRead, Stall::BeforeUpload] {
            let (etcd, prefix) = test_etcd("backup-race").await;
            let client = etcd.client().await.unwrap();
            let store = changelog::store(&client, &prefix);
            let inner = store.inner();
            let etcd_key = format!("{}config/changelog", inner.prefix());
            let key = object_key(&prefix);
            let without_group =
                br#"{"topic":"personhog_updates_64","partitions":64,"epoch":1,"writer_group":"personhog-writer","draining":[]}"#.to_vec();
            let with_group =
                br#"{"topic":"personhog_updates_64","partitions":64,"epoch":1,"writer_group":"personhog-writer","retire_groups":["relay"],"draining":[]}"#.to_vec();
            let backups = Arc::new(MemoryBackups::default());
            inner
                .put_raw(&etcd_key, without_group.clone())
                .await
                .unwrap();
            sync(&store, backups.as_ref(), &key).await.unwrap();

            inner.put_raw(&etcd_key, with_group.clone()).await.unwrap();
            let stalled = Stalled::new(backups.clone(), at);
            let pod_a = spawn_sync(&client, &prefix, stalled.clone());
            stalled.arrived.notified().await;

            // Pod B backs up the added group; the group is removed again,
            // which restores the original bytes, and pod C backs that up.
            sync(&store, backups.as_ref(), &key).await.unwrap();
            inner
                .put_raw(&etcd_key, without_group.clone())
                .await
                .unwrap();
            sync(&store, backups.as_ref(), &key).await.unwrap();
            stalled.release.notify_one();
            // Pod A either finds the backup current or has its upload refused.
            let _refused = pod_a.await.unwrap();

            assert_eq!(
                backups.configs().last(),
                Some(&without_group),
                "stalled {at:?}"
            );
        }
    }

    #[tokio::test]
    async fn a_refused_upload_fails_and_the_next_attempt_backs_up_the_latest_config() {
        let (etcd, prefix) = test_etcd("backup-retry").await;
        let client = etcd.client().await.unwrap();
        let store = changelog::store(&client, &prefix);
        let inner = store.inner();
        let etcd_key = format!("{}config/changelog", inner.prefix());
        let config = |epoch: u64| {
            format!(r#"{{"topic":"personhog_updates_64","partitions":64,"epoch":{epoch},"writer_group":"personhog-writer","draining":[]}}"#)
                .into_bytes()
        };
        let backups = Arc::new(MemoryBackups::default());
        inner.put_raw(&etcd_key, config(1)).await.unwrap();
        sync(&store, backups.as_ref(), &object_key(&prefix))
            .await
            .unwrap();

        inner.put_raw(&etcd_key, config(2)).await.unwrap();
        let w = Stalled::new(backups.clone(), Stall::BeforeUpload);
        let pod_w = spawn_sync(&client, &prefix, w.clone());
        w.arrived.notified().await;
        inner.put_raw(&etcd_key, config(3)).await.unwrap();
        let a = Stalled::new(backups.clone(), Stall::BeforeUpload);
        let pod_a = spawn_sync(&client, &prefix, a.clone());
        a.arrived.notified().await;

        // W's older upload lands first, so A's is refused.
        w.release.notify_one();
        pod_w.await.unwrap().unwrap();
        a.release.notify_one();
        assert!(pod_a.await.unwrap().is_err(), "the refusal is reported");
        assert_eq!(backups.configs().last(), Some(&config(2)));

        sync(&store, backups.as_ref(), &object_key(&prefix))
            .await
            .unwrap();

        assert_eq!(backups.configs().last(), Some(&config(3)));
    }
}