Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

11.3 KB
Raw Blame
//! Garbage collection for versioned multi-write metadata.

use std::collections::BTreeMap;
use std::sync::Arc;
use std::time::Duration;

use crate::core::errors::{Error, Result};
use crate::multibackend::constants::SEGMENT_FILE_EXTENSION;
use crate::multibackend::meta::{MetadataStore, MultiWriteWorker};
use crate::multibackend::model::{is_checkpoint_directory_name, CheckpointsManifest, DirectoryEvent, LatestCheckpoint,PartitionState, PartitionsManifest, ProtocolStatus, ScopeKey, SegmentManifest,};
use crate::multibackend::provider::MultiWriteProvider;
use tracing::info;

struct ScopeSnapshot { latest: Option<LatestCheckpoint>, segments: SegmentManifest}

/// Collects obsolete segment and checkpoint metadata after publication.
pub struct MetadataGc {
    store: Arc<MetadataStore>,
    provider: Arc<dyn MultiWriteProvider>,
}

impl MetadataGc {
    /// Create a collector over the shared metadata store and segment provider.
    pub fn new(store: Arc<MetadataStore>, provider: Arc<dyn MultiWriteProvider>) -> Self {
        Self { store, provider }
    }

    /// Collect one scope while treating concurrent account deletion as complete.
    pub async fn collect_scope(&self, scope: &ScopeKey) -> Result<()> {
        info!(
            account = %scope.account_id,
            partition = scope.partition_id,
            epoch = scope.epoch,
            "multi-write metadata gc started"
        );
        match self.collect_scope_inner(scope).await {
            Ok(()) | Err(Error::NotFound(_)) => {
                info!(
                    account = %scope.account_id,
                    partition = scope.partition_id,
                    epoch = scope.epoch,
                    "multi-write metadata gc completed"
                );
                Ok(())
            }
            Err(error) => {
                self.store.record_worker_error(MultiWriteWorker::Gc);
                Err(error)
            }
        }
    }

    /// Apply account safety checks before collecting one Stable scope.
    async fn collect_scope_inner(&self, scope: &ScopeKey) -> Result<()> {
        if self.store.protocol_status().await? != ProtocolStatus::Stable {
            return Ok(());
        }
        scope.validate()?;
        let account = self.store.update_partitions_manifest(&scope.account_id,
            |manifest| Ok(manifest.clone())).await?;
        if account.partitions.values().any(|entry| entry.state != PartitionState::Stable)
            || account.epoch != scope.epoch
            || !account.partitions.contains_key(&scope.partition_id)
        {
            return Ok(());
        }
        let stable = stable_partitions(&account);
        let positions_complete =
            directory_positions_complete(&account.directory_events, &stable, account.epoch);
        let retained = if positions_complete {
            self.collect_completed_events(&scope.account_id, &account, &stable).await?
        } else {
            Some(account.directory_events.clone())
        };
        let Some(retained) = retained else {
            return Ok(());
        };
        let allow_segments =
            positions_complete && directory_positions_complete(&retained, &stable, account.epoch);
        let latest = self.read_latest(scope).await?;
        if allow_segments {
            self.collect_segments(scope, latest.as_ref(), &retained).await?;
        }
        self.collect_checkpoints(scope).await
    }

    /// Remove completed events only if their account identity and content remain unchanged.
    async fn collect_completed_events(&self, account_id: &str,
        account: &PartitionsManifest, stable: &[u32]) -> Result<Option<Vec<DirectoryEvent>>> {
        let mut snapshots = BTreeMap::new();
        for &partition_id in stable {
            let scope = ScopeKey { account_id: account_id.to_string(),
                partition_id, epoch: account.epoch };
            snapshots.insert(partition_id, self.read_scope_snapshot(&scope).await?);
        }
        let completed = account.directory_events.iter()
            .filter(|event| event_is_complete(event, &snapshots))
            .cloned().collect::<Vec<_>>();
        let account_id = account_id.to_string();
        let expected_epoch = account.epoch;
        let expected_stable = stable.to_vec();
        self.store.update_partitions_manifest(&account_id, move |current| {
                if current.epoch != expected_epoch
                    || stable_partitions(current) != expected_stable
                {
                    return Ok(None);
                }
                current.directory_events.retain(|event|
                    !completed.iter().any(|candidate| candidate == event));
                Ok(Some(current.directory_events.clone()))
            }).await
    }

    /// Read a checkpoint pointer briefly and pair it with segment progress.
    async fn read_scope_snapshot(&self, scope: &ScopeKey) -> Result<ScopeSnapshot> {
        let latest = self.read_latest(scope).await?;
        let segments = self.provider.read_manifest(scope).await?;
        segments.validate()?;
        Ok(ScopeSnapshot { latest, segments })
    }

    /// Read the latest checkpoint while holding only its pointer Exact lock.
    async fn read_latest(&self, scope: &ScopeKey) -> Result<Option<LatestCheckpoint>> {
        let path = self.store.paths().checkpoints_manifest(
            &scope.account_id, scope.partition_id)?.0;
        let lease = self.store.pathlock_manager().acquire_exact(
            &path,
   self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)),
            None,
        ).await?;
        let operation = async {
            let pointer: CheckpointsManifest = self.store.read_json(&path).await?;
            pointer.validate()?;
            Ok(pointer.latest_checkpoint)
        }.await;
        let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from);
        finish_with_release(operation, release)
    }

    /// Prune one eligible sealed prefix and remove every unreferenced segment blob.
    async fn collect_segments(&self, scope: &ScopeKey, latest: Option<&LatestCheckpoint>,
        retained: &[DirectoryEvent]) -> Result<()> {
        let manifest = self.provider.read_manifest(scope).await?;
        manifest.validate()?;
        let removable = manifest.segments.iter().take_while(|descriptor| {
            descriptor.segment_to_seq.is_some_and(|to| {
                latest.is_some_and(|checkpoint| checkpoint.checkpoint_to_seq >= to)
                    && manifest.backend_states.values().all(|state| state.synced_seq >= to)
                    && !retained.iter().any(|event| event.positions.iter().any(|position| {
                        position.partition_id == scope.partition_id
                            && position.epoch == scope.epoch
                            && (descriptor.segment_from_seq..=to).contains(&position.seq)
                    }))
            })
        }).map(|descriptor| descriptor.path.clone()).collect::<Vec<_>>();
        if !removable.is_empty() {
            for descriptor in self.provider.prune_sealed(scope, &removable).await? {
                self.remove_segment(scope, &descriptor.path).await?;
            }
        }
        let directory = self.store.paths().segments_dir(
            &scope.account_id, scope.partition_id)?.0;
        for entry in self.store.list_directory(&directory).await? {
            if !entry.is_dir && entry.name.ends_with(SEGMENT_FILE_EXTENSION) {
                self.remove_segment(scope, &entry.name).await?;
            }
        }
        Ok(())
    }

    /// Delete one segment blob while holding its Exact path lock.
    async fn remove_segment(&self, scope: &ScopeKey, name: &str) -> Result<()> {
        let directory = self.store.paths().segments_dir(
            &scope.account_id, scope.partition_id)?.0;
        let path = format!("{directory}/{name}");
        let lease = self.store.pathlock_manager().acquire_exact(
            &path,
   self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)),
            None,
        ).await?;
        let operation = async {
            let manifest = self.provider.read_manifest(scope).await?;
            if manifest.segments.iter().any(|descriptor| descriptor.path == name) {
                return Ok(());
            }
            self.store.remove_file(&path).await
        }.await;
        let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from);
        finish_with_release(operation, release)
    }

    /// Delete non-latest checkpoint candidates in manifest, chunks, directory order.
    async fn collect_checkpoints(&self, scope: &ScopeKey) -> Result<()> {
        let directory = self.store.paths().checkpoints_dir(
            &scope.account_id, scope.partition_id)?.0;
        for entry in self.store.list_directory(&directory).await? {
            if !entry.is_dir || !is_checkpoint_directory_name(&entry.name) {
                continue;
            }
            let candidate = format!("{directory}/{}", entry.name);
            let lease = self.store.pathlock_manager().acquire_exact(
                &candidate,
       self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)),
                None,
            ).await?;
            let operation = async {
                if self.read_latest(scope).await?.as_ref().is_some_and(
                    |latest| latest.path.trim_end_matches('/') == entry.name)
                {
                    return Ok(());
                }
                self.store.remove_all(&candidate).await
            }.await;
            let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from);
            finish_with_release(operation, release)?;
        }
        Ok(())
    }
}

/// Return Stable partition identifiers in deterministic order.
fn stable_partitions(manifest: &PartitionsManifest) -> Vec<u32> {
    manifest.partitions.iter().filter_map(|(&partition_id, entry)| {
        (entry.state == PartitionState::Stable).then_some(partition_id)
    }).collect()
}

/// Return whether every event has one current-epoch position per Stable partition.
fn directory_positions_complete(
    events: &[DirectoryEvent],
    stable: &[u32],
    epoch: u64,
) -> bool {
    events.iter().all(|event| stable.iter().all(|partition_id| {
        event.positions.iter().any(|position| {
            position.partition_id == *partition_id && position.epoch == epoch
        })
    }))
}

/// Return whether every marker is absorbed and applied by every current backend.
fn event_is_complete(
    event: &DirectoryEvent,
    snapshots: &BTreeMap<u32, ScopeSnapshot>,
) -> bool {
    event.positions.iter().all(|position| snapshots.get(&position.partition_id)
        .is_some_and(|snapshot| {
            snapshot.latest.as_ref().is_some_and(
                |latest| latest.checkpoint_to_seq >= position.seq)
                && snapshot.segments.backend_states.values().all(
                    |state| state.synced_seq >= position.seq)
        }))
}

/// Preserve an operation failure while requiring lock release to succeed.
fn finish_with_release<T>(operation: Result<T>, release: Result<()>) -> Result<T> {
    match (operation, release) {
        (Ok(value), Ok(())) => Ok(value),
        (Ok(_), Err(error)) | (Err(error), Ok(())) => Err(error),
        (Err(error), Err(release_error)) => Err(Error::internal(format!(
            "{error}; lock release failed: {release_error}"
        ))),
    }
}