Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

9.8 KB
Raw Blame
//! Logical path mapping and account partition routing.

use std::collections::HashSet;
use std::sync::Arc;

use xxhash_rust::xxh3::xxh3_64;

use crate::core::errors::{Error, Result};
use crate::multibackend::constants::{
    CHECKPOINTS_DIR, MANIFEST_FILE, MULTIWRITE_MOUNT_PREFIX, MULTIWRITE_PROTOCOL_FILE,
    PARTITIONS_DIR, SEGMENTS_DIR, SYSTEM_DIR, VBUCKETS,
};
use crate::multibackend::meta::MetadataStore;
use crate::multibackend::model::{PartitionContext, PartitionsManifest, RouteEntry, ScopeKey};

/// Maps canonical logical metadata paths to raw backend paths.
#[derive(Debug, Clone)]
pub struct MultiWritePaths {
    mount_prefix: String,
}

impl MultiWritePaths {
    /// Create a path mapper for one canonical absolute mount prefix.
    pub fn new(mount_prefix: &str) -> Result<Self> {
        validate_canonical_path(mount_prefix)?;
        if mount_prefix != MULTIWRITE_MOUNT_PREFIX {
            return Err(Error::invalid_path(format!(
                "unsupported multi-write mount prefix: {mount_prefix}"
            )));
        }
        Ok(Self {
            mount_prefix: mount_prefix.to_string(),
        })
    }

    /// Return logical and backend paths for the mount protocol manifest.
    pub fn mount_protocol(&self) -> (String, String) {
        self.pair(&format!(
            "{}/{SYSTEM_DIR}/{MULTIWRITE_PROTOCOL_FILE}",
            self.mount_prefix
        ))
    }

    /// Return logical and backend paths for an account partitions directory.
    pub fn account_partitions(&self, account_id: &str) -> Result<(String, String)> {
        validate_account(account_id)?;
        Ok(self.pair(&format!(
            "{}/{account_id}/{SYSTEM_DIR}/{PARTITIONS_DIR}",
            self.mount_prefix
        )))
    }

    /// Return logical and backend paths for an account partitions manifest.
    pub fn account_manifest(&self, account_id: &str) -> Result<(String, String)> {
        let (logical, _) = self.account_partitions(account_id)?;
        Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}")))
    }

    /// Return logical and backend paths for one partition manifest.
    pub fn partition_manifest(
        &self,
        account_id: &str,
        partition_id: u32,
    ) -> Result<(String, String)> {
        let (logical, _) = self.account_partitions(account_id)?;
        Ok(self.pair(&format!("{logical}/{partition_id}/{MANIFEST_FILE}")))
    }

    /// Return logical and backend paths for one partition segments directory.
    pub fn segments_dir(&self, account_id: &str, partition_id: u32) -> Result<(String, String)> {
        let (logical, _) = self.account_partitions(account_id)?;
        Ok(self.pair(&format!("{logical}/{partition_id}/{SEGMENTS_DIR}")))
    }

    /// Return logical and backend paths for one segment manifest.
    pub fn segment_manifest(
        &self,
        account_id: &str,
        partition_id: u32,
    ) -> Result<(String, String)> {
        let (logical, _) = self.segments_dir(account_id, partition_id)?;
        Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}")))
    }

    /// Return logical and backend paths for one partition checkpoints directory.
    pub fn checkpoints_dir(&self, account_id: &str, partition_id: u32) -> Result<(String, String)> {
        let (logical, _) = self.account_partitions(account_id)?;
        Ok(self.pair(&format!("{logical}/{partition_id}/{CHECKPOINTS_DIR}")))
    }

    /// Return logical and backend paths for one checkpoints manifest.
    pub fn checkpoints_manifest(
        &self,
        account_id: &str,
        partition_id: u32,
    ) -> Result<(String, String)> {
        let (logical, _) = self.checkpoints_dir(account_id, partition_id)?;
        Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}")))
    }

    /// Convert a canonical logical account path to its raw backend path.
    pub fn backend_path(&self, account_id: &str, logical_path: &str) -> Result<String> {
        validate_account(account_id)?;
        let account_root = format!("{}/{account_id}", self.mount_prefix);
        if logical_path != account_root
            && !logical_path
                .strip_prefix(&account_root)
                .is_some_and(|suffix| suffix.starts_with('/'))
        {
            return Err(Error::invalid_path(format!(
                "logical path does not belong to account '{account_id}': {logical_path}"
            )));
        }
        self.raw_backend_path(logical_path)
    }

    /// Convert any canonical logical mount path to its raw backend path.
    pub fn raw_backend_path(&self, logical_path: &str) -> Result<String> {
        validate_canonical_path(logical_path)?;
        let suffix = logical_path
            .strip_prefix(&self.mount_prefix)
            .filter(|suffix| suffix.starts_with('/'))
            .ok_or_else(|| {
                Error::invalid_path(format!(
                    "logical path is outside mount '{}': {logical_path}",
                    self.mount_prefix
                ))
            })?;
        Ok(suffix.to_string())
    }

    /// Hash a canonical full logical account path into its fixed vbucket.
    pub fn vbucket(&self, account_id: &str, logical_path: &str) -> Result<u32> {
        self.backend_path(account_id, logical_path)?;
        Ok((xxh3_64(logical_path.as_bytes()) & u64::from(VBUCKETS - 1)) as u32)
    }

    /// Pair one validated logical path with its raw backend form.
    fn pair(&self, logical_path: &str) -> (String, String) {
        (
            logical_path.to_string(),
            logical_path[self.mount_prefix.len()..].to_string(),
        )
    }
}

/// Routes account paths through the current persisted partitions manifest.
pub struct AccountRouter {
    store: Arc<MetadataStore>,
}

impl AccountRouter {
    /// Create an account router backed by the supplied metadata store.
    pub fn new(store: Arc<MetadataStore>) -> Self {
        Self { store }
    }

    /// Route one canonical logical account path to its partition context.
    pub async fn route(&self, account_id: &str, logical_path: &str) -> Result<PartitionContext> {
        let bucket = self.store.paths().vbucket(account_id, logical_path)?;
        let manifest_path = self.store.paths().account_manifest(account_id)?.0;
        let manifest: PartitionsManifest = self.store.read_json(&manifest_path).await?;
        manifest.validate()?;
        let index = manifest.routes.partition_point(|route| route.end < bucket);
        let route = manifest
            .routes
            .get(index)
            .ok_or_else(|| Error::Serialization("vbucket route is missing".to_string()))?;
        let segment_dir = self
            .store
            .paths()
            .segments_dir(account_id, route.partition)?
            .1;
        Ok(PartitionContext {
            scope: ScopeKey {
                account_id: account_id.to_string(),
                partition_id: route.partition,
                epoch: manifest.epoch,
            },
            segment_dir,
        })
    }
}

/// Validate compressed complete routes against the known partition identifiers.
pub fn validate_routes(
    routes: &[RouteEntry],
    partition_ids: impl IntoIterator<Item = u32>,
) -> Result<()> {
    let partition_ids = partition_ids.into_iter().collect::<HashSet<_>>();
    let mut next = 0;
    let mut previous_owner = None;
    for route in routes {
        route.validate()?;
        if route.start != next {
            return Err(Error::Serialization(
                "routes are not contiguous".to_string(),
            ));
        }
        if !partition_ids.contains(&route.partition) {
            return Err(Error::Serialization("route owner is unknown".to_string()));
        }
        if previous_owner == Some(route.partition) {
            return Err(Error::Serialization(
                "adjacent routes have the same owner".to_string(),
            ));
        }
        next = route.end + 1;
        previous_owner = Some(route.partition);
    }
    if next != VBUCKETS {
        return Err(Error::Serialization(
            "routes do not cover all vbuckets".to_string(),
        ));
    }
    Ok(())
}

/// Build deterministic balanced routes for the requested initial partition count.
pub fn build_initial_routes(partition_count: u32) -> Result<Vec<RouteEntry>> {
    if !(1..=VBUCKETS).contains(&partition_count) {
        return Err(Error::Serialization(format!(
            "partition count must be between 1 and {VBUCKETS}"
        )));
    }
    let owners = (0..VBUCKETS)
        .map(|bucket| { ((u64::from(bucket) * u64::from(partition_count)) / u64::from(VBUCKETS)) as u32})
        .collect::<Vec<_>>();
    let routes = compress_owners(&owners);
    validate_routes(&routes, 0..partition_count)?;
    Ok(routes)
}

/// Compress a complete owner table into inclusive route ranges.
pub(crate) fn compress_owners(owners: &[u32]) -> Vec<RouteEntry> {
    let mut routes = Vec::new();
    let mut start = 0;
    for index in 1..=owners.len() {
        if index == owners.len() || owners[index] != owners[start] {
            routes.push(RouteEntry {
                start: start as u32,
                end: index as u32 - 1,
                partition: owners[start],
            });
            start = index;
        }
    }
    routes
}

/// Validate an account identifier as one canonical path component.
fn validate_account(account_id: &str) -> Result<()> {
    if account_id.is_empty() || account_id == "." || account_id == ".." || account_id.contains('/')
    {
        return Err(Error::invalid_path(format!(
            "invalid account identifier: {account_id}"
        )));
    }
    Ok(())
}

/// Validate an absolute path without empty, dot, or trailing components.
fn validate_canonical_path(path: &str) -> Result<()> {
    if !path.starts_with('/')
        || path.len() <= 1
        || path.ends_with('/')
        || path.split('/').skip(1).any(|part| {
            part.is_empty() || part == "." || part == ".."
        })
    {
        return Err(Error::invalid_path(format!("non-canonical path: {path}")));
    }
    Ok(())
}