Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

8.9 KB
Raw Blame
//! V2 protocol initialization and resumable full-data import.

use std::sync::Arc;

use uuid::Uuid;

use crate::core::context::{FsContextInner, FS_CTX};
use crate::core::errors::{Error, Result};
use crate::core::filesystem::FileSystem;
use crate::core::internal_names::is_multiwrite_internal_path;
use crate::multibackend::constants::{MULTIWRITE_MOUNT_PREFIX, SYSTEM_DIR};
use crate::multibackend::meta::MetadataStore;
use crate::multibackend::model::{
    PendingEvent, PendingEventKind, ProtocolState, ProtocolStatus, ProtocolVersion,
    SegmentEventType,
};
use crate::multibackend::provider::{bootstrap_account, MultiWriteProvider};
use crate::multibackend::runtime::read_active_accounts;

/// Reads or initializes the persisted V2 multi-write protocol state.
pub struct ProtocolDetector {
    store: Arc<MetadataStore>,
}

impl ProtocolDetector {
    /// Create a detector backed by the supplied raw-primary metadata store.
    pub fn new(store: Arc<MetadataStore>) -> Self {
        Self { store }
    }

    /// Read the protocol state, creating V2 migrating state when absent.
    pub async fn detect(&self) -> Result<ProtocolState> {
        let (protocol_path, _) = self.store.paths().mount_protocol();
        match self.store.read_bytes(&protocol_path).await {
            Ok(bytes) => decode_protocol(&bytes),
            Err(Error::NotFound(_)) => {
                let state = ProtocolState {
                    protocol_version: ProtocolVersion::V2,
                    status: ProtocolStatus::Migrating,
                };
                let bytes = serde_json::to_vec(&state)?;
                match self.store.write_immutable(&protocol_path, &bytes).await {
                    Ok(()) => Ok(state),
                    Err(Error::AlreadyExists(_)) => {
                        decode_protocol(&self.store.read_bytes(&protocol_path).await?)
                    }
                    Err(error) => Err(error),
                }
            }
            Err(error) => Err(error),
        }
    }
}

/// Imports current primary files into V2 segment metadata.
pub struct FullDataImporter {
    primary: Arc<dyn FileSystem>,
    store: Arc<MetadataStore>,
    provider: Arc<dyn MultiWriteProvider>,
    initial_partitions: u32,
    backup_names: Vec<String>,
    reconcile_existing_data: bool,
}

impl FullDataImporter {
    /// Create an importer for one primary, metadata store, and persistence provider.
    pub fn new(
        primary: Arc<dyn FileSystem>,
        store: Arc<MetadataStore>,
        provider: Arc<dyn MultiWriteProvider>,
        initial_partitions: u32,
        backup_names: Vec<String>,
        reconcile_existing_data: bool,
    ) -> Self {
        Self {
            primary,
            store,
            provider,
            initial_partitions,
            backup_names,
            reconcile_existing_data,
        }
    }

    /// Import every current account file and atomically mark V2 stable.
    pub async fn run(&self) -> Result<()> {
        let (protocol_path, _) = self.store.paths().mount_protocol();
        let migrating_bytes = self.store.read_bytes(&protocol_path).await?;
        if decode_protocol(&migrating_bytes)?.status == ProtocolStatus::Stable {
            return Ok(());
        }

        // ponytail: fixed delay assumes account bootstrap finishes within 10s; use readiness signaling if that changes.
        tokio::time::sleep(std::time::Duration::from_secs(10)).await;
        let mut accounts = read_active_accounts(self.primary.as_ref())
            .await?
            .into_iter()
            .collect::<Vec<_>>();
        accounts.sort();
        self.import_scope(SYSTEM_DIR, self.reconcile_existing_data)
            .await?;
        for account in &accounts {
            if !read_active_accounts(self.primary.as_ref())
                .await?
                .contains(account)
            {
                continue;
            }
            self.import_scope(account, self.reconcile_existing_data)
                .await?;
        }
        self.cleanup_v1_global_metadata().await?;

        let stable_bytes = serde_json::to_vec(&ProtocolState {
            protocol_version: ProtocolVersion::V2,
            status: ProtocolStatus::Stable,
        })?;
        if self
            .store
            .compare_protocol(&migrating_bytes, &stable_bytes)
            .await?
        {
            return Ok(());
        }
        if decode_protocol(&self.store.read_bytes(&protocol_path).await?)?.status
            == ProtocolStatus::Stable
        {
            Ok(())
        } else {
            Err(Error::invalid_operation(
                "multi-write protocol changed during full-data import",
            ))
        }
    }

    /// Initialize one V2 scope and import its current file states.
    async fn import_scope(&self, account: &str, reconcile_existing_data: bool) -> Result<()> {
        let partitions = self
            .store
            .initialize_account(account, self.initial_partitions, &self.backup_names)
            .await?;
        bootstrap_account(
            self.provider.as_ref(),
            self.store.as_ref(),
            account,
            &partitions,
        )
        .await?;
        if !reconcile_existing_data {
            tracing::info!(
                account = %account,
                "multi-write existing data reconciliation skipped"
            );
            return Ok(());
        }
        let context = Arc::new(FsContextInner::new(account).with_auto_pathlock_disabled());
        FS_CTX.scope(context, self.import_account(account)).await
    }

    /// Scan one account through the business filesystem.
    async fn import_account(&self, account: &str) -> Result<()> {
        let root = format!("/{account}");
        let mut pending = vec![root];
        let mut scanned_dirs = 0usize;
        let mut imported_files = 0usize;
        let mut sync_logs_removed = 0usize;
        tracing::info!(account = %account, "multi-write full-data import scope started");
        while let Some(directory) = pending.pop() {
            scanned_dirs += 1;
            let mut entries = self.primary.read_internal_dir(&directory).await?;
            entries.sort_by(|left, right| left.name.cmp(&right.name));
            for entry in entries {
                let path = child_path(&directory, &entry.name);
                if entry.name == ".sync_log.json" && !entry.is_dir {
                    match self.primary.remove(&path).await {
                        Ok(()) => sync_logs_removed += 1,
                        Err(Error::NotFound(_)) => {}
                        Err(error) => return Err(error),
                    }
                    continue;
                }
                let logical_path = format!("{MULTIWRITE_MOUNT_PREFIX}{path}");
                if is_multiwrite_internal_path(&logical_path) {
                    continue;
                }
                if entry.is_dir {
                    pending.push(path);
                    continue;
                }
                self.provider
                    .flush(
                        None,
                        vec![PendingEvent {
                            operation_id: Uuid::new_v4(),
                            account_id: account.to_string(),
                            kind: PendingEventKind::Data(SegmentEventType::Write),
                            path: logical_path,
                            destination_path: None,
                            route_hint: None,
                        }],
                    )
                    .await?;
                imported_files += 1;
                if imported_files % 10_000 == 0 {
                    tracing::info!(
                        account = %account,
                        dirs = scanned_dirs,
                        files = imported_files,
                        sync_logs_removed,
                        pending_dirs = pending.len(),
                        "multi-write full-data import scope progress"
                    );
                }
            }
        }
        tracing::info!(
            account = %account,
            dirs = scanned_dirs,
            files = imported_files,
            sync_logs_removed,
            "multi-write full-data import scope completed"
        );
        Ok(())
    }

    /// Remove V1 global metadata through the configured primary filesystem.
    async fn cleanup_v1_global_metadata(&self) -> Result<()> {
        match self
            .primary
            .remove("/_system/.multiwrite.global.json")
            .await
        {
            Ok(()) | Err(Error::NotFound(_)) => Ok(()),
            Err(error) => Err(error),
        }
    }
}

/// Deserialize and validate exact V2 protocol bytes.
fn decode_protocol(bytes: &[u8]) -> Result<ProtocolState> {
    let state: ProtocolState = serde_json::from_slice(bytes)?;
    state.validate()?;
    Ok(state)
}

/// Join one direct child name to a canonical mount-relative directory.
fn child_path(directory: &str, name: &str) -> String {
    if directory == "/" {
        format!("/{name}")
    } else {
        format!("{directory}/{name}")
    }
}