Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(storage): implement multi-write v2 metadata partitioning (#5741)
baojun-zhangff790205a58a
diff --git a/Cargo.lock b/Cargo.lockindex 8b2079660..626c9324f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4098,6 +4098,7 @@ dependencies = ["url","uuid","windows-sys 0.61.2",+ "xxhash-rust",][[package]]@@ -6510,6 +6511,12 @@ version = "0.13.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"+[[package]]+name = "xxhash-rust"+version = "0.8.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7"+[[package]]name = "xz2"version = "0.1.7"diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rsindex 407823453..2484267ba 100644--- a/crates/ov_cli/src/client.rs+++ b/crates/ov_cli/src/client.rs@@ -510,20 +510,6 @@ impl HttpClient {self.post("/api/v1/system/consistency", &body).await}- pub async fn backend_sync_status(&self, uri: &str) -> Result<serde_json::Value> {- let body = serde_json::json!({- "uri": uri,- });- self.post("/api/v1/system/backend/sync-status", &body).await- }-- pub async fn backend_sync_retry(&self, uri: &str) -> Result<serde_json::Value> {- let body = serde_json::json!({- "uri": uri,- });- self.post("/api/v1/system/backend/sync-retry", &body).await- }-/// Download file as raw bytespub async fn get_bytes(&self, uri: &str) -> Result<Vec<u8>> {let url = format!("{}/api/v1/content/download", self.base.base_url);diff --git a/crates/ov_cli/src/commands/system.rs b/crates/ov_cli/src/commands/system.rsindex 5ad24fff8..037cfae04 100644--- a/crates/ov_cli/src/commands/system.rs+++ b/crates/ov_cli/src/commands/system.rs@@ -81,28 +81,6 @@ pub async fn consistency(Ok(())}-pub async fn backend_sync_status(- client: &HttpClient,- uri: &str,- output_format: OutputFormat,- compact: bool,-) -> Result<()> {- let response: serde_json::Value = client.backend_sync_status(uri).await?;- output_success(&response, output_format, compact);- Ok(())-}--pub async fn backend_sync_retry(- client: &HttpClient,- uri: &str,- output_format: OutputFormat,- compact: bool,-) -> Result<()> {- let response: serde_json::Value = client.backend_sync_retry(uri).await?;- output_success(&response, output_format, compact);- Ok(())-}-fn output_consistency_table(response: &serde_json::Value, compact: bool) {let summary = json!({"ok": response.get("ok").and_then(|v| v.as_bool()).unwrap_or(false),diff --git a/crates/ov_cli/src/error_ui.rs b/crates/ov_cli/src/error_ui.rsindex c94397d8f..45db15290 100644--- a/crates/ov_cli/src/error_ui.rs+++ b/crates/ov_cli/src/error_ui.rs@@ -529,12 +529,6 @@ fn system_help_command(program: &str, tokens: &[&str]) -> String {Some("init-key") => format!("{program} system crypto init-key --help"),_ => format!("{program} system crypto --help"),},- Some("backend") => match tokens.get(3).copied() {- Some("sync-status" | "sync-retry") => {- format!("{program} system backend {} --help", tokens[3])- }- _ => format!("{program} system backend --help"),- },Some("wait" | "status" | "health" | "consistency") => {format!("{program} system {} --help", tokens[2])}diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rsindex 8c927503c..8bf132c1f 100644--- a/crates/ov_cli/src/handlers.rs+++ b/crates/ov_cli/src/handlers.rs@@ -415,7 +415,7 @@ pub async fn handle_restore(.await}-use crate::{SystemBackendCommands, SystemCommands};+use crate::SystemCommands;pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> {let client = ctx.get_client();@@ -440,16 +440,6 @@ pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> {commands::system::consistency(&client, &uri, ctx.output_format, ctx.compact).await}SystemCommands::Crypto { action } => commands::crypto::handle_crypto(action).await,- SystemCommands::Backend { action } => match action {- SystemBackendCommands::SyncStatus { uri } => {- commands::system::backend_sync_status(&client, &uri, ctx.output_format, ctx.compact)- .await- }- SystemBackendCommands::SyncRetry { uri } => {- commands::system::backend_sync_retry(&client, &uri, ctx.output_format, ctx.compact)- .await- }- },}}diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rsindex d386936be..17f4015f1 100644--- a/crates/ov_cli/src/help_ui.rs+++ b/crates/ov_cli/src/help_ui.rs@@ -1340,7 +1340,7 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[},CommandHelpSpec {path: &["system"],- purpose: "Run server utility, health, consistency, backend sync, and crypto commands.",+ purpose: "Run server utility, health, consistency, and crypto commands.",examples: &[HelpItem {label: "ov system health",@@ -1350,34 +1350,12 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[label: "ov system consistency viking://projects/acme",description: "Check filesystem/vector consistency.",},- HelpItem {- label: "ov system backend sync-status viking://projects/acme",- description: "Inspect multi-write backend sync lag.",- },],next_steps: &[HelpItem {label: "ov status",description: "Use the standard status view.",}],},- CommandHelpSpec {- path: &["system", "backend"],- purpose: "Inspect and repair multi-write backend sync state.",- examples: &[- HelpItem {- label: "ov system backend sync-status viking://resources",- description: "Show pending and acknowledged backend sync state.",- },- HelpItem {- label: "ov system backend sync-retry viking://resources",- description: "Retry lagging backend sync targets.",- },- ],- next_steps: &[HelpItem {- label: "ov system backend sync-status <uri>",- description: "Inspect the subtree again after retry.",- }],- },CommandHelpSpec {path: &["reindex"],purpose: "Reindex semantic/vector artifacts for a URI.",@@ -2992,7 +2970,6 @@ mod tests {fn curated_help_lists_timeout_for_waiting_commands() {for args in [["ov", "add-resource", "--help"],- ["ov", "add-skill", "--help"],["ov", "rm", "--help"],["ov", "write", "--help"],] {@@ -3214,7 +3191,7 @@ mod tests {("admin", "Manage accounts, users, roles, and API keys."),("system",- "Run server utility, health, consistency, backend sync, and crypto commands.",+ "Run server utility, health, consistency, and crypto commands.",),("observer", "Inspect specific OpenViking server subsystems."),] {@@ -3281,20 +3258,6 @@ mod tests {);}- #[test]- fn supported_nested_curated_help_still_renders() {- let rendered = strip_ansi(- &render_command_help_request(&os_args(&["ov", "system", "backend", "--help"]))- .expect("system backend help should render curated nested help"),- );-- assert!(rendered.contains("ov system backend"));- assert!(rendered.contains("sync-status"));- assert!(rendered.contains("sync-retry"));- assert!(!rendered.contains("help Print this message"));- assert!(rendered.contains("--sudo"));- }-#[test]fn prefixed_help_with_positional_value_renders_curated_command_help() {let rendered = strip_ansi(diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rsindex 30ff8c4d6..00630c3ed 100644--- a/crates/ov_cli/src/main.rs+++ b/crates/ov_cli/src/main.rs@@ -1589,29 +1589,6 @@ enum SystemCommands {#[command(subcommand)]action: commands::crypto::CryptoCommands,},- /// Backend sync inspection and repair commands- Backend {- #[command(subcommand)]- action: SystemBackendCommands,- },-}--#[derive(Subcommand)]-enum SystemBackendCommands {- /// Show multi-write backend sync status for a URI subtree- #[command(name = "sync-status")]- SyncStatus {- /// Viking URI to inspect- #[arg(value_name = "uri")]- uri: String,- },- /// Retry pending multi-write backend sync work for a URI subtree- #[command(name = "sync-retry")]- SyncRetry {- /// Viking URI to repair- #[arg(value_name = "uri")]- uri: String,- },}#[derive(Subcommand)]@@ -2544,14 +2521,6 @@ fn plain_help_misuse(args: &[OsString]) -> Option<PlainHelpMisuse> {{path.push(watch_subcommand.to_string());}- if command == "system"- && path.get(1).is_some_and(|token| token == "backend")- && let Some(backend_subcommand) =- tokens.get(2).map(|token| canonical_plain_help_token(token))- && backend_subcommand != "help"- {- path.push(backend_subcommand.to_string());- }return Some(PlainHelpMisuse {help_command: prefixed_help_command(&path),});@@ -2763,10 +2732,6 @@ fn known_task_command_requires_config(tokens: &[String]) -> bool {fn known_system_command_requires_config(tokens: &[String]) -> bool {match tokens.get(1).map(String::as_str) {Some("wait" | "status" | "health" | "consistency") => true,- Some("backend") => matches!(- tokens.get(2).map(String::as_str),- None | Some("sync-status" | "sync-retry")- ),Some("crypto") => matches!(tokens.get(2).map(String::as_str), None | Some("init-key")),_ => false,}@@ -4032,7 +3997,7 @@ mod tests {};use crate::config::{Config, DEFAULT_CUSTOM_URL};use crate::output::OutputFormat;- use crate::{AdminCommands, SystemBackendCommands, SystemCommands, handlers};+ use crate::{AdminCommands, handlers};use clap::{CommandFactory, Parser};use std::ffi::OsString;@@ -4382,22 +4347,6 @@ mod tests {}}- #[test]- fn cli_parses_system_backend_sync_status() {- let cli = Cli::try_parse_from(["ov", "system", "backend", "sync-status", "viking://a"])- .expect("system backend sync-status should parse");-- match cli.command {- Commands::System { action } => match action {- SystemCommands::Backend { action } => match action {- SystemBackendCommands::SyncStatus { uri } => assert_eq!(uri, "viking://a"),- _ => panic!("expected backend sync-status command"),- },- _ => panic!("expected system backend command"),- },- _ => panic!("expected system command"),- }- }#[test]fn server_commands_require_existing_cli_config() {@@ -4623,7 +4572,6 @@ mod tests {&["ov", "config", "validate"],&["ov", "config", "show"],&["ov", "system", "consistency"],- &["ov", "system", "backend", "sync-status"],&["ov", "admin", "list-accounts"],] {assert!(@@ -4708,9 +4656,6 @@ mod tests {&["ov", "system", "status"],&["ov", "system", "health"],&["ov", "system", "consistency"],- &["ov", "system", "backend"],- &["ov", "system", "backend", "sync-status"],- &["ov", "system", "backend", "sync-retry"],&["ov", "system", "crypto"],&["ov", "system", "crypto", "init-key"],&["ov", "session", "new"],@@ -5628,10 +5573,6 @@ mod tests {vec!["ov", "system", "consistency", "help"],"ov system consistency --help",),- (- vec!["ov", "system", "backend", "sync-status", "help"],- "ov system backend sync-status --help",- ),] {let misuse = plain_help_misuse(&os_args(&args)).unwrap_or_else(|| panic!("{args:?} should be plain-help misuse"));diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rsindex bab219ebc..b09adc8c8 100644--- a/crates/ragfs-python/src/lib.rs+++ b/crates/ragfs-python/src/lib.rs@@ -956,6 +956,7 @@ fn to_py_err(e: ragfs::core::Error) -> PyErr {ragfs::core::Error::IsADirectory(_) => new_py_err("AGFSIsADirectoryError", msg),ragfs::core::Error::DirectoryNotEmpty(_) => new_py_err("AGFSDirectoryNotEmptyError", msg),ragfs::core::Error::InvalidOperation(_) => new_py_err("AGFSInvalidOperationError", msg),+ ragfs::core::Error::NotSupported(_) => new_py_err("AGFSNotSupportedError", msg),ragfs::core::Error::Io(_) => new_py_err("AGFSIoError", msg),ragfs::core::Error::Plugin(_) => {// Check if the plugin error message contains known patterns@@ -1628,7 +1629,11 @@ impl RAGFSBindingClient {cache_config.stack_config(),)).map_err(|error| {- PyRuntimeError::new_err(format!("Failed to build RAGFS stack: {error}"))+ if matches!(error, ragfs::core::Error::NotSupported(_)) {+ to_py_err(error)+ } else {+ PyRuntimeError::new_err(format!("Failed to build RAGFS stack: {error}"))+ }})?;// Build the git service from inline config when present; otherwise fall@@ -1652,7 +1657,7 @@ impl RAGFSBindingClient {})}- /// Stop mounted background services, then close the shared CacheRuntime.+ /// Stop mounts requiring explicit cleanup, then close the shared CacheRuntime.fn close(&self, py: Python<'_>) -> PyResult<()> {let mountable = Arc::clone(&self.mountable);let cache_runtime = self.cache_runtime.clone();@@ -2169,7 +2174,6 @@ impl RAGFSBindingClient {/// - "backups": nested dict matching BackendsConfig schema/// - "server_encryption_enabled": bool/// - "primary_encryption_enabled": bool- /// - "primary_redirects": list of redirect policy dicts#[pyo3(signature = (fstype, path, config=None))]fn mount(&self,@@ -2433,58 +2437,6 @@ impl RAGFSBindingClient {})}- /// Query multi-write sync status under a file or directory path.- ///- /// Args:- /// path: Absolute AGFS path rooted at the mounted namespace- /// ctx: Optional FsContext dict (e.g. {"account_id": ...})- ///- /// Returns:- /// A JSON-like dict describing pending and acknowledged backup sync state.- #[pyo3(signature = (path, ctx=None))]- fn system_sync_status(- &self,- py: Python<'_>,- path: String,- ctx: Option<HashMap<String, String>>,- ) -> PyResult<Py<PyAny>> {- let fs_ctx = build_fs_context(ctx);- let mountable = self.mountable.clone();- let result = self- .run_scoped(py, fs_ctx, move || async move {- mountable.system_sync_status(&path).await- })- .map_err(to_py_err)?;-- Python::attach(|py| serde_json_to_py(py, &result))- }-- /// Manually retry lagging multi-write backup sync operations under a path.- ///- /// Args:- /// path: Absolute AGFS path rooted at the mounted namespace- /// ctx: Optional FsContext dict (e.g. {"account_id": ...})- ///- /// Returns:- /// A JSON-like dict summarizing retry results per target backend.- #[pyo3(signature = (path, ctx=None))]- fn system_sync_retry(- &self,- py: Python<'_>,- path: String,- ctx: Option<HashMap<String, String>>,- ) -> PyResult<Py<PyAny>> {- let fs_ctx = build_fs_context(ctx);- let mountable = self.mountable.clone();- let result = self- .run_scoped(py, fs_ctx, move || async move {- mountable.system_sync_retry(&path).await- })- .map_err(to_py_err)?;-- Python::attach(|py| serde_json_to_py(py, &result))- }-/// Calculate file digest (not yet implemented in ragfs).#[pyo3(signature = (path, algorithm="xxh3"))]fn digest(&self, path: String, algorithm: &str) -> PyResult<HashMap<String, String>> {diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.tomlindex 787c1ffcf..b941490a7 100644--- a/crates/ragfs/Cargo.toml+++ b/crates/ragfs/Cargo.toml@@ -18,11 +18,6 @@ path = "src/lib.rs"name = "ragfs-shell"path = "src/shell/main.rs"-[[test]]-name = "cache_wrapper"-path = "tests/cache_wrapper.rs"-required-features = ["cache", "test-utils"]-[[test]]name = "cache_runtime"path = "tests/cache_runtime.rs"@@ -67,6 +62,7 @@ filelocks = "0.1.0"# Data structuresradix_trie = "0.2"+xxhash-rust = { version = "0.8", features = ["xxh3"] }# Error handlinganyhow = "1.0"diff --git a/crates/ragfs/src/core/builder.rs b/crates/ragfs/src/core/builder.rsindex 61c47051e..4c5f4b67c 100644--- a/crates/ragfs/src/core/builder.rs+++ b/crates/ragfs/src/core/builder.rs@@ -200,6 +200,8 @@ pub async fn build_configured_stack(CacheNamespace::new(&cache.cachefs.namespace),cache.cachefs.policy,))+ } else if let Some(runtime) = runtime.as_ref() {+ Arc::new(MountableFS::with_shared_cache_runtime(runtime.clone()))} else {Arc::new(MountableFS::new())};diff --git a/crates/ragfs/src/core/errors.rs b/crates/ragfs/src/core/errors.rsindex 9ee84e5c2..43f10ac2c 100644--- a/crates/ragfs/src/core/errors.rs+++ b/crates/ragfs/src/core/errors.rs@@ -56,6 +56,10 @@ pub enum Error {#[error("invalid operation: {0}")]InvalidOperation(String),+ /// Operation is intentionally unsupported by the current protocol.+ #[error("not supported: {0}")]+ NotSupported(String),+/// I/O error#[error("I/O error: {0}")]Io(#[from] io::Error),@@ -188,6 +192,7 @@ impl Error {Self::IsADirectory(_) => "is_a_directory",Self::DirectoryNotEmpty(_) => "directory_not_empty",Self::InvalidOperation(_) => "invalid_operation",+ Self::NotSupported(_) => "not_supported",Self::Io(_) => "io",Self::Plugin(_) => "plugin",Self::Config(_) => "config",@@ -208,6 +213,11 @@ impl Error {pub fn invalid_operation(msg: impl Into<String>) -> Self {Self::InvalidOperation(msg.into())}++ /// Create a NotSupported error.+ pub fn not_supported(msg: impl Into<String>) -> Self {+ Self::NotSupported(msg.into())+ }}#[cfg(test)]diff --git a/crates/ragfs/src/core/internal_names.rs b/crates/ragfs/src/core/internal_names.rsindex dadfc36cd..f73ddd27d 100644--- a/crates/ragfs/src/core/internal_names.rs+++ b/crates/ragfs/src/core/internal_names.rs@@ -1,8 +1,7 @@//! Static built-in internal names shared across the RAGFS stack.//!-//! Every module that references `.path.ovlock`, `.exact.ovlock.*`, `.redirect.json`,-//! or `.sync_log.json` must import these constants from this single source of truth.-//! There is no dynamic extension mechanism.+//! Every module that references `.path.ovlock` or `.exact.ovlock.*` must import+//! these constants from this single source of truth./// Path-lock marker for directory-level locks.pub const PATH_LOCK_FILE: &str = ".path.ovlock";@@ -10,21 +9,49 @@ pub const PATH_LOCK_FILE: &str = ".path.ovlock";/// Prefix for exact (sidecar) lock files: `.exact.ovlock.<safe_name>.<sha1-prefix>`.pub const EXACT_LOCK_FILE_PREFIX: &str = ".exact.ovlock.";-/// Multi-write redirect metadata file.-pub const REDIRECT_FILE: &str = ".redirect.json";--/// Multi-write sync-log metadata file.-pub const SYNC_LOG_FILE: &str = ".sync_log.json";-/// Returns `true` when `name` is a runtime path-lock file (`.path.ovlock` or `.exact.ovlock.*`).pub fn is_hidden_runtime_lock_name(name: &str) -> bool {// Reserve the whole prefix, including names not produced by the lock resolver.name == PATH_LOCK_FILE || name.starts_with(EXACT_LOCK_FILE_PREFIX)}-/// Returns `true` when `name` is any hidden internal name (lock files, redirect, sync-log).+/// Returns `true` when `name` is a hidden runtime lock name.pub fn is_hidden_internal_name(name: &str) -> bool {- is_hidden_runtime_lock_name(name) || name == REDIRECT_FILE || name == SYNC_LOG_FILE+ is_hidden_runtime_lock_name(name)+}++/// Returns `true` when a logical path belongs to V2 multi-write internals.+pub fn is_multiwrite_internal_path(path: &str) -> bool {+ let Some(relative) = path.strip_prefix('/') else {+ return false;+ };+ let components = relative.split('/').collect::<Vec<_>>();+ if components+ .last()+ .is_some_and(|name| is_hidden_internal_name(name))+ {+ return true;+ }+ if matches!(+ components.as_slice(),+ ["local", "_system", ".multiwrite.json"]+ ) {+ return true;+ }+ if matches!(+ components.as_slice(),+ ["local", "_system", name] if name.starts_with(".multiwrite.catch-up.")+ ) || matches!(components.as_slice(), ["local", "_system", "_system", ..])+ {+ return true;+ }+ matches!(+ components.as_slice(),+ ["local", account, "_system", "partitions", ..] if !account.is_empty()+ ) || matches!(+ components.as_slice(),+ ["local", account, "temp", ..] if !account.is_empty()+ )}#[cfg(test)]diff --git a/crates/ragfs/src/core/mod.rs b/crates/ragfs/src/core/mod.rsindex fb75b3d52..9e4a9be9d 100644--- a/crates/ragfs/src/core/mod.rs+++ b/crates/ragfs/src/core/mod.rs@@ -23,7 +23,7 @@ pub mod stats_wrapper;pub mod types;// Re-export commonly used types-pub use crate::multibackend::{FsContextResolver, MetaStateStore};+pub use crate::multibackend::FsContextResolver;#[cfg(feature = "cache")]pub use builder::{build_configured_stack, CacheFsConfig, CacheRuntimeProviderConfig, CacheStackConfig,@@ -37,13 +37,12 @@ pub use encryption_wrapper::EncryptionWrappedFS;pub use errors::{Error, Result};pub use filesystem::FileSystem;pub use mountable::MountableFS;-pub use multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS, SyncMode};+pub use multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS};pub use plugin::{HealthStatus, PluginRegistry, ServicePlugin};pub use stats::{FilesystemStats, FsOperation, OperationStats, OperationTimer, StatsCollector};pub use stats_wrapper::StatsWrappedFS;pub use types::{- BackendItemConfig, BackendRole, BackendSyncState, BackendsConfig, ConfigParameter, ConfigValue,- EncryptionConfig, FileInfo, GlobEntry, GlobPage, GrepMatch, GrepOptions, GrepResult,- ListSortBy, OperationItemConfig, PluginConfig, RedirectEntry, RedirectMeta, RedirectPolicy,- SortOrder, SyncLogEntry, SyncLogMeta, SyncOp, SyncType, TreeEntry, WriteFlag,+ BackendItemConfig, BackendRole, BackendsConfig, ConfigParameter, ConfigValue, EncryptionConfig,+ FileInfo, GlobEntry, GlobPage, GrepMatch, GrepOptions, GrepResult, ListSortBy, PluginConfig,+ SortOrder, TreeEntry, WriteFlag,};diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rsindex 2b7f625ab..32d9c201c 100644--- a/crates/ragfs/src/core/mountable.rs+++ b/crates/ragfs/src/core/mountable.rs@@ -6,7 +6,6 @@use async_trait::async_trait;use radix_trie::{Trie, TrieCommon};-use serde_json::Value;use std::collections::HashMap;use std::sync::{Arc, OnceLock};use std::time::Duration;@@ -14,10 +13,8 @@ use tokio::sync::RwLock;use tracing::warn;use crate::lock::{AutoPathLockAction, PathLockKind, PathLockManager, PathLockRequest};-use crate::metrics::{- lock_metrics, merge_metrics, operation_metrics, RagfsMetric, RagfsMetricValue,-};-use crate::multibackend::factory::build_multi_write_fs;+use crate::metrics::{lock_metrics, merge_metrics, operation_metrics, RagfsMetric};+use crate::multibackend::factory::build_inactive_multi_write_fs;use crate::multibackend::types::MultiBackendBuildContext;use crate::plugins::QueueFileSystem;use crate::shape::validate::ensure_backend_shape;@@ -88,6 +85,10 @@ pub struct MountableFS {/// Optional cache configuration shared by mounted filesystems.#[cfg(feature = "cache")]cache: Option<MountCacheConfig>,++ /// Optional shared runtime, independent of mount-level CacheFS enablement.+ #[cfg(feature = "cache")]+ cache_runtime: Option<Arc<CacheRuntime>>,}#[cfg(feature = "cache")]@@ -169,9 +170,6 @@ impl MountableFS {if let Some(queuefs) = Self::as_queuefs_ref(mount_info.fs.as_ref()) {queuefs.shutdown().await?;}- if let Some(multiwrite) = Self::as_multiwrite(&mount_info.fs) {- multiwrite.shutdown().await?;- }Ok(())}@@ -220,22 +218,6 @@ impl MountableFS {Self::encryption_handles_pathlock_ref(mount_info.fs.as_ref())}- /// Query multi-write sync status for a mounted path.- pub async fn system_sync_status(&self, path: &str) -> Result<Value> {- let (mount_info, rel_path) = self.find_mount(path).await?;- let multiwrite = Self::as_multiwrite(&mount_info.fs)- .ok_or_else(|| Error::invalid_operation("mounted filesystem is not multi-write"))?;- multiwrite.system_sync_status(&rel_path).await- }-- /// Manually retry pending multi-write sync operations under a mounted path.- pub async fn system_sync_retry(&self, path: &str) -> Result<Value> {- let (mount_info, rel_path) = self.find_mount(path).await?;- let multiwrite = Self::as_multiwrite(&mount_info.fs)- .ok_or_else(|| Error::invalid_operation("mounted filesystem is not multi-write"))?;- multiwrite.system_sync_retry(&rel_path).await- }-/// Create a new MountableFSpub fn new() -> Self {Self {@@ -246,6 +228,22 @@ impl MountableFS {pathlock_manager: OnceLock::new(),#[cfg(feature = "cache")]cache: None,+ #[cfg(feature = "cache")]+ cache_runtime: None,+ }+ }++ /// Create a MountableFS that shares a runtime without enabling CacheFS.+ #[cfg(feature = "cache")]+ pub fn with_shared_cache_runtime(runtime: Arc<CacheRuntime>) -> Self {+ Self {+ mounts: Arc::new(RwLock::new(Trie::new())),+ registry: Arc::new(RwLock::new(HashMap::new())),+ encryption_root_key: RwLock::new(None),+ encryption_provider_type: RwLock::new(None),+ pathlock_manager: OnceLock::new(),+ cache: None,+ cache_runtime: Some(runtime),}}@@ -263,10 +261,11 @@ impl MountableFS {encryption_provider_type: RwLock::new(None),pathlock_manager: OnceLock::new(),cache: Some(MountCacheConfig {- runtime,+ runtime: runtime.clone(),namespace,policy,}),+ cache_runtime: Some(runtime),}}@@ -448,9 +447,13 @@ impl MountableFS {stats: stats_collector,plugin_name: config.name.clone(),};-+ let mounted_fs = mount_info.fs.clone();let mut mounts = self.mounts.write().await;mounts.insert(normalized_path, mount_info);+ drop(mounts);+ if let Some(multiwrite) = Self::as_multiwrite(&mounted_fs) {+ multiwrite.activate();+ }Ok(())}@@ -471,7 +474,7 @@ impl MountableFS {let pathlock_manager = self.pathlock_manager.get().cloned().ok_or_else(|| {Error::config("pathlock manager must be initialized before multi-write mount")})?;- build_multi_write_fs(+ build_inactive_multi_write_fs(&self.registry,config,bc,@@ -480,6 +483,8 @@ impl MountableFS {enc_provider_type,pathlock_manager,backend_prefix: config.mount_path.clone(),+ #[cfg(feature = "cache")]+ cache_runtime: self.cache_runtime.clone(),},).await@@ -617,28 +622,7 @@ impl MountableFS {metrics.extend(crate::metrics::cache_metrics(cache.metrics().snapshot()));}if let Some(multiwrite) = Self::as_multiwrite(&mount.fs) {- metrics.push(RagfsMetric {- name: "ragfs_multiwrite_background_tasks".into(),- labels: Default::default(),- value: RagfsMetricValue::Gauge(multiwrite.background_task_count() as f64),- });- let routes = multiwrite.inner.read_route_metrics();- for (route, key) in [- ("primary", "primary_hits"),- ("backup", "backup_hits"),- ("redirect", "redirect_hits"),- ("miss", "misses"),- ] {- let count = routes[key].as_u64().ok_or_else(|| {- Error::internal(format!("invalid read-route counter '{key}'"))- })?;- metrics.push(RagfsMetric::counter(- "ragfs_multiwrite_read_routes_total",- &[("route", route)],- count,- 1.0,- ));- }+ metrics.extend(multiwrite.metrics().await?);}}if let Some(manager) = self.pathlock_manager.get() {@@ -1250,9 +1234,7 @@ impl FileSystem for MountableFS {mod tests {use super::*;use crate::core::BackendItemConfig;- #[cfg(feature = "cache")]use crate::core::ConfigValue;- use crate::core::RedirectPolicy;use crate::shape::SHAPE_MANIFEST_PATH;use serde_json::Value;use std::collections::HashMap;@@ -1274,29 +1256,66 @@ mod tests {mount_path: mount_path.to_string(),params: HashMap::new(),backups: Some(BackendsConfig {- sync_type: "async".to_string(),- write_ack_count: None,- write_ack_timeout_ms: None,- write_concurrency: None,- retry_interval_ms: None,- retry_backoff_base_ms: None,- retry_max_retries_per_round: None,- retry_quarantine_after_failures: None,- read_probe_cache_ttl_ms: None,+ namespace: "default".to_string(),+ initial_partitions: 16,+ checkpoint_interval_secs: 86_400,+ provider: "filesystem".to_string(),items: vec![BackendItemConfig {name: "backup1".to_string(),backend: backup_backend.to_string(),params: Value::Null,- timeout: None,encryption: None,- operations: None,- excludes: None,}],}),..PluginConfig::default()}}+ /// Build a two-backup LocalFS configuration from three persistent directories.+ fn local_multiwrite_test_config(+ primary_dir: &std::path::Path,+ backup_dirs: [(&str, &std::path::Path); 2],+ ) -> PluginConfig {+ let mut config = multiwrite_test_config("localfs", "localfs", "/local");+ config.params.insert(+ "local_dir".to_string(),+ ConfigValue::String(primary_dir.to_string_lossy().into_owned()),+ );+ config.backups.as_mut().unwrap().initial_partitions = 1;+ config.backups.as_mut().unwrap().items = backup_dirs+ .into_iter()+ .map(|(name, path)| BackendItemConfig {+ name: name.to_string(),+ backend: "localfs".to_string(),+ params: serde_json::json!({+ "local_dir": path.to_string_lossy(),+ }),+ encryption: None,+ })+ .collect();+ config+ }++ /// Mount one LocalFS multi-write configuration with the shared lock provider.+ async fn mount_local_multiwrite_for_test(+ config: PluginConfig,+ provider: Arc<dyn crate::lock::PathLockProvider>,+ ) -> Arc<MountableFS> {+ use crate::lock::{PathLockConfig, PathLockManager};+ use crate::plugins::{memfs::MemFileSystem, LocalFSPlugin};++ let mfs = Arc::new(MountableFS::new());+ mfs.register_plugin(LocalFSPlugin::new()).await;+ let manager = Arc::new(PathLockManager::new(+ Arc::new(MemFileSystem::new()),+ provider,+ PathLockConfig::default(),+ ));+ mfs.set_pathlock_manager(manager).await;+ mfs.mount(config).await.unwrap();+ mfs+ }+// Mock filesystem for testingstruct MockFS {name: String,@@ -1338,10 +1357,10 @@ mod tests {}async fn read(&self, path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {- if path == SHAPE_MANIFEST_PATH- || path.ends_with(".redirect.json")- || path.ends_with(".sync_log.json")- {+ if path == "/_system/accounts.json" {+ return Ok(br#"{"accounts":{}}"#.to_vec());+ }+ if path == SHAPE_MANIFEST_PATH || path == "/_system/.multiwrite.json" {return Err(Error::not_found(path));}Ok(self.name.as_bytes().to_vec())@@ -1373,6 +1392,9 @@ mod tests {}async fn stat(&self, path: &str) -> Result<FileInfo> {+ if path == "/_system/.multiwrite.json" {+ return Err(Error::not_found(path));+ }Ok(FileInfo::new_file(path.to_string(), 0, 0o644))}@@ -1561,33 +1583,6 @@ mod tests {mfs}- /// Create a MountableFS backed by the real in-memory plugin.- async fn mounted_memfs(mount_path: &str) -> MountableFS {- use crate::plugins::MemFSPlugin;-- let mfs = MountableFS::new();- mfs.register_plugin(MemFSPlugin).await;- mfs.mount(test_config("memfs", mount_path)).await.unwrap();- mfs- }-- /// Create a cache-enabled MountableFS backed by the real in-memory plugin.- #[cfg(feature = "cache")]- async fn mounted_cached_memfs(namespace: &str, mount_path: &str) -> MountableFS {- use crate::cache::{CacheNamespace, CachePolicy};- use crate::cache_runtime::CacheRuntime;- use crate::plugins::MemFSPlugin;-- let mfs = MountableFS::with_cache_runtime(- CacheRuntime::memory(),- CacheNamespace::new(namespace),- CachePolicy::default(),- );- mfs.register_plugin(MemFSPlugin).await;- mfs.mount(test_config("memfs", mount_path)).await.unwrap();- mfs- }-/// Create a MountableFS with two mounted mock plugins.async fn mounted_two_mocks() -> MountableFS {let mfs = MountableFS::new();@@ -1733,6 +1728,32 @@ mod tests {);}+ /// Verifies service shutdown leaves multi-write workers to process teardown.+ #[tokio::test]+ async fn shutdown_does_not_stop_multiwrite_runtime() {+ use crate::lock::MemoryPathLockProvider;++ let primary = tempfile::tempdir().unwrap();+ let backup_a = tempfile::tempdir().unwrap();+ let backup_b = tempfile::tempdir().unwrap();+ let config = local_multiwrite_test_config(+ primary.path(),+ [("backup-a", backup_a.path()), ("backup-b", backup_b.path())],+ );+ let mfs =+ mount_local_multiwrite_for_test(config, Arc::new(MemoryPathLockProvider::new())).await;+ let mount = mfs.mounts.read().await.get("/local").unwrap().clone();++ mfs.shutdown().await.unwrap();++ assert_eq!(+ MountableFS::as_multiwrite(&mount.fs)+ .unwrap()+ .background_task_count(),+ 1+ );+ }+#[cfg(feature = "cache")]#[tokio::test]async fn shutdown_reaches_cache_queuefs_through_stats_wrapper() {@@ -1949,115 +1970,6 @@ mod tests {);}- #[cfg(feature = "cache")]- #[tokio::test]- async fn encrypted_multiwrite_mount_does_not_install_plaintext_cache() {- use crate::cache::{CacheNamespace, CachePolicy};- use crate::cache_runtime::CacheRuntime;- use crate::lock::{- MemoryPathLockProvider, PathLockConfig, PathLockManager, PathLockProvider,- };- use crate::plugins::MemFSPlugin;-- let mfs = Arc::new(MountableFS::with_cache_runtime(- CacheRuntime::memory(),- CacheNamespace::new("encrypted-multiwrite-test"),- CachePolicy::default(),- ));- mfs.register_plugin(MemFSPlugin).await;- mfs.set_encryption_config(Some([9u8; 32]), Some(1)).await;- let provider: Arc<dyn PathLockProvider> = Arc::new(MemoryPathLockProvider::new());- let manager = Arc::new(PathLockManager::new(- mfs.clone() as Arc<dyn FileSystem>,- provider,- PathLockConfig::default(),- ));- mfs.set_pathlock_manager(manager).await;-- let mut config = multiwrite_test_config("memfs", "memfs", "/local");- config.server_encryption_enabled = true;- config.primary_encryption_enabled = true;- mfs.mount(config).await.unwrap();-- let mounts = mfs.mounts.read().await;- let mount_info = mounts.get("/local").expect("mounted entry should exist");- let stats = (mount_info.fs.as_ref() as &dyn std::any::Any)- .downcast_ref::<StatsWrappedFS>()- .expect("stats wrapper should be outermost");- assert!(- (stats.inner_fs().as_ref() as &dyn std::any::Any)- .downcast_ref::<MultiWriteWrappedFS>()- .is_some(),- "encrypted multi-write must not install a plaintext cache outside MultiWriteWrappedFS"- );- }-- #[cfg(feature = "cache")]- #[tokio::test]- async fn cached_unencrypted_multiwrite_keeps_admin_and_copy_fast_paths() {- use crate::cache::{CacheNamespace, CachePolicy};- use crate::cache_runtime::CacheRuntime;- use crate::core::{FsContextInner, FS_CTX};- use crate::plugins::MemFSPlugin;-- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::with_cache_runtime(- CacheRuntime::memory(),- CacheNamespace::new("cached-multiwrite-test"),- CachePolicy::default(),- )))- .await;- mfs.register_plugin(MemFSPlugin).await;-- let mut config = multiwrite_test_config("memfs", "memfs", "/local");- config.backups.as_mut().unwrap().items[0].name = "backup1".to_string();- mfs.mount(config).await.unwrap();-- let ctx = Arc::new(FsContextInner::new("acct"));- FS_CTX- .scope(ctx.clone(), async {- let status = mfs.system_sync_status("/local").await.unwrap();- assert_eq!(status["path"], "/");- assert_eq!(status["entry_count"], 0);-- let retry = mfs.system_sync_retry("/local").await.unwrap();- assert_eq!(retry["path"], "/");- assert_eq!(retry["retried"], 0);-- mfs.mkdir("/local/docs", 0o755).await.unwrap();- mfs.write("/local/docs/src.md", b"copied", 0, WriteFlag::Create)- .await- .unwrap();-- assert!(mfs- .copy_within_mount("/local/docs/src.md", "/local/docs/dst.md")- .await- .unwrap());- assert_eq!(- mfs.read("/local/docs/dst.md", 0, 0).await.unwrap(),- b"copied"- );- })- .await;-- mfs.unmount("/local").await.unwrap();- assert!(mfs.list_mounts().await.is_empty());- }-- #[tokio::test]- async fn multiwrite_mount_without_pathlock_returns_config_error() {- use crate::plugins::MemFSPlugin;-- let mfs = MountableFS::new();- mfs.register_plugin(MemFSPlugin).await;-- let error = mfs- .mount(multiwrite_test_config("memfs", "memfs", "/local"))- .await- .unwrap_err();-- assert!(matches!(error, Error::Config(_)));- }-#[tokio::test]async fn test_mount_duplicate_error() {let mfs = Arc::new(MountableFS::new());@@ -2346,180 +2258,6 @@ mod tests {assert_eq!(result[0].rel_path, "a/b/c.txt");}- #[tokio::test]- async fn test_build_multi_write_fs_rejects_primary_encryption_disable() {- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;- mfs.register_plugin(MockPlugin::new("primary")).await;- mfs.register_plugin(MockPlugin::new("backupfs")).await;- mfs.set_encryption_config(Some([7u8; 32]), Some(1)).await;-- let backups = BackendsConfig {- sync_type: "async".to_string(),- write_ack_count: None,- write_ack_timeout_ms: None,- write_concurrency: None,- retry_interval_ms: None,- retry_backoff_base_ms: None,- retry_max_retries_per_round: None,- retry_quarantine_after_failures: None,- read_probe_cache_ttl_ms: None,- items: vec![BackendItemConfig {- name: "backup1".to_string(),- backend: "backupfs".to_string(),- params: serde_json::Value::Null,- timeout: None,- encryption: None,- operations: None,- excludes: None,- }],- };- let config = PluginConfig {- name: "primary".to_string(),- mount_path: "/local".to_string(),- params: HashMap::new(),- backups: Some(backups.clone()),- server_encryption_enabled: true,- ..PluginConfig::default()- };-- let result = mfs.build_multi_write_fs(&config, &backups).await;- assert!(result.is_err());- assert!(matches!(result.err(), Some(Error::Config(_))));- }-- #[tokio::test]- async fn encrypted_multiwrite_reports_pathlock_ownership() {- use crate::core::{FsContextInner, PathLockContext, FS_CTX};- use crate::lock::{- MemoryPathLockProvider, PathLockConfig, PathLockManager, PathLockProvider,- };- use crate::plugins::MemFSPlugin;-- let mfs = Arc::new(MountableFS::new());- mfs.register_plugin(MemFSPlugin).await;- mfs.set_encryption_config(Some([7u8; 32]), Some(1)).await;- let provider: Arc<dyn PathLockProvider> = Arc::new(MemoryPathLockProvider::new());- let manager = Arc::new(PathLockManager::new(- mfs.clone() as Arc<dyn FileSystem>,- provider,- PathLockConfig::default(),- ));- mfs.set_pathlock_manager(manager.clone()).await;-- let mut config = multiwrite_test_config("memfs", "memfs", "/local");- config.server_encryption_enabled = true;- config.primary_encryption_enabled = true;- mfs.mount(config).await.unwrap();-- assert!(mfs.encryption_handles_pathlock("/local/file.txt").await);-- let outer_ctx = Arc::new(FsContextInner::new("tenant"));- let outer = FS_CTX- .scope(outer_ctx, async {- manager- .acquire_exact("/local/tenant/file.txt", std::time::Duration::ZERO, None)- .await- })- .await- .unwrap();- let ctx = Arc::new(FsContextInner::with_pathlock(- "tenant",- PathLockContext {- lease_ref: Some(outer.lease.lease_ref.clone()),- disable_auto_pathlock: false,- },- ));- FS_CTX- .scope(ctx, async {- mfs.write("/local/tenant/file.txt", b"content", 0, WriteFlag::Create)- .await- })- .await- .unwrap();- manager.release(&outer).await.unwrap();- }-- #[tokio::test]- async fn test_build_multi_write_fs_rejects_exclude_target_field() {- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;- mfs.register_plugin(MockPlugin::new("primary")).await;- mfs.register_plugin(MockPlugin::new("backupfs")).await;-- let backups = BackendsConfig {- sync_type: "async".to_string(),- write_ack_count: None,- write_ack_timeout_ms: None,- write_concurrency: None,- retry_interval_ms: None,- retry_backoff_base_ms: None,- retry_max_retries_per_round: None,- retry_quarantine_after_failures: None,- read_probe_cache_ttl_ms: None,- items: vec![BackendItemConfig {- name: "backup1".to_string(),- backend: "backupfs".to_string(),- params: serde_json::Value::Null,- timeout: None,- encryption: None,- operations: None,- excludes: Some(vec![RedirectPolicy::FileExtensionPolicy {- extensions: vec!["\\.tmp$".to_string()],- target: Some(vec!["should-not-exist".to_string()]),- }]),- }],- };- let config = PluginConfig {- name: "primary".to_string(),- mount_path: "/local".to_string(),- params: HashMap::new(),- backups: Some(backups.clone()),- ..PluginConfig::default()- };-- let result = mfs.build_multi_write_fs(&config, &backups).await;- assert!(result.is_err());- assert!(matches!(result.err(), Some(Error::Config(_))));- }-- #[tokio::test]- async fn test_build_multi_write_fs_rejects_reserved_primary_backup_name() {- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;- mfs.register_plugin(MockPlugin::new("primary")).await;- mfs.register_plugin(MockPlugin::new("backupfs")).await;-- let backups = BackendsConfig {- sync_type: "async".to_string(),- write_ack_count: None,- write_ack_timeout_ms: None,- write_concurrency: None,- retry_interval_ms: None,- retry_backoff_base_ms: None,- retry_max_retries_per_round: None,- retry_quarantine_after_failures: None,- read_probe_cache_ttl_ms: None,- items: vec![BackendItemConfig {- name: "primary".to_string(),- backend: "backupfs".to_string(),- params: serde_json::Value::Null,- timeout: None,- encryption: None,- operations: None,- excludes: None,- }],- };- let config = PluginConfig {- name: "primary".to_string(),- mount_path: "/local".to_string(),- params: HashMap::new(),- backups: Some(backups.clone()),- ..PluginConfig::default()- };-- let result = mfs.build_multi_write_fs(&config, &backups).await;- assert!(result.is_err());- assert!(matches!(result.err(), Some(Error::Config(_))));- }-#[tokio::test]async fn test_single_backend_mount_wraps_stats_outside_encryption() {use crate::lock::{@@ -2552,83 +2290,4 @@ mod tests {"single-backend encrypted mount should place encryption under stats");}-- #[tokio::test]- async fn test_multiwrite_mount_wraps_stats_outside_multiwrite() {- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;- mfs.register_plugin(MockPlugin::new("primary")).await;- mfs.register_plugin(MockPlugin::new("backupfs")).await;-- mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))- .await- .unwrap();-- let mounts = mfs.mounts.read().await;- let mount_info = mounts.get("/local").expect("mounted entry should exist");- let wrapped = (mount_info.fs.as_ref() as &dyn std::any::Any)- .downcast_ref::<StatsWrappedFS>()- .expect("stats wrapper should be outermost");- assert!(- (wrapped.inner_fs().as_ref() as &dyn std::any::Any)- .downcast_ref::<MultiWriteWrappedFS>()- .is_some(),- "multi-write mount should place MultiWriteWrappedFS directly under stats"- );- }-- #[tokio::test]- async fn test_multiwrite_raw_access_is_rejected() {- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;- mfs.register_plugin(MockPlugin::new("primary")).await;- mfs.register_plugin(MockPlugin::new("backupfs")).await;-- mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))- .await- .unwrap();-- let read_err = mfs.read_raw("/local/data.txt", 0, 0).await.unwrap_err();- assert!(matches!(read_err, Error::InvalidOperation(_)));-- let write_err = mfs- .write_raw("/local/data.txt", b"raw", WriteFlag::Create)- .await- .unwrap_err();- assert!(matches!(write_err, Error::InvalidOperation(_)));- }-- #[tokio::test]- async fn test_mountable_multiwrite_admin_smoke() {- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;- mfs.register_plugin(MockPlugin::new("primary")).await;- mfs.register_plugin(MockPlugin::new("backupfs")).await;-- mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))- .await- .unwrap();-- let status = crate::core::FS_CTX- .scope(- Arc::new(crate::core::context::FsContextInner::new(- "acct".to_string(),- )),- async { mfs.system_sync_status("/local").await.unwrap() },- )- .await;- assert_eq!(status["path"], "/");- assert_eq!(status["entry_count"], 0);- assert_eq!(status["pending_target_count"], 0);- assert_eq!(status["capabilities"]["multi_instance_safe"], false);-- let retry = crate::core::FS_CTX- .scope(- Arc::new(crate::core::context::FsContextInner::new(- "acct".to_string(),- )),- async { mfs.system_sync_retry("/local").await.unwrap() },- )- .await;- assert_eq!(retry["path"], "/");- assert_eq!(retry["retried"], 0);- assert_eq!(retry["failed"], 0);- }}diff --git a/crates/ragfs/src/core/multibackend_wrapper.rs b/crates/ragfs/src/core/multibackend_wrapper.rsindex c8f8149d8..8d5553a37 100644--- a/crates/ragfs/src/core/multibackend_wrapper.rs+++ b/crates/ragfs/src/core/multibackend_wrapper.rs@@ -1,1415 +1,591 @@-//! Multi-write wrapper — routes operations across primary and backup backends.-//!-//! Implements `MultiWriteWrappedFS` which handles:-//! - Write fanout to primary + backup backends (sync/async)-//! - Read routing with priority-based fallback chain-//! - Redirect policy evaluation-//! - Exclude policy filtering-//! - `.redirect.json` / `.sync_log.json` metadata management+//! Primary filesystem wrapper that submits V2 multi-write metadata events.-use std::collections::{HashMap, HashSet};-use std::future::Future;-use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};+use std::collections::{BTreeMap, BTreeSet};use std::sync::Arc;use std::time::Duration;use async_trait::async_trait;-use futures::stream::{self, StreamExt};-use regex::Regex;-use serde_json::Value;-use tokio::sync::{Mutex, Notify};-use tracing::warn;+use uuid::Uuid;use super::context::{FsContext, FS_CTX};use super::encryption_wrapper::EncryptionWrappedFS;use super::errors::{Error, Result};-use super::filesystem::{- apply_read_dir_options, normalize_prefix_path, paginate_entries, relative_depth,- relative_match_file, FileSystem,-};+use super::filesystem::{normalize_prefix_path, FileSystem};use super::types::{- BackendRole, BackendSyncState, FileInfo, GlobEntry, GlobPage, GrepOptions, GrepResult,- ListSortBy, OperationItemConfig, RedirectEntry, RedirectPolicy, SortOrder, SyncLogEntry,- SyncOp, SyncType, TreeEntry, WriteFlag,-};-use crate::core::glob::{- compare_rel_paths, decode_offset_token, encode_offset_token, PreparedGlob,+ BackendRole, FileInfo, GlobPage, GrepOptions, GrepResult, ListSortBy, SortOrder, TreeEntry,+ WriteFlag,};-use crate::core::internal_names::is_hidden_internal_name;+use crate::core::internal_names::is_multiwrite_internal_path;+use crate::metrics::{RagfsMetric, RagfsMetricValue};+use crate::multibackend::catch_up::CatchUpTarget;use crate::multibackend::meta::{- current_required_ctx, file_name, parent_dir, DefaultFsContextResolver, FsContextResolver,- MetaStateStore, PathSerializer, MULTIWRITE_INTERNAL_NAMES,+ current_required_ctx, DefaultFsContextResolver, FsContextResolver, MetadataStore,+ MultiWriteWorker,};+use crate::multibackend::migration::{FullDataImporter, ProtocolDetector};+use crate::multibackend::model::{+ PartitionState, PartitionsManifest, PendingEvent, PendingEventKind, ProtocolState,+ ProtocolStatus, ProtocolVersion, SegmentEventType, SegmentManifest,+};+use crate::multibackend::provider::{bootstrap_account, MultiWriteProvider};+use crate::multibackend::runtime::MultiWriteRuntime;-mod retry;-mod routing;-#[cfg(test)]-mod tests;--/// Default chunk size used when copying file state from primary to backup.-const DEFAULT_COPY_CHUNK_SIZE: usize = 8 * 1024 * 1024;-/// Default retry loop interval.-const DEFAULT_RETRY_INTERVAL: Duration = Duration::from_secs(30);-/// Default retry backoff base in milliseconds.-const DEFAULT_RETRY_BACKOFF_BASE_MS: u64 = 1000;-/// Default maximum retries per file per retry round.-const DEFAULT_MAX_RETRIES_PER_ROUND: usize = 3;-/// Default failure threshold before a target is quarantined.-const DEFAULT_QUARANTINE_AFTER_FAILURES: u32 = 9;-/// Default wait timeout when shutting down background tasks.-const DEFAULT_SHUTDOWN_WAIT: Duration = Duration::from_secs(5);-/// Maximum concurrent metadata reads when merging redirect entries into tree output.-const DEFAULT_TREE_REDIRECT_META_CONCURRENCY: usize = 32;--#[derive(Debug)]-struct SyncFanoutTaskResult {- backend: String,- result: Result<()>,-}--#[derive(Clone)]-enum BackupWriteOp {- Replay(SyncOp),- WriteFile {- data: Arc<Vec<u8>>,- offset: u64,- flags: WriteFlag,- },- EnsureParentDirs {- mode: u32,- },-}--impl BackupWriteOp {- /// Apply this backup-side operation under the provided request context.- async fn apply(- &self,- inner: &Inner,- backup: Arc<dyn FileSystem>,- path: &str,- ctx: &FsContext,- ) -> Result<()> {- match self {- Self::Replay(op) => {- op.replay(inner.primary().backend.clone(), backup, path, ctx)- .await- }- Self::WriteFile {- data,- offset,- flags,- } => {- let data = Arc::clone(data);- let offset = *offset;- let flags = *flags;- FS_CTX- .scope(ctx.clone(), async move {- backup.ensure_parent_dirs(path, 0o755).await?;- backup.write(path, data.as_slice(), offset, flags).await?;- Ok(())- })- .await- }- Self::EnsureParentDirs { mode } => {- let mode = *mode;- FS_CTX- .scope(ctx.clone(), async move {- backup.ensure_parent_dirs(path, mode).await- })- .await- }- }- }-}--/// Cloned target data passed to fanout strategies without borrowing `Inner`.-#[derive(Clone)]-struct FanoutTarget {- name: String,- backend: Arc<dyn FileSystem>,-}--/// Effective sync work item with resolved target backend names.-pub(crate) struct SyncWorkEntry {- pub(crate) file_path: String,- pub(crate) entry: SyncLogEntry,- pub(crate) targets: Vec<String>,-}--#[derive(Clone, Copy)]-enum ReadRouteSource {- Backup,- Primary,- Redirect,- Miss,-}--/// Builder-style sync mode configuration.-pub enum SyncMode {- /// Synchronous fanout requiring backup acknowledgement.- Sync {- /// Minimum backup acknowledgements required for a successful write.- ack_count: usize,- /// Maximum time to wait for backup acknowledgements, in milliseconds.- timeout_ms: u64,- },- /// Asynchronous fanout with background retry.- Async,-}+const COPY_CHUNK_SIZE: usize = 8 * 1024 * 1024;-/// A backend entry within the multi-write wrapper.+/// One configured primary or backup filesystem.pub struct BackendEntry {- /// Logical name (globally unique)+ /// Logical backend name.pub name: String,- /// Role: Primary or Backup+ /// Backend role.pub role: BackendRole,- /// The backend filesystem handle (may be encrypted)+ /// Filesystem handle, including encryption when configured.pub backend: Arc<dyn FileSystem>,- /// Optional raw backend handle used by primary verbatim copy fast-paths.+ /// Optional raw filesystem handle.pub raw_backend: Option<Arc<dyn FileSystem>>,- /// Operations this backend participates in (only for Backup)- pub operations: Vec<OperationItemConfig>,- /// Exclude policies (only for Backup)- pub excludes: Vec<RedirectPolicy>,-}--impl BackendEntry {- /// Check if this backend participates in read operations.- fn participates_in_read(&self) -> bool {- self.operations.iter().any(|op| op.operation == "read")- }-- /// Check if this backend participates in write operations.- /// Backups default to write-enabled when operations is empty.- fn participates_in_write(&self) -> bool {- if self.operations.is_empty() {- true- } else {- self.operations.iter().any(|op| op.operation == "write")- }- }-- /// Get read priority (lower = higher priority). Returns None if not read-enabled.- fn read_priority(&self) -> Option<u32> {- self.operations- .iter()- .find(|op| op.operation == "read")- .map(|op| op.priority)- }-- /// Convert this backend entry into a fanout target.- fn fanout_target(&self) -> FanoutTarget {- FanoutTarget {- name: self.name.clone(),- backend: self.backend.clone(),- }- }-}--/// File policy trait — shared by redirects and excludes.-pub trait FilePolicy {- /// Check if this policy matches the given file.- fn matches(&self, path: &str, size: u64) -> bool;-}--impl FilePolicy for RedirectPolicy {- fn matches(&self, path: &str, size: u64) -> bool {- match self {- RedirectPolicy::FileOverSizePolicy { max_size_mb, .. } => {- let max_bytes = max_size_mb * 1024 * 1024;- size > max_bytes- }- RedirectPolicy::FileExtensionPolicy { extensions, .. } => {- let name = file_name(path);- extensions.iter().any(|ext_pattern| {- if let Ok(re) = Regex::new(ext_pattern) {- re.is_match(name)- } else {- name.ends_with(ext_pattern.as_str())- }- })- }- }- }-}--impl SyncOp {- /// Replay this operation on one backup backend using the original request semantics.- async fn replay(- &self,- primary: Arc<dyn FileSystem>,- backup: Arc<dyn FileSystem>,- file_path: &str,- ctx: &FsContext,- ) -> Result<()> {- match self {- SyncOp::SyncFile { size } => {- let size = *size;- copy_current_primary_state(primary, backup, file_path, size, ctx).await- }- SyncOp::Create => {- FS_CTX- .scope(ctx.clone(), async { backup.create(file_path).await })- .await- }- SyncOp::Mkdir { mode } => {- let mode = *mode;- FS_CTX- .scope(ctx.clone(), async { backup.mkdir(file_path, mode).await })- .await- }- SyncOp::Remove => {- match FS_CTX- .scope(ctx.clone(), async { backup.remove(file_path).await })- .await- {- Ok(()) | Err(Error::NotFound(_)) => Ok(()),- Err(e) => Err(e),- }- }- SyncOp::RemoveAll => {- match FS_CTX- .scope(ctx.clone(), async { backup.remove_all(file_path).await })- .await- {- Ok(()) | Err(Error::NotFound(_)) => Ok(()),- Err(e) => Err(e),- }- }- SyncOp::Rename { to } => {- let to = to.clone();- FS_CTX- .scope(ctx.clone(), async {- backup.ensure_parent_dirs(&to, 0o755).await?;- backup.rename(file_path, &to).await- })- .await- }- SyncOp::Chmod { mode } => {- let mode = *mode;- FS_CTX- .scope(ctx.clone(), async { backup.chmod(file_path, mode).await })- .await- }- }- }-}--/// Copy one file between two filesystem handles in bounded-size chunks.-async fn copy_file_state(- source: Arc<dyn FileSystem>,- source_path: &str,- destination: Arc<dyn FileSystem>,- destination_path: &str,- size: u64,- ctx: &FsContext,-) -> Result<()> {- FS_CTX- .scope(ctx.clone(), async {- destination- .ensure_parent_dirs(destination_path, 0o755)- .await?;- if size == 0 {- if destination.exists(destination_path).await {- return destination.truncate(destination_path, 0).await;- }- return destination.create(destination_path).await;- }-- let mut offset = 0u64;- while offset < size {- let chunk_len = (size - offset).min(DEFAULT_COPY_CHUNK_SIZE as u64);- let chunk = source.read(source_path, offset, chunk_len).await?;- if chunk.is_empty() {- if offset == 0 {- if destination.exists(destination_path).await {- destination.truncate(destination_path, 0).await?;- } else {- destination.create(destination_path).await?;- }- }- break;- }- let flag = if offset == 0 {- WriteFlag::Create- } else {- WriteFlag::None- };- destination- .write(destination_path, &chunk, offset, flag)- .await?;- offset = offset.saturating_add(chunk.len() as u64);- }- Ok(())- })- .await-}--/// Copy the current file state from primary to backup in bounded-size chunks.-async fn copy_current_primary_state(- primary: Arc<dyn FileSystem>,- backup: Arc<dyn FileSystem>,- file_path: &str,- size: u64,- ctx: &FsContext,-) -> Result<()> {- copy_file_state(primary, file_path, backup, file_path, size, ctx).await-}--/// Copy one file within the primary raw backend without decrypting and re-encrypting bytes.-async fn copy_raw_primary_state(- primary_raw: Arc<dyn FileSystem>,- source_path: &str,- destination_path: &str,- ctx: &FsContext,-) -> Result<()> {- let raw_size = FS_CTX- .scope(ctx.clone(), async {- let source_info = primary_raw.stat(source_path).await?;- if source_info.is_dir {- return Err(Error::IsADirectory(source_path.to_string()));- }- Ok::<u64, Error>(source_info.size)- })- .await?;-- copy_file_state(- primary_raw.clone(),- source_path,- primary_raw,- destination_path,- raw_size,- ctx,- )- .await}-/// Inner state shared via Arc for async spawn and retry_loop.pub(crate) struct Inner {- /// All backend entries (primary at index 0)- backends: Vec<BackendEntry>,- /// Index of the primary backend- primary_idx: usize,- /// Sync type: Async or Sync.- sync_type: SyncType,- /// Minimum backup ack count for sync mode- write_ack_count: usize,- /// Timeout for waiting backup ack in sync mode (ms)- write_ack_timeout_ms: u64,- /// Semaphore for async write concurrency control- write_sem: Option<Arc<tokio::sync::Semaphore>>,- /// Primary redirect policies- redirects: Vec<RedirectPolicy>,- /// Metadata store (encrypted via primary_backend)- pub(crate) meta_store: MetaStateStore,- /// Per-path serialization queues- path_queues: PathSerializer,- /// Directories that currently have outstanding retry work.- pending_dirs: Mutex<HashSet<String>>,- /// Retry loop interval.- retry_interval: Duration,- /// Base retry backoff in milliseconds.- pub(crate) retry_backoff_base_ms: u64,- /// Maximum retry attempts for one target in one round.- pub(crate) max_retry_per_round: usize,- /// Failure threshold before quarantining one target.- quarantine_after_failures: u32,- /// Number of background tasks currently in flight.- background_tasks: AtomicUsize,- /// Notifier fired when background task count reaches zero.- idle_notify: Notify,- /// Read route hit metrics.- read_backup_hits: AtomicU64,- read_primary_hits: AtomicU64,- read_redirect_hits: AtomicU64,- read_misses: AtomicU64,- /// Cancellation flag for the background retry loop.- retry_cancelled: AtomicBool,- /// Wake-up signal used to stop retry_loop promptly on drop.- retry_shutdown: Notify,+ primary: BackendEntry,+ runtime: MultiWriteRuntime,+ ctx_resolver: Arc<dyn FsContextResolver>,}-/// Multi-write wrapped filesystem.+/// Filesystem wrapper that commits primary operations before queuing V2 events.+#[derive(Clone)]pub struct MultiWriteWrappedFS {pub(crate) inner: Arc<Inner>,+ metadata_store: Arc<MetadataStore>,+ metadata_provider: Arc<dyn MultiWriteProvider>,}-/// Builder for `MultiWriteWrappedFS`.+/// Builder for one V2 multi-write wrapper and its flush runtime.pub struct MultiWriteWrappedFSBuilder {primary_backend: Arc<dyn FileSystem>,primary_raw_backend: Option<Arc<dyn FileSystem>>,backup_entries: Vec<BackendEntry>,- redirects: Vec<RedirectPolicy>,- sync_mode: SyncMode,- write_concurrency: Option<usize>,- retry_interval: Duration,- retry_backoff_base_ms: u64,- max_retry_per_round: usize,- quarantine_after_failures: u32,+ metadata_store: Option<Arc<MetadataStore>>,+ metadata_provider: Option<Arc<dyn MultiWriteProvider>>,+ initial_partitions: u32,+ checkpoint_interval: Duration,ctx_resolver: Arc<dyn FsContextResolver>,}impl MultiWriteWrappedFSBuilder {- /// Attach the raw primary backend so the wrapper can do verbatim primary copies.+ /// Attach the raw primary backend used by internal operations and physical copies.pub fn with_primary_raw_backend(mut self, primary_raw_backend: Arc<dyn FileSystem>) -> Self {self.primary_raw_backend = Some(primary_raw_backend);self}- /// Set backup backend entries on the builder.+ /// Attach configured backup handles for later catch-up workers.pub fn with_backups(mut self, backup_entries: Vec<BackendEntry>) -> Self {self.backup_entries = backup_entries;self}- /// Set redirect policies for the primary backend.- pub fn with_redirects(mut self, redirects: Vec<RedirectPolicy>) -> Self {- self.redirects = redirects;+ /// Attach the metadata store used for account initialization and routing.+ pub fn with_metadata_store(mut self, metadata_store: Arc<MetadataStore>) -> Self {+ self.metadata_store = Some(metadata_store);self}- /// Select the sync mode used by write fanout.- pub fn sync_mode(mut self, sync_mode: SyncMode) -> Self {- self.sync_mode = sync_mode;+ /// Attach the provider used by the background flush worker.+ pub fn with_metadata_provider(+ mut self,+ metadata_provider: Arc<dyn MultiWriteProvider>,+ ) -> Self {+ self.metadata_provider = Some(metadata_provider);self}- /// Set the maximum number of concurrent async backup writes.- pub fn write_concurrency(mut self, write_concurrency: Option<usize>) -> Self {- self.write_concurrency = write_concurrency;+ /// Set the initial metadata partition count for newly observed accounts.+ pub fn initial_partitions(mut self, initial_partitions: u32) -> Self {+ self.initial_partitions = initial_partitions;self}- /// Configure retry loop interval.- pub fn retry_interval(mut self, retry_interval: Duration) -> Self {- self.retry_interval = retry_interval;+ /// Set the interval between checkpoint discovery rounds.+ pub fn checkpoint_interval(mut self, checkpoint_interval: Duration) -> Self {+ self.checkpoint_interval = checkpoint_interval;self}- /// Configure retry backoff base duration in milliseconds.- pub fn retry_backoff_base_ms(mut self, retry_backoff_base_ms: u64) -> Self {- self.retry_backoff_base_ms = retry_backoff_base_ms;- self- }-- /// Configure the maximum number of retries per round.- pub fn max_retry_per_round(mut self, max_retry_per_round: usize) -> Self {- self.max_retry_per_round = max_retry_per_round.max(1);- self- }-- /// Configure quarantine threshold for one path/backup pair.- pub fn quarantine_after_failures(mut self, quarantine_after_failures: u32) -> Self {- self.quarantine_after_failures = quarantine_after_failures.max(1);- self- }-- /// Configure the context resolver used by retry and admin background paths.+ /// Configure the resolver for mount-relative foreground paths.pub fn ctx_resolver(mut self, ctx_resolver: Arc<dyn FsContextResolver>) -> Self {self.ctx_resolver = ctx_resolver;self}- /// Build the multi-write wrapper and start the retry loop when needed.- pub fn build(self) -> Result<MultiWriteWrappedFS> {- let mut backends = Vec::new();- backends.push(BackendEntry {+ /// Build the wrapper after asynchronously starting its V2 runtime.+ pub async fn build(self) -> Result<MultiWriteWrappedFS> {+ let fs = self.build_inactive().await?;+ fs.activate();+ Ok(fs)+ }++ /// Build the wrapper while keeping its V2 runtime inactive.+ pub(crate) async fn build_inactive(self) -> Result<MultiWriteWrappedFS> {+ let metadata_store = self+ .metadata_store+ .ok_or_else(|| Error::config("multi-write metadata store is required"))?;+ let metadata_provider = self+ .metadata_provider+ .ok_or_else(|| Error::config("multi-write metadata provider is required"))?;+ let backup_names: Vec<String> = self+ .backup_entries+ .iter()+ .map(|entry| entry.name.clone())+ .collect();+ let catch_up_targets = self+ .backup_entries+ .iter()+ .map(|entry| CatchUpTarget {+ backend_id: entry.name.clone(),+ backend: entry.backend.clone(),+ })+ .collect();+ let protocol = ProtocolDetector::new(metadata_store.clone())+ .detect()+ .await?;+ let configured_backups = backup_names.iter().collect::<BTreeSet<_>>();+ for account in metadata_store.initialized_accounts().await? {+ let path = metadata_store.paths().account_manifest(&account)?.0;+ let manifest: PartitionsManifest = match metadata_store.read_json(&path).await {+ Ok(manifest) => manifest,+ Err(Error::NotFound(_)) => continue,+ Err(error) => return Err(error),+ };+ for &partition_id in manifest.partitions.keys() {+ let path = metadata_store+ .paths()+ .segment_manifest(&account, partition_id)?+ .0;+ let segments: SegmentManifest = metadata_store.read_json(&path).await?;+ segments.validate()?;+ let persisted = segments.backend_states.keys().collect::<BTreeSet<_>>();+ if persisted != configured_backups {+ return Err(Error::not_supported(format!(+ "backup membership changed for account {account}"+ )));+ }+ }+ bootstrap_account(+ metadata_provider.as_ref(),+ metadata_store.as_ref(),+ &account,+ &manifest,+ )+ .await?;+ }+ let runtime = MultiWriteRuntime::prepare(+ metadata_store.clone(),+ metadata_provider.clone(),+ self.initial_partitions,+ backup_names.clone(),+ Some(self.primary_backend.clone()),+ catch_up_targets,+ self.checkpoint_interval,+ )+ .await;+ let primary = BackendEntry {name: "primary".to_string(),role: BackendRole::Primary,- backend: self.primary_backend.clone(),- raw_backend: self.primary_raw_backend.clone(),- operations: Vec::new(),- excludes: Vec::new(),- });- backends.extend(self.backup_entries);-- let (sync_type, write_ack_count, write_ack_timeout_ms) = match self.sync_mode {- SyncMode::Sync {- ack_count,- timeout_ms,- } => (SyncType::Sync, ack_count, timeout_ms),- SyncMode::Async => (SyncType::Async, usize::MAX, 0),+ backend: self.primary_backend,+ raw_backend: self.primary_raw_backend,};-- let write_sem = self- .write_concurrency- .filter(|&n| n > 0)- .map(|n| Arc::new(tokio::sync::Semaphore::new(n)));-- let meta_store = MetaStateStore::new(self.primary_backend, self.ctx_resolver);-- let inner = Arc::new(Inner {- backends,- primary_idx: 0,- sync_type,- write_ack_count,- write_ack_timeout_ms,- write_sem,- redirects: self.redirects,- meta_store,- path_queues: PathSerializer::new(),- pending_dirs: Mutex::new(HashSet::new()),- retry_interval: self.retry_interval,- retry_backoff_base_ms: self.retry_backoff_base_ms,- max_retry_per_round: self.max_retry_per_round,- quarantine_after_failures: self.quarantine_after_failures,- background_tasks: AtomicUsize::new(0),- idle_notify: Notify::new(),- read_backup_hits: AtomicU64::new(0),- read_primary_hits: AtomicU64::new(0),- read_redirect_hits: AtomicU64::new(0),- read_misses: AtomicU64::new(0),- retry_cancelled: AtomicBool::new(false),- retry_shutdown: Notify::new(),- });-- // Start retry_loop if there are write-enabled backups.- if inner.write_backups().next().is_some() {- inner.background_task_started();- tokio::spawn(Inner::retry_loop(Arc::clone(&inner)));+ let fs = MultiWriteWrappedFS {+ inner: Arc::new(Inner {+ primary,+ runtime,+ ctx_resolver: self.ctx_resolver,+ }),+ metadata_store: metadata_store.clone(),+ metadata_provider: metadata_provider.clone(),+ };+ if protocol.status != ProtocolStatus::Stable {+ let importer_primary: Arc<dyn FileSystem> = Arc::new(fs.clone());+ let importer = FullDataImporter::new(+ importer_primary,+ metadata_store,+ metadata_provider,+ self.initial_partitions,+ backup_names,+ );+ fs.inner.runtime.set_import_worker(async move {+ let result = importer.run().await;+ if let Err(error) = &result {+ tracing::error!(error = %error, "multi-write full-data import failed");+ }+ result+ });}-- Ok(MultiWriteWrappedFS { inner })+ Ok(fs)}}impl MultiWriteWrappedFS {- /// Return this wrapper's current background task count without scanning metadata.- pub(crate) fn background_task_count(&self) -> usize {- self.inner.background_tasks.load(Ordering::SeqCst)- }-- /// Start building a multi-write wrapper from a primary backend.+ /// Start building a wrapper around the supplied primary filesystem.pub fn builder(primary_backend: Arc<dyn FileSystem>) -> MultiWriteWrappedFSBuilder {MultiWriteWrappedFSBuilder {primary_backend,primary_raw_backend: None,backup_entries: Vec::new(),- redirects: Vec::new(),- sync_mode: SyncMode::Async,- write_concurrency: None,- retry_interval: DEFAULT_RETRY_INTERVAL,- retry_backoff_base_ms: DEFAULT_RETRY_BACKOFF_BASE_MS,- max_retry_per_round: DEFAULT_MAX_RETRIES_PER_ROUND,- quarantine_after_failures: DEFAULT_QUARANTINE_AFTER_FAILURES,+ metadata_store: None,+ metadata_provider: None,+ initial_partitions: 16,+ checkpoint_interval: Duration::from_secs(86_400),ctx_resolver: Arc::new(DefaultFsContextResolver),}}+ /// Release prepared runtime work after the wrapper becomes reachable.+ pub(crate) fn activate(&self) {+ self.inner.runtime.activate();+ }+/// Return the raw primary backend for mount-level internal-name operations.pub(crate) fn primary_raw_backend(&self) -> Option<Arc<dyn FileSystem>> {- self.inner.primary().raw_backend.clone()+ self.inner.primary.raw_backend.clone()}- /// Return whether encrypted backends own dual-path PathLock acquisition.+ /// Return whether the primary encryption wrapper owns PathLock acquisition.pub(crate) fn encryption_handles_pathlock(&self) -> bool {- let any = self.inner.primary().backend.as_ref() as &dyn std::any::Any;+ let any = self.inner.primary.backend.as_ref() as &dyn std::any::Any;any.downcast_ref::<EncryptionWrappedFS>().is_some()}-}--impl Inner {- /// Build the per-path/per-backend queue key used by both fanout and retry.- fn backup_queue_key(path: &str, backup_name: &str) -> String {- format!("{}\0{}", path, backup_name)- }-- /// Iterate over write-enabled backup entries.- fn write_backups(&self) -> impl Iterator<Item = &BackendEntry> {- self.backends[self.primary_idx + 1..]- .iter()- .filter(|be| be.participates_in_write())- }-- /// Resolve write-enabled backup targets after applying exclude policies.- fn write_targets(&self, path: &str, size: u64) -> Vec<FanoutTarget> {- self.write_backups()- .filter(|be| !self.is_excluded(be, path, size))- .map(BackendEntry::fanout_target)- .collect()- }-- /// Resolve explicitly named backup targets.- fn named_targets(&self, target_names: &[String]) -> Vec<FanoutTarget> {- target_names- .iter()- .filter_map(|name| self.backup_by_name(name))- .map(BackendEntry::fanout_target)- .collect()- }-- /// Execute one backup write synchronously so redirect visibility only appears- /// after at least one target has durably materialized the file contents.- async fn write_first_target(- inner: &Arc<Inner>,- path: &str,- target: &FanoutTarget,- ctx: &FsContext,- op: BackupWriteOp,- ) -> Result<()> {- let queue_key = Self::backup_queue_key(path, &target.name);- let fs = target.backend.clone();- let ack_ctx = ctx.clone();-- inner- .path_queues- .with_path_lock(&queue_key, || async move {- op.apply(inner, fs, path, ctx).await- })- .await?;-- inner- .update_backup_acked_seq(path, &target.name, &ack_ctx)- .await- }-- /// Resolve effective target backend names for sync/retry work.- pub(crate) fn target_backend_names(- &self,- redirect_meta: &super::types::RedirectMeta,- file_name: &str,- file_path: &str,- sync_entry: &SyncLogEntry,- ) -> Vec<String> {- if let Some(redir) = redirect_meta.entries.get(file_name) {- return redir.targets.clone();- }- let policy_size = self.retry_policy_size(sync_entry);- self.write_backups()- .filter(|be| !self.is_excluded(be, file_path, policy_size))- .map(|be| be.name.clone())- .collect()- }- /// Iterate over read-enabled backup entries sorted by priority.- fn read_backups_sorted(&self) -> Vec<&BackendEntry> {- let mut read_backups: Vec<&BackendEntry> = self.backends[self.primary_idx + 1..]- .iter()- .filter(|be| be.participates_in_read())- .collect();- read_backups.sort_by_key(|be| be.read_priority().unwrap_or(u32::MAX));- read_backups- }-- /// Get the primary backend entry.- pub(crate) fn primary(&self) -> &BackendEntry {- &self.backends[self.primary_idx]- }-- /// Get a backup entry by name.- fn backup_by_name(&self, name: &str) -> Option<&BackendEntry> {- self.backends.iter().find(|be| be.name == name)- }-- /// Check if a file should be excluded from a backup.- fn is_excluded(&self, backup: &BackendEntry, path: &str, size: u64) -> bool {- backup- .excludes- .iter()- .any(|policy| policy.matches(path, size))- }-- /// Check if a file matches any redirect policy.- fn check_redirect(&self, path: &str, size: u64) -> Option<Vec<String>> {- for policy in &self.redirects {- if policy.matches(path, size) {- let targets = match policy {- RedirectPolicy::FileOverSizePolicy { target, .. } => target.clone(),- RedirectPolicy::FileExtensionPolicy { target, .. } => target.clone(),- };- return targets;- }- }- None- }-- /// Resolve persisted redirect targets for one already-written file path.- async fn redirect_targets_for_path(- &self,- path: &str,- ctx: &FsContext,- ) -> Result<Option<Vec<String>>> {- if self.redirects.is_empty() {- return Ok(None);- }-- let dir = parent_dir(path);- let name = file_name(path);- let redirect_meta = self.meta_store.get_redirect_meta(&dir, ctx).await?;- Ok(redirect_meta- .entries- .get(name)- .map(|entry| entry.targets.clone()))- }-- /// Generate and persist the next sequence number.- async fn next_seq(&self) -> Result<u64> {- self.meta_store.next_seq().await- }-- /// Resolve a file size for retry-time policy decisions.- fn retry_policy_size(&self, sync_entry: &SyncLogEntry) -> u64 {- match &sync_entry.op {- SyncOp::SyncFile { size } => *size,- _ => 0,- }- }-- /// Execute the primary-write, sync-log and backup-fanout pipeline.- async fn execute_write<R, P, PFut>(- inner: &Arc<Self>,- path: String,- size: u64,- sync_op: Option<SyncOp>,- backup_op: Option<BackupWriteOp>,- primary_fn: P,- ) -> Result<R>- where- P: FnOnce(Arc<dyn FileSystem>) -> PFut + Send,- PFut: Future<Output = Result<R>> + Send,- {- let ctx = current_required_ctx()?;- inner.invalidate_read_route(&path).await;-- let prepared_entry = if let Some(entry) = sync_op {- let dir = parent_dir(&path);- let name = file_name(&path).to_string();- let seq = inner.next_seq().await?;- inner- .meta_store- .update_dir_meta(&dir, &ctx, {- let name = name.clone();- move |_redirect, sync_log| {- sync_log.entries.insert(name, SyncLogEntry::new(seq, entry));- Ok(())- }- })- .await?;- Some((dir, name, seq))- } else {- None- };-- let result = match FS_CTX- .scope(ctx.clone(), primary_fn(inner.primary().backend.clone()))- .await- {- Ok(result) => result,- Err(err) => {- if let Some((dir, name, seq)) = prepared_entry.as_ref() {- let _ = inner- .meta_store- .update_dir_meta(dir, &ctx, {- let name = name.clone();- let seq = *seq;- move |_redirect, sync_log| {- let should_remove =- sync_log.entries.get(&name).is_some_and(|entry| {- entry.latest_seq == seq && !entry.is_primary_committed()- });- if should_remove {- sync_log.entries.remove(&name);- }- Ok(())- }- })- .await;+ /// Return one while the flush worker is active and zero after it exits.+ pub(crate) fn background_task_count(&self) -> usize {+ usize::from(self.inner.runtime.is_running())+ }++ /// Read and validate one complete V2 multi-write metrics snapshot.+ pub(crate) async fn metrics(&self) -> Result<Vec<RagfsMetric>> {+ let protocol_path = self.metadata_store.paths().mount_protocol().0;+ let protocol: ProtocolState = self.metadata_store.read_json(&protocol_path).await?;+ protocol.validate()?;+ let mut lag = BTreeMap::<(String, u32), u64>::new();+ for account in self.metadata_store.initialized_accounts().await? {+ let path = self.metadata_store.paths().account_manifest(&account)?.0;+ let manifest: PartitionsManifest = self.metadata_store.read_json(&path).await?;+ manifest.validate()?;+ for (&partition_id, entry) in &manifest.partitions {+ if entry.state != PartitionState::Stable {+ continue;}- return Err(err);- }- };-- if let Some((dir, name, seq)) = prepared_entry {- let mut commit_superseded = false;- inner- .meta_store- .update_dir_meta(&dir, &ctx, |_redirect, sync_log| {- let entry = sync_log.entries.get_mut(&name).ok_or_else(|| {- Error::internal(format!(- "prepared sync log entry missing while committing '{}'",- name- ))- })?;- if entry.latest_seq != seq {- warn!(- path = %path,- name = %name,- expected_seq = seq,- actual_seq = entry.latest_seq,- "prepared sync log entry was superseded while committing; skip stale backup fanout"- );- commit_superseded = true;- return Ok(());- }- entry.mark_primary_committed();- Ok(())- })- .await?;- if commit_superseded {- inner.refresh_pending_dir(&dir, &ctx).await?;- return Ok(result);- }- inner.mark_pending_dir(&dir).await;- }-- let dir = parent_dir(&path);- let fanout_result = if let Some(backup_op) = backup_op {- Inner::fanout_write(inner, &path, size, ctx.clone(), backup_op).await- } else {- Ok(())- };- inner.refresh_pending_dir(&dir, &ctx).await?;- fanout_result?;- Ok(result)- }-- /// Execute a write that may be redirected away from the primary backend.- async fn execute_write_with_redirect<P, PFut>(- inner: &Arc<Self>,- path: String,- size: u64,- sync_op: SyncOp,- backup_op: BackupWriteOp,- primary_fn: P,- ) -> Result<u64>- where- P: FnOnce(Arc<dyn FileSystem>) -> PFut + Send,- PFut: Future<Output = Result<u64>> + Send,- {- let ctx = current_required_ctx()?;- inner.invalidate_read_route(&path).await;-- if let Some(targets) = inner.check_redirect(&path, size) {- let dir = parent_dir(&path);- let name = file_name(&path).to_string();- let seq = inner.next_seq().await?;- let resolved_targets = inner.named_targets(&targets);- let first_target = resolved_targets.first().cloned().ok_or_else(|| {- Error::internal(format!(- "redirect path '{}' resolved no writable targets",- path- ))- })?;- Inner::write_first_target(inner, &path, &first_target, &ctx, backup_op.clone()).await?;- let targets_clone = targets.clone();- let first_target_name = first_target.name.clone();- inner- .meta_store- .update_dir_meta(&dir, &ctx, move |redirect, sync_log| {- redirect.entries.insert(- name.clone(),- RedirectEntry {- targets: targets_clone.clone(),- },- );- let mut committed = SyncLogEntry::committed(seq, sync_op);- committed- .backends- .insert(first_target_name.clone(), BackendSyncState::acked(seq));- sync_log.entries.insert(name, committed);- Ok(())- })- .await?;-- let remaining_targets: Vec<FanoutTarget> =- resolved_targets.into_iter().skip(1).collect();- if !remaining_targets.is_empty() {- inner.mark_pending_dir(&dir).await;- Inner::fanout_async(inner, &path, remaining_targets, &ctx, backup_op).await;- }- inner.refresh_pending_dir(&dir, &ctx).await?;- return Ok(size);- }-- Inner::execute_write(- inner,- path,- size,- Some(sync_op),- Some(backup_op),- primary_fn,- )- .await- }-- /// Fanout a write operation to all write-enabled backups.- /// Takes `&Arc<Inner>` so spawned tasks can clone the Arc for acked_seq updates.- /// `ctx` is required for encrypted backup backends and acked_seq updates.- async fn fanout_write(- inner: &Arc<Inner>,- path: &str,- size: u64,- ctx: FsContext,- op: BackupWriteOp,- ) -> Result<()> {- Inner::fanout_targets(inner, path, inner.write_targets(path, size), ctx, op).await- }-- /// Fanout a write operation to explicitly named backup targets (used by redirect path).- /// Resolves names to BackendEntry references, then delegates to sync/async state machine.- async fn fanout_write_to_targets(- inner: &Arc<Inner>,- path: &str,- target_names: &[String],- ctx: FsContext,- op: BackupWriteOp,- ) -> Result<()> {- Inner::fanout_targets(inner, path, inner.named_targets(target_names), ctx, op).await- }-- /// Fanout to already resolved targets using the configured sync mode.- async fn fanout_targets(- inner: &Arc<Inner>,- path: &str,- targets: Vec<FanoutTarget>,- ctx: FsContext,- op: BackupWriteOp,- ) -> Result<()> {- if targets.is_empty() {- if matches!(inner.sync_type, SyncType::Sync)- && inner.write_ack_count > 0- && inner.write_backups().next().is_some()- {- return Err(Error::SyncWriteQuorum {- succeeded: 0,- required: inner.write_ack_count,- attempted: 0,- failures: Vec::new(),- });- }- return Ok(());- }-- match inner.sync_type {- SyncType::Sync => Inner::fanout_sync(inner, path, &targets, &ctx, op).await,- SyncType::Async => {- Inner::fanout_async(inner, path, targets, &ctx, op).await;- Ok(())- }- }- }-- /// Synchronous fanout: execute writes in parallel, wait for quorum.- async fn fanout_sync(- inner: &Arc<Inner>,- path: &str,- targets: &[FanoutTarget],- ctx: &FsContext,- op: BackupWriteOp,- ) -> Result<()> {- let ack_count = inner.write_ack_count.min(targets.len());- let timeout = if inner.write_ack_timeout_ms > 0 {- Some(Duration::from_millis(inner.write_ack_timeout_ms))- } else {- None- };-- let path_owned = path.to_string();- let ctx = Some(ctx.clone());-- // Launch parallel tasks for all backup writes.- let mut handles = Vec::new();- for target in targets {- let fs = target.backend.clone();- let name = target.name.clone();- let path = path_owned.clone();- let inner = Arc::clone(inner);- let ctx = ctx.clone();- let op_clone = op.clone();- let queue_key = Self::backup_queue_key(&path, &name);-- handles.push(tokio::spawn(async move {- // Wrap in FS_CTX.scope so encrypted backends can access account_id.- let exec = inner.path_queues.with_path_lock(&queue_key, || async {- op_clone- .apply(&inner, fs, &path, ctx.as_ref().unwrap())- .await- });-- let result = if let Some(timeout) = timeout {- match tokio::time::timeout(timeout, exec).await {- Ok(Ok(())) => Ok(()),- Ok(Err(e)) => Err(e),- Err(_) => Err(Error::timeout(format!(- "backup '{}' timed out waiting for sync acknowledgement",- name- ))),- }- } else {- exec.await+ let scope = crate::multibackend::model::ScopeKey {+ account_id: account.clone(),+ partition_id,+ epoch: manifest.epoch,};-- // Update acked_seq on success.- if result.is_ok() {- if let Some(ref ctx) = ctx {- let _ = inner.update_backup_acked_seq(&path, &name, ctx).await;- }- }-- SyncFanoutTaskResult {- backend: name,- result,- }- }));- }-- let results = futures::future::join_all(handles).await;-- let mut successes = 0usize;- let mut failures = Vec::new();-- for result in results {- match result {- Ok(SyncFanoutTaskResult {- backend: _,- result: Ok(()),- }) => {- successes += 1;- }- Ok(SyncFanoutTaskResult {- backend,- result: Err(e),- }) => {- failures.push(super::errors::SyncWriteFailureDetail {- backend,- kind: e.kind_name().to_string(),- message: e.to_string(),- });- }- Err(e) => {- failures.push(super::errors::SyncWriteFailureDetail {- backend: "<spawn>".to_string(),- kind: "join_error".to_string(),- message: e.to_string(),- });+ let segments = self.metadata_provider.read_manifest(&scope).await?;+ segments.validate()?;+ let tail = segments.next_seq - 1;+ for (backend, state) in segments.backend_states {+ *lag.entry((backend, partition_id)).or_default() += tail - state.synced_seq;}}}-- if successes >= ack_count {- Ok(())- } else {- Err(Error::SyncWriteQuorum {- succeeded: successes,- required: ack_count,- attempted: targets.len(),- failures,- })- }- }-- /// Asynchronous fanout: spawn background tasks that update acked_seq on completion.- /// Uses per-path serialization to prevent out-of-order application on backup backends.- async fn fanout_async(- inner: &Arc<Inner>,- path: &str,- targets: Vec<FanoutTarget>,- ctx: &FsContext,- op: BackupWriteOp,- ) {- let path_owned = path.to_string();- let sem = inner.write_sem.clone();-- for target in targets {- let fs = target.backend.clone();- let name = target.name.clone();- let path = path_owned.clone();- let ctx = ctx.clone();- let sem = sem.clone();- let inner = Arc::clone(inner);- let op_clone = op.clone();- let queue_key = Self::backup_queue_key(&path, &name);- inner.background_task_started();-- tokio::spawn(async move {- {- // Per (path, backup) serialization preserves FIFO without blocking other backups.- let result = inner- .path_queues- .with_path_lock(&queue_key, || async {- let _permit = if let Some(ref sem) = sem {- sem.acquire().await.ok()- } else {- None- };-- op_clone.apply(&inner, fs, &path, &ctx).await- })- .await;-- // Update acked_seq on successful write.- if result.is_ok() {- let _ = inner.update_backup_acked_seq(&path, &name, &ctx).await;- }- }- inner.background_task_finished();+ let mut metrics = vec![+ RagfsMetric {+ name: "ragfs_multiwrite_background_tasks".into(),+ labels: BTreeMap::new(),+ value: RagfsMetricValue::Gauge(self.background_task_count() as f64),+ },+ RagfsMetric {+ name: "ragfs_multiwrite_pending_events".into(),+ labels: BTreeMap::new(),+ value: RagfsMetricValue::Gauge(self.inner.runtime.pending_event_count() as f64),+ },+ ];+ for (worker, label) in [+ (MultiWriteWorker::Flush, "flush"),+ (MultiWriteWorker::Checkpoint, "checkpoint"),+ (MultiWriteWorker::CatchUp, "catch_up"),+ (MultiWriteWorker::Gc, "gc"),+ ] {+ metrics.push(RagfsMetric::counter(+ "ragfs_multiwrite_errors_total",+ &[("worker", label)],+ self.metadata_store.worker_error_count(worker),+ 1.0,+ ));+ }+ metrics.push(RagfsMetric {+ name: "ragfs_multiwrite_protocol_version".into(),+ labels: BTreeMap::from([("version".into(), "v2".into())]),+ value: RagfsMetricValue::Gauge(u64::from(+ protocol.protocol_version == ProtocolVersion::V2,+ ) as f64),+ });+ for (status, value) in [+ (+ "migrating",+ u64::from(protocol.status == ProtocolStatus::Migrating),+ ),+ (+ "stable",+ u64::from(protocol.status == ProtocolStatus::Stable),+ ),+ ] {+ metrics.push(RagfsMetric {+ name: "ragfs_multiwrite_protocol_status".into(),+ labels: BTreeMap::from([("status".into(), status.into())]),+ value: RagfsMetricValue::Gauge(value as f64),});}+ metrics.extend(+ lag.into_iter()+ .map(|((backend, partition), value)| RagfsMetric {+ name: "ragfs_multiwrite_backend_lag_events".into(),+ labels: BTreeMap::from([+ ("backend".into(), backend),+ ("partition".into(), partition.to_string()),+ ]),+ value: RagfsMetricValue::Gauge(value as f64),+ }),+ );+ Ok(metrics)+ }++ /// Resolve one account context and reject disagreement between path and task.+ fn resolve_context(&self, path: &str) -> Result<FsContext> {+ let path_ctx = self.inner.ctx_resolver.resolve(path)?;+ match current_required_ctx() {+ Ok(ctx) if !ctx.account_id().trim().is_empty() => {+ if ctx.account_id() != path_ctx.account_id() {+ return Err(Error::invalid_path(format!(+ "path account '{}' conflicts with context account '{}'",+ path_ctx.account_id(),+ ctx.account_id()+ )));+ }+ Ok(ctx)+ }+ _ => Ok(path_ctx),+ }}- /// Record rename metadata and migrate redirect state when needed.- async fn record_rename_meta(+ /// Build one canonical V2 event from a foreground mount-relative path.+ fn pending_event(&self,- old_path: &str,- new_path: &str,- ctx: &FsContext,- ) -> Result<()> {- let source_dir = parent_dir(old_path);- let target_dir = parent_dir(new_path);- let old_name = file_name(old_path).to_string();- let new_name = file_name(new_path).to_string();- let seq = self.next_seq().await?;- let rename_op = SyncOp::Rename {- to: new_path.to_string(),- };-- if source_dir == target_dir {- self.meta_store- .update_dir_meta(&source_dir, ctx, move |redirect, sync_log| {- sync_log- .entries- .insert(old_name.clone(), SyncLogEntry::committed(seq, rename_op));- if let Some(redirect_entry) = redirect.entries.remove(&old_name) {- redirect.entries.insert(new_name, redirect_entry);- }- Ok(())- })- .await- } else {- self.meta_store- .update_dual_dir_meta(- &source_dir,- &target_dir,- ctx,- move |src_redirect, src_sync_log, tgt_redirect, _tgt_sync_log| {- src_sync_log- .entries- .insert(old_name.clone(), SyncLogEntry::committed(seq, rename_op));- if let Some(redirect_entry) = src_redirect.entries.remove(&old_name) {- tgt_redirect.entries.insert(new_name, redirect_entry);- }- Ok(())- },- )- .await+ path: &str,+ kind: SegmentEventType,+ destination_path: Option<&str>,+ ) -> Result<Option<PendingEvent>> {+ let ctx = self.resolve_context(path)?;+ let account_id = ctx.account_id().to_string();+ let mut path = logical_path(&account_id, path)?;+ let mut destination_path = destination_path+ .map(|destination| logical_path(&account_id, destination))+ .transpose()?;+ let mut kind = kind;+ let source_internal = is_multiwrite_internal_path(&path);+ let destination_internal = destination_path+ .as_deref()+ .is_some_and(is_multiwrite_internal_path);+ match (kind, source_internal, destination_internal) {+ (SegmentEventType::MoveTree, true, false) => {+ path = destination_path+ .take()+ .ok_or_else(|| Error::invalid_operation("move event requires a destination"))?;+ kind = SegmentEventType::RemoveTree;+ }+ (SegmentEventType::MoveTree, false, true) => {+ kind = SegmentEventType::RemoveTree;+ destination_path = None;+ }+ (_, true, _) | (_, _, true) => return Ok(None),+ _ => {}}+ Ok(Some(PendingEvent {+ operation_id: Uuid::new_v4(),+ account_id,+ kind: PendingEventKind::Data(kind),+ path,+ destination_path,+ route_hint: None,+ }))}-}--/// Return true when `path` falls under `exclude_path` (including itself).-fn is_excluded_grep_path(path: &str, exclude_path: Option<&str>) -> bool {- let Some(exclude_path) = exclude_path.map(normalize_prefix_path) else {- return false;- };- let normalized_path = normalize_prefix_path(path);- normalized_path == exclude_path- || normalized_path- .strip_prefix(&exclude_path)- .is_some_and(|suffix| suffix.starts_with('/'))-}--// ── FileSystem trait implementation ──--impl Drop for MultiWriteWrappedFS {- /// Signal retry_loop to exit when the wrapper is unmounted or dropped.- fn drop(&mut self) {- self.inner.retry_cancelled.store(true, Ordering::SeqCst);- self.inner.retry_shutdown.notify_waiters();- }-}-impl MultiWriteWrappedFS {- /// Stop background retry work and wait for in-flight async fanout to drain.- pub async fn shutdown(&self) -> Result<()> {- self.inner.retry_cancelled.store(true, Ordering::SeqCst);- self.inner.retry_shutdown.notify_waiters();- self.inner.wait_idle(DEFAULT_SHUTDOWN_WAIT).await+ /// Submit one prebuilt event after its primary operation has succeeded.+ fn submit_event(&self, event: Option<PendingEvent>) {+ if let Some(event) = event {+ self.inner.runtime.submit(event);+ }}- /// Copy one primary-resident file to a new path while preserving physical primary bytes.+ /// Copy raw primary bytes to another primary path and queue one write event.pub async fn copy_within_primary(&self, src_path: &str, dst_path: &str) -> Result<bool> {if normalize_prefix_path(src_path) == normalize_prefix_path(dst_path) {return Ok(true);}-- let ctx = current_required_ctx()?;- let inner = &self.inner;- let source_path = src_path.to_string();- let destination_path = dst_path.to_string();-- let source_exists_on_primary = FS_CTX- .scope(ctx.clone(), async {- inner.primary().backend.exists(&source_path).await- })- .await;- if !source_exists_on_primary {+ let Some(primary_raw) = self.inner.primary.raw_backend.clone() else {return Ok(false);- }-- let source_size = FS_CTX- .scope(ctx.clone(), async {- let source_info = inner.primary().backend.stat(&source_path).await?;- if source_info.is_dir {- return Err(Error::IsADirectory(source_path.clone()));- }- Ok::<u64, Error>(source_info.size)- })- .await?;-- if inner- .check_redirect(&destination_path, source_size)- .is_some()- {+ };+ if !primary_raw.exists(src_path).await {return Ok(false);}-- let Some(primary_raw_backend) = inner.primary().raw_backend.clone() else {- return Ok(false);- };-- let primary_destination = destination_path.clone();- Inner::execute_write(- inner,- destination_path,- source_size,- Some(SyncOp::SyncFile { size: source_size }),- Some(BackupWriteOp::Replay(SyncOp::SyncFile {- size: source_size,- })),- move |_ignored_fs| {- let primary_raw_backend = primary_raw_backend.clone();- let primary_source = source_path.clone();- let primary_destination = primary_destination.clone();- let primary_ctx = ctx.clone();- async move {- copy_raw_primary_state(- primary_raw_backend,- &primary_source,- &primary_destination,- &primary_ctx,- )- .await?;- Ok(())- }- },- )- .await?;-+ let ctx = self.resolve_context(src_path)?;+ logical_path(ctx.account_id(), dst_path)?;+ let event = self.pending_event(dst_path, SegmentEventType::Write, None)?;+ copy_raw_primary_state(primary_raw, src_path, dst_path, &ctx).await?;+ self.submit_event(event);Ok(true)}+}- /// Execute one non-redirecting write-like operation through the shared multi-write pipeline.- async fn execute_simple_write<R, P, PFut>(- &self,- path: &str,- size: u64,- sync_op: Option<SyncOp>,- primary_fn: P,- ) -> Result<R>- where- R: Send + 'static,- P: FnOnce(Arc<dyn FileSystem>, String) -> PFut + Send,- PFut: Future<Output = Result<R>> + Send,+/// Convert a wrapper path into one canonical account-owned logical path.+fn logical_path(account_id: &str, path: &str) -> Result<String> {+ let normalized = normalize_prefix_path(path);+ let logical = if normalized.starts_with('/') {+ format!("/local{normalized}")+ } else {+ format!("/local/{normalized}")+ };+ let account_root = format!("/local/{account_id}");+ if logical != account_root+ && !logical+ .strip_prefix(&account_root)+ .is_some_and(|suffix| suffix.starts_with('/')){- let path_owned = path.to_string();- let primary_path = path_owned.clone();- let backup_op = sync_op.clone().map(BackupWriteOp::Replay);- Inner::execute_write(- &self.inner,- path_owned,- size,- sync_op,- backup_op,- move |fs| primary_fn(fs, primary_path),- )+ return Err(Error::invalid_path(format!(+ "path does not belong to account '{account_id}': {logical}"+ )));+ }+ Ok(logical)+}++/// Copy one complete raw file in bounded chunks under the request context.+async fn copy_raw_primary_state(+ primary_raw: Arc<dyn FileSystem>,+ source_path: &str,+ destination_path: &str,+ ctx: &FsContext,+) -> Result<()> {+ FS_CTX+ .scope(ctx.clone(), async {+ let source = primary_raw.stat(source_path).await?;+ if source.is_dir {+ return Err(Error::IsADirectory(source_path.to_string()));+ }+ primary_raw+ .ensure_parent_dirs(destination_path, 0o755)+ .await?;+ if source.size == 0 {+ if primary_raw.exists(destination_path).await {+ return primary_raw.truncate(destination_path, 0).await;+ }+ return primary_raw.create(destination_path).await;+ }+ let mut offset = 0;+ while offset < source.size {+ let size = (source.size - offset).min(COPY_CHUNK_SIZE as u64);+ let chunk = primary_raw.read(source_path, offset, size).await?;+ if chunk.is_empty() {+ return Err(Error::internal(format!(+ "short read while copying raw primary file: {source_path}"+ )));+ }+ let flag = if offset == 0 {+ WriteFlag::Create+ } else {+ WriteFlag::None+ };+ primary_raw+ .write(destination_path, &chunk, offset, flag)+ .await?;+ offset += chunk.len() as u64;+ }+ Ok(())+ }).await+}++/// Build the V2 event specs required to represent one completed rename.+fn rename_event_specs<'a>(+ source: &FileInfo,+ old_path: &'a str,+ new_path: &'a str,+) -> Vec<(SegmentEventType, &'a str, Option<&'a str>)> {+ if source.is_dir {+ vec![(SegmentEventType::MoveTree, old_path, Some(new_path))]+ } else {+ vec![+ (SegmentEventType::Write, new_path, None),+ (SegmentEventType::Remove, old_path, None),+ ]}}#[async_trait]impl FileSystem for MultiWriteWrappedFS {+ /// Create a primary file and queue a write event after success.async fn create(&self, path: &str) -> Result<()> {- if is_hidden_internal_name(file_name(path)) {- return self.inner.primary().backend.create(path).await;- }- self.execute_simple_write(path, 0, Some(SyncOp::Create), |fs, path| async move {- fs.create(&path).await- })- .await+ let event = self.pending_event(path, SegmentEventType::Write, None)?;+ self.inner.primary.backend.create(path).await?;+ self.submit_event(event);+ Ok(())}+ /// Create a primary directory without producing a file-state event.async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {- self.execute_simple_write(- path,- 0,- Some(SyncOp::Mkdir { mode }),- move |fs, path| async move { fs.mkdir(&path, mode).await },- )- .await+ self.inner.primary.backend.mkdir(path, mode).await}+ /// Remove a primary file and queue a remove event after success.async fn remove(&self, path: &str) -> Result<()> {- if is_hidden_internal_name(file_name(path)) {- return self.inner.primary().backend.remove(path).await;- }- self.execute_simple_write(path, 0, Some(SyncOp::Remove), |fs, path| async move {- fs.remove(&path).await- })- .await+ let event = self.pending_event(path, SegmentEventType::Remove, None)?;+ self.inner.primary.backend.remove(path).await?;+ self.submit_event(event);+ Ok(())}+ /// Remove a primary tree and queue a directory removal event after success.async fn remove_all(&self, path: &str) -> Result<()> {- if is_hidden_internal_name(file_name(path)) {- return self.inner.primary().backend.remove_all(path).await;+ let event = self+ .pending_event(path, SegmentEventType::RemoveTree, None)?+ .map(|mut event| {+ if event.path == format!("/local/{}", event.account_id) {+ event.kind = PendingEventKind::DeleteAccount;+ }+ event+ });+ let is_delete_account = event+ .as_ref()+ .is_some_and(|event| matches!(event.kind, PendingEventKind::DeleteAccount));+ match self.inner.primary.backend.remove_all(path).await {+ Ok(()) => {+ self.submit_event(event);+ Ok(())+ }+ Err(error @ Error::NotFound(_)) if is_delete_account => {+ self.submit_event(event);+ Err(error)+ }+ Err(error) => Err(error),}- self.execute_simple_write(path, 0, Some(SyncOp::RemoveAll), |fs, path| async move {- fs.remove_all(&path).await- })- .await}+ /// Read bytes directly from the primary backend.async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {- if is_hidden_internal_name(file_name(path)) {- return self.inner.primary().backend.read(path, offset, size).await;- }- if let Some(fs) = self.inner.resolve_read_backend(path).await {- return fs.read(path, offset, size).await;- }- Err(Error::not_found(path))+ self.inner.primary.backend.read(path, offset, size).await}+ /// Write primary bytes and queue a write event after success.async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {- if is_hidden_internal_name(file_name(path)) {- return self- .inner- .primary()- .backend- .write(path, data, offset, flags)- .await;- }- let inner = &self.inner;- let data_len = data.len() as u64;- let path_owned = path.to_string();- let backup_op = BackupWriteOp::WriteFile {- data: Arc::new(data.to_vec()),- offset,- flags,- };- let d = data.to_vec();- let primary_path = path_owned.clone();- Inner::execute_write_with_redirect(- inner,- path_owned,- data_len,- SyncOp::SyncFile { size: data_len },- backup_op,- move |fs: Arc<dyn FileSystem>| async move {- fs.write(&primary_path, &d, offset, flags).await- },- )- .await+ let event = self.pending_event(path, SegmentEventType::Write, None)?;+ let written = self+ .inner+ .primary+ .backend+ .write(path, data, offset, flags)+ .await?;+ self.submit_event(event);+ Ok(written)}+ /// List one directory directly from the primary backend.async fn read_dir(&self,path: &str,@@ -1418,379 +594,89 @@ impl FileSystem for MultiWriteWrappedFS {sort_by: Option<ListSortBy>,sort_order: Option<SortOrder>,) -> Result<Vec<FileInfo>> {- let inner = &self.inner;- let mut entries = inner- .primary()+ self.inner+ .primary.backend- .read_dir(path, None, None, None, None)- .await?;-- // Filter multi-write internal names.- entries.retain(|e| !MULTIWRITE_INTERNAL_NAMES.contains(&e.name.as_str()));-- if inner.redirects.is_empty() {- return Ok(apply_read_dir_options(- entries, offset, limit, sort_by, sort_order,- ));- }-- // Merge redirect entries so users can see redirected files in listings.- let ctx =- current_required_ctx().or_else(|_| inner.meta_store.ctx_resolver().resolve(path))?;-- if let Ok(redirect_meta) = inner.meta_store.get_redirect_meta(path, &ctx).await {- for (name, redirect_entry) in &redirect_meta.entries {- if !entries.iter().any(|e| &e.name == name) {- let virtual_path = if path == "/" {- format!("/{}", name)- } else {- format!("{}/{}", path.trim_end_matches('/'), name)- };- entries.push(- inner- .redirect_file_info(&virtual_path, name, redirect_entry)- .await,- );- }- }- }+ .read_dir(path, offset, limit, sort_by, sort_order)+ .await+ }- Ok(apply_read_dir_options(- entries, offset, limit, sort_by, sort_order,- ))+ /// List internal entries directly from the primary backend.+ async fn read_internal_dir(&self, path: &str) -> Result<Vec<FileInfo>> {+ self.inner.primary.backend.read_internal_dir(path).await}+ /// Read metadata directly from the primary backend.async fn stat(&self, path: &str) -> Result<FileInfo> {- if is_hidden_internal_name(file_name(path)) {- return self.inner.primary().backend.stat(path).await;- }- if let Some(fs) = self.inner.resolve_read_backend(path).await {- return fs.stat(path).await;- }- Err(Error::not_found(path))+ self.inner.primary.backend.stat(path).await}+ /// Rename a primary path and queue file or tree events after success.async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {- if is_hidden_internal_name(file_name(old_path))- || is_hidden_internal_name(file_name(new_path))- {- return self- .inner- .primary()- .backend- .rename(old_path, new_path)- .await;- }- let ctx = current_required_ctx()?;- let inner = &self.inner;- let old_owned = old_path.to_string();- let new_owned = new_path.to_string();- let source_dir = parent_dir(&old_owned);- let redirect_targets = inner.redirect_targets_for_path(&old_owned, &ctx).await?;- inner.invalidate_read_route(&old_owned).await;- inner.invalidate_read_route(&new_owned).await;-- let primary_has_old = inner.primary().backend.exists(&old_owned).await;- if primary_has_old {- FS_CTX- .scope(ctx.clone(), async {- inner.primary().backend.rename(&old_owned, &new_owned).await- })- .await?;- } else if redirect_targets.is_none() {- return Err(Error::NotFound(old_owned));- }-- inner- .record_rename_meta(&old_owned, &new_owned, &ctx)+ let old_ctx = self.resolve_context(old_path)?;+ let new_ctx = self.resolve_context(new_path)?;+ if old_ctx.account_id() != new_ctx.account_id() {+ return Err(Error::invalid_path(format!(+ "rename paths cross accounts: '{}' -> '{}'",+ old_path, new_path+ )));+ }+ let source = self.inner.primary.backend.stat(old_path).await?;+ let events = rename_event_specs(&source, old_path, new_path)+ .into_iter()+ .map(|(kind, path, destination)| self.pending_event(path, kind, destination))+ .collect::<Result<Vec<_>>>()?;+ self.inner+ .primary+ .backend+ .rename(old_path, new_path).await?;- inner.mark_pending_dir(&source_dir).await;-- let fanout_path = old_owned.clone();- let backup_op = BackupWriteOp::Replay(SyncOp::Rename {- to: new_owned.clone(),- });- if let Some(targets) = redirect_targets {- Inner::fanout_write_to_targets(inner, &fanout_path, &targets, ctx.clone(), backup_op)- .await?;- } else {- Inner::fanout_write(inner, &fanout_path, 0, ctx.clone(), backup_op).await?;+ for event in events {+ self.submit_event(event);}- inner.refresh_pending_dir(&source_dir, &ctx).await?;-Ok(())}+ /// Change primary permissions and queue a write event after success.async fn chmod(&self, path: &str, mode: u32) -> Result<()> {- if is_hidden_internal_name(file_name(path)) {- return self.inner.primary().backend.chmod(path, mode).await;- }- self.execute_simple_write(- path,- 0,- Some(SyncOp::Chmod { mode }),- move |fs, path| async move { fs.chmod(&path, mode).await },- )- .await+ let event = self.pending_event(path, SegmentEventType::Write, None)?;+ self.inner.primary.backend.chmod(path, mode).await?;+ self.submit_event(event);+ Ok(())}+ /// Truncate a primary file and queue a write event after success.async fn truncate(&self, path: &str, size: u64) -> Result<()> {- if is_hidden_internal_name(file_name(path)) {- return self.inner.primary().backend.truncate(path, size).await;- }- self.execute_simple_write(- path,- size,- Some(SyncOp::SyncFile { size }),- move |fs, path| async move { fs.truncate(&path, size).await },- )- .await+ let event = self.pending_event(path, SegmentEventType::Write, None)?;+ self.inner.primary.backend.truncate(path, size).await?;+ self.submit_event(event);+ Ok(())}+ /// Create primary parent directories without producing a file-state event.async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {- let path_owned = path.to_string();- let primary_path = path_owned.clone();- Inner::execute_write(- &self.inner,- path_owned,- 0,- None,- Some(BackupWriteOp::EnsureParentDirs { mode }),- move |fs| async move { fs.ensure_parent_dirs(&primary_path, mode).await },- )- .await+ self.inner+ .primary+ .backend+ .ensure_parent_dirs(path, mode)+ .await}+ /// Search content directly on the primary backend.async fn grep(&self,path: &str,pattern: &str,options: GrepOptions<'_>,) -> Result<GrepResult> {- let inner = &self.inner;- let path_owned = path.to_string();- let pattern_owned = pattern.to_string();- let exclude_owned = options.exclude_path.map(str::to_string);- let options = GrepOptions {- exclude_path: exclude_owned.as_deref(),- ..options- };- let recursive = options.recursive;- let node_limit = options.node_limit;- let level_limit = options.level_limit;-- let mut result = inner- .primary()- .backend- .grep(&path_owned, &pattern_owned, options)- .await?;-- // Filter out multi-write internal metadata files from grep results.- result.matches.retain(|m| {- let file_name = m.file.rsplit('/').next().unwrap_or(&m.file);- !MULTIWRITE_INTERNAL_NAMES.contains(&file_name)- });- result.count = result.matches.len();-- if inner.redirects.is_empty() {- return Ok(result);- }-- // For redirect files, also grep in target backends.- let ctx = current_required_ctx()- .or_else(|_| inner.meta_store.ctx_resolver().resolve(&path_owned))?;-- let search_dir = if inner- .primary()+ self.inner+ .primary.backend- .stat(&path_owned)+ .grep(path, pattern, options).await- .map(|s| s.is_dir)- .unwrap_or(false)- {- path_owned.clone()- } else {- parent_dir(&path_owned)- };-- if recursive && search_dir == path_owned {- let redirect_entries = self- .tree_directory(- &path_owned,- true,- None,- level_limit,- None,- None,- None,- false,- )- .await?;- for entry in redirect_entries {- if node_limit.is_some_and(|limit| result.count >= limit) {- break;- }- if entry.info.is_dir- || entry.extra.get("redirect").and_then(Value::as_bool) != Some(true)- {- continue;- }- if is_excluded_grep_path(&entry.path, exclude_owned.as_deref()) {- continue;- }- let Some(read_backend) = inner.resolve_read_backend(&entry.path).await else {- continue;- };- let rel_path = relative_match_file(&path_owned, &entry.path);- let target_result = match read_backend- .grep(- &entry.path,- &pattern_owned,- GrepOptions {- recursive: false,- node_limit: node_limit.map(|limit| limit.saturating_sub(result.count)),- exclude_path: None,- level_limit: None,- ..options- },- )- .await- {- Ok(found) => found,- Err(_) => continue,- };- for mut m in target_result.matches {- if node_limit.is_some_and(|limit| result.count >= limit) {- break;- }- m.file = rel_path.clone();- result.matches.push(m);- result.count += 1;- }- }- return Ok(result);- }-- if let Ok(redirect_meta) = inner.meta_store.get_redirect_meta(&search_dir, &ctx).await {- for (name, redirect_entry) in &redirect_meta.entries {- for target_name in &redirect_entry.targets {- if let Some(be) = inner.backup_by_name(target_name) {- let redirect_path = if search_dir == "/" {- format!("/{}", name)- } else {- format!("{}/{}", search_dir, name)- };- if let Ok(target_result) = be- .backend- .grep(- &redirect_path,- &pattern_owned,- GrepOptions {- recursive: false,- exclude_path: None,- level_limit: None,- ..options- },- )- .await- {- let rel_path = relative_match_file(&path_owned, &redirect_path);- for mut m in target_result.matches {- if node_limit.is_some_and(|limit| result.count >= limit) {- break;- }- m.file = rel_path.clone();- result.matches.push(m);- result.count += 1;- }- }- }- }- }- }-- Ok(result)- }-- async fn glob_directory(- &self,- path: &str,- pattern: &str,- show_hidden: bool,- page_size: Option<usize>,- level_limit: Option<usize>,- continuation_token: Option<String>,- ) -> Result<GlobPage> {- if self.inner.redirects.is_empty() {- return self- .inner- .primary()- .backend- .glob_directory(- path,- pattern,- show_hidden,- page_size,- level_limit,- continuation_token,- )- .await;- }-- let matcher = PreparedGlob::new(pattern)?;- if matches!(page_size, Some(0)) {- return Err(Error::invalid_operation("page_size must be positive"));- }-- let entries = self- .tree_directory(- path,- show_hidden,- None,- level_limit,- None,- None,- None,- false,- )- .await?;-- let mut matched = Vec::new();- for entry in entries {- if matcher.is_match(&entry.rel_path) {- matched.push(GlobEntry {- path: entry.path,- rel_path: entry.rel_path,- name: entry.info.name,- is_dir: entry.info.is_dir,- });- }- }- matched.sort_by(|left, right| compare_rel_paths(&left.rel_path, &right.rel_path));-- let start = decode_offset_token(- continuation_token.as_deref(),- path,- pattern,- show_hidden,- level_limit,- )?;- if start > matched.len() {- return Err(Error::invalid_operation("continuation token out of range"));- }- let end = page_size- .map(|limit| start.saturating_add(limit))- .unwrap_or(matched.len())- .min(matched.len());- let next_token = (end < matched.len())- .then(|| encode_offset_token(end, path, pattern, show_hidden, level_limit));-- Ok(GlobPage {- entries: matched[start..end].to_vec(),- next_token,- })}+ /// Traverse a tree directly on the primary backend.async fn tree_directory(&self,path: &str,@@ -1802,130 +688,43 @@ impl FileSystem for MultiWriteWrappedFS {sort_order: Option<SortOrder>,directories_only: bool,) -> Result<Vec<TreeEntry>> {- let base = normalize_prefix_path(path);- if sort_by.is_some() {- let mut entries = Vec::new();- let traversal_limit = node_limit.map(|limit| offset.unwrap_or(0).saturating_add(limit));- self.tree_directory_internal(- &base,- &base,+ self.inner+ .primary+ .backend+ .tree_directory(+ path,show_hidden,- traversal_limit,+ node_limit,level_limit,+ offset,sort_by,sort_order,directories_only,- &mut entries,)- .await?;- return Ok(paginate_entries(entries, offset, node_limit));- }+ .await+ }- let mut entries = self- .inner- .primary()+ /// Match paths directly on the primary backend.+ async fn glob_directory(+ &self,+ path: &str,+ pattern: &str,+ show_hidden: bool,+ page_size: Option<usize>,+ level_limit: Option<usize>,+ continuation_token: Option<String>,+ ) -> Result<GlobPage> {+ self.inner+ .primary.backend- .tree_directory(+ .glob_directory(path,+ pattern,show_hidden,- None,+ page_size,level_limit,- None,- None,- None,- directories_only,+ continuation_token,)- .await?;-- entries.retain(|e| {- let name = file_name(&e.path);- !MULTIWRITE_INTERNAL_NAMES.contains(&name) && (!directories_only || e.info.is_dir)- });-- if self.inner.redirects.is_empty() {- return Ok(paginate_entries(entries, offset, node_limit));- }-- let ctx = current_required_ctx()- .or_else(|_| self.inner.meta_store.ctx_resolver().resolve(&base))?;- let mut seen_paths: HashSet<String> = entries.iter().map(|e| e.path.clone()).collect();- let mut dir_paths = Vec::new();- let mut queued_dirs: HashSet<String> = HashSet::new();- if !matches!(level_limit, Some(0)) {- queued_dirs.insert(base.clone());- dir_paths.push(base.clone());- }- for entry in &entries {- if entry.info.is_dir {- let dir = normalize_prefix_path(&entry.path);- let may_emit_redirect_children = match level_limit {- Some(limit) => {- let rel = relative_match_file(&base, &dir);- relative_depth(&rel) < limit- }- None => true,- };- if may_emit_redirect_children && queued_dirs.insert(dir.clone()) {- dir_paths.push(dir);- }- }- }-- let inner = Arc::clone(&self.inner);- let mut redirect_results = stream::iter(dir_paths)- .map(|dir| {- let inner = Arc::clone(&inner);- let ctx = ctx.clone();- async move {- let redirect_meta = inner.meta_store.get_redirect_meta(&dir, &ctx).await.ok();- (dir, redirect_meta)- }- })- .buffered(DEFAULT_TREE_REDIRECT_META_CONCURRENCY);-- while let Some((dir, redirect_meta)) = redirect_results.next().await {- let Some(redirect_meta) = redirect_meta else {- continue;- };- for (name, redirect_entry) in redirect_meta.entries {- let virtual_path = if dir == "/" {- format!("/{}", name)- } else {- format!("{}/{}", dir, name)- };- if seen_paths.contains(&virtual_path) {- continue;- }- let rel_path = if base == "/" {- virtual_path.trim_start_matches('/').to_string()- } else {- virtual_path- .strip_prefix(&base)- .unwrap_or(&virtual_path)- .trim_start_matches('/')- .to_string()- };- if level_limit.is_some_and(|limit| relative_depth(&rel_path) > limit) {- continue;- }- let mut extra = HashMap::new();- extra.insert("redirect".to_string(), Value::Bool(true));- entries.push(TreeEntry {- path: virtual_path.clone(),- rel_path,- info: self- .inner- .redirect_file_info(&virtual_path, &name, &redirect_entry)- .await,- extra,- });- seen_paths.insert(virtual_path);- }- }-- if directories_only {- entries.retain(|entry| entry.info.is_dir);- }- Ok(paginate_entries(entries, offset, node_limit))+ .await}}diff --git a/crates/ragfs/src/core/multibackend_wrapper/retry.rs b/crates/ragfs/src/core/multibackend_wrapper/retry.rsdeleted file mode 100644index c4581bf22..000000000--- a/crates/ragfs/src/core/multibackend_wrapper/retry.rs+++ /dev/null@@ -1,270 +0,0 @@-use std::time::{Duration, Instant};--use rand::Rng;-use tracing::warn;--use super::*;--impl Inner {- /// Mark one directory as pending retry work.- pub(super) async fn mark_pending_dir(&self, dir: &str) {- self.pending_dirs.lock().await.insert(dir.to_string());- }-- /// Remove one directory from the retry set.- async fn clear_pending_dir(&self, dir: &str) {- self.pending_dirs.lock().await.remove(dir);- }-- /// Snapshot the current set of pending retry directories.- pub(crate) async fn pending_dirs_snapshot(&self) -> Vec<String> {- self.pending_dirs.lock().await.iter().cloned().collect()- }-- /// Recompute whether one directory still has pending retry work.- pub(super) async fn refresh_pending_dir(&self, dir: &str, ctx: &FsContext) -> Result<()> {- if self.sync_work_in_dir(dir, ctx).await?.iter().any(|work| {- work.entry.is_primary_committed()- && work.targets.iter().any(|target| {- !work.entry.is_in_sync(target) && !work.entry.is_quarantined(target)- })- }) {- self.mark_pending_dir(dir).await;- } else {- self.clear_pending_dir(dir).await;- }- Ok(())- }-- /// Collect sync work entries for one metadata directory.- pub(crate) async fn sync_work_in_dir(- &self,- dir: &str,- ctx: &FsContext,- ) -> Result<Vec<SyncWorkEntry>> {- let sync_log = self.meta_store.get_sync_log_meta(dir, ctx).await?;- if sync_log.entries.is_empty() {- return Ok(Vec::new());- }-- let redirect_meta = self.meta_store.get_redirect_meta(dir, ctx).await?;- Ok(sync_log- .entries- .into_iter()- .map(|(file_name, entry)| {- let file_path = if dir == "/" {- format!("/{}", file_name)- } else {- format!("{}/{}", dir, file_name)- };- let targets =- self.target_backend_names(&redirect_meta, &file_name, &file_path, &entry);- SyncWorkEntry {- file_path,- entry,- targets,- }- })- .collect())- }-- /// Record a started background task.- pub(super) fn background_task_started(&self) {- self.background_tasks.fetch_add(1, Ordering::SeqCst);- }-- /// Record a finished background task and wake idle waiters if needed.- pub(super) fn background_task_finished(&self) {- if self.background_tasks.fetch_sub(1, Ordering::SeqCst) == 1 {- self.idle_notify.notify_waiters();- }- }-- /// Wait for all background tasks to drain.- pub(super) async fn wait_idle(&self, timeout: Duration) -> Result<()> {- let deadline = Instant::now() + timeout;- loop {- if self.background_tasks.load(Ordering::SeqCst) == 0 {- return Ok(());- }- let now = Instant::now();- if now >= deadline {- return Err(Error::timeout(- "timed out while waiting multi-write background tasks to drain",- ));- }- let wait = deadline.saturating_duration_since(now);- tokio::time::timeout(wait, self.idle_notify.notified())- .await- .map_err(|_| {- Error::timeout("timed out while waiting multi-write background tasks to drain")- })?;- }- }-- /// Update the acked_seq for a backup in the sync log.- pub(super) async fn update_backup_acked_seq(- &self,- path: &str,- backup_name: &str,- ctx: &FsContext,- ) -> Result<()> {- self.update_backend_state(path, backup_name, ctx, |state, latest_seq| {- state.mark_acked(latest_seq);- })- .await?;- self.refresh_pending_dir(&parent_dir(path), ctx).await- }-- /// Record a replay failure and quarantine the target after repeated failures.- pub(crate) async fn record_backup_retry_failure(- &self,- path: &str,- backup_name: &str,- ctx: &FsContext,- ) -> Result<()> {- self.update_backend_state(path, backup_name, ctx, |state, _latest_seq| {- state.mark_retry_failed(self.quarantine_after_failures);- })- .await?;- self.refresh_pending_dir(&parent_dir(path), ctx).await- }-- /// Update per-backend sync state for a file entry.- async fn update_backend_state<F>(- &self,- path: &str,- backup_name: &str,- ctx: &FsContext,- update: F,- ) -> Result<()>- where- F: FnOnce(&mut BackendSyncState, u64) + Send,- {- let dir = parent_dir(path);- let backup_name = backup_name.to_string();- let name = file_name(path).to_string();- self.meta_store- .update_dir_meta(&dir, ctx, move |_redirect, sync_log| {- if let Some(entry) = sync_log.entries.get_mut(&name) {- let state = entry.backends.entry(backup_name).or_default();- update(state, entry.latest_seq);- }- Ok(())- })- .await- }-- /// Replay a single operation on a lagging backup.- pub(crate) async fn replay_operation(- &self,- file_path: &str,- backup_name: &str,- ctx: &FsContext,- ) -> Result<()> {- let queue_key = Self::backup_queue_key(file_path, backup_name);- self.path_queues- .with_path_lock(&queue_key, || async {- let dir = parent_dir(file_path);- let name = file_name(file_path).to_string();- let sync_log = self.meta_store.get_sync_log_meta(&dir, ctx).await?;- let entry = sync_log- .entries- .get(&name)- .ok_or_else(|| Error::not_found(file_path))?- .clone();- let backup = match self.backup_by_name(backup_name) {- Some(b) => b,- None => {- return Err(Error::internal(format!(- "backup '{}' not found",- backup_name- )))- }- };-- entry- .op- .replay(- self.primary().backend.clone(),- backup.backend.clone(),- file_path,- ctx,- )- .await?;-- self.update_backup_acked_seq(file_path, backup_name, ctx)- .await?;-- Ok(())- })- .await- }-- /// Retry all pending backup work for one directory and keep metadata errors visible.- pub(crate) async fn retry_pending_dir(&self, dir: &str) -> Result<()> {- let ctx = self.meta_store.ctx_resolver().resolve(dir)?;-- for work in self.sync_work_in_dir(dir, &ctx).await? {- if !work.entry.is_primary_committed() {- continue;- }-- for backup_name in &work.targets {- if work.entry.is_in_sync(backup_name) || work.entry.is_quarantined(backup_name) {- continue;- }-- let mut success = false;- for attempt in 0..self.max_retry_per_round {- if self.retry_cancelled.load(Ordering::SeqCst) {- return Ok(());- }- if self- .replay_operation(&work.file_path, backup_name, &ctx)- .await- .is_ok()- {- success = true;- break;- }- let base_ms = self.retry_backoff_base_ms.saturating_mul(1u64 << attempt);- let jitter_ms = rand::thread_rng().gen_range(0..=50);- tokio::time::sleep(Duration::from_millis(base_ms + jitter_ms)).await;- }-- if !success {- self.record_backup_retry_failure(&work.file_path, backup_name, &ctx)- .await?;- }- }- }-- self.refresh_pending_dir(dir, &ctx).await- }-- /// Background retry loop: periodically scans sync_log for lagging backups and replays.- pub(super) async fn retry_loop(inner: Arc<Inner>) {- loop {- if inner.retry_cancelled.load(Ordering::SeqCst) {- break;- }-- tokio::select! {- _ = tokio::time::sleep(inner.retry_interval) => {}- _ = inner.retry_shutdown.notified() => break,- }- let dirs = inner.pending_dirs_snapshot().await;- for dir in dirs {- if let Err(err) = inner.retry_pending_dir(&dir).await {- warn!(- dir = %dir,- error = %err,- "multi-write retry skipped one pending directory due to metadata or replay error"- );- }- }- }- inner.background_task_finished();- }-}diff --git a/crates/ragfs/src/core/multibackend_wrapper/routing.rs b/crates/ragfs/src/core/multibackend_wrapper/routing.rsdeleted file mode 100644index 1dff3f491..000000000--- a/crates/ragfs/src/core/multibackend_wrapper/routing.rs+++ /dev/null@@ -1,125 +0,0 @@-use std::sync::Arc;--use serde_json::{json, Value};--use super::*;-use crate::multibackend::meta::{current_required_ctx, file_name, parent_dir};--impl Inner {- /// Keep write-side call sites explicit even though read routing is uncached.- pub(super) async fn invalidate_read_route(&self, _path: &str) {}-- /// Record read-route counters in one place so hot paths stay explicit.- fn record_read_route(&self, source: ReadRouteSource) {- match source {- ReadRouteSource::Backup => {- self.read_backup_hits.fetch_add(1, Ordering::Relaxed);- }- ReadRouteSource::Primary => {- self.read_primary_hits.fetch_add(1, Ordering::Relaxed);- }- ReadRouteSource::Redirect => {- self.read_redirect_hits.fetch_add(1, Ordering::Relaxed);- }- ReadRouteSource::Miss => {- self.read_misses.fetch_add(1, Ordering::Relaxed);- }- }- }-- /// Export read-route metrics for operational introspection.- pub(crate) fn read_route_metrics(&self) -> Value {- json!({- "cache_hits": 0,- "backup_hits": self.read_backup_hits.load(Ordering::Relaxed),- "primary_hits": self.read_primary_hits.load(Ordering::Relaxed),- "redirect_hits": self.read_redirect_hits.load(Ordering::Relaxed),- "misses": self.read_misses.load(Ordering::Relaxed),- })- }-- /// Stat the first reachable redirect target and return user-visible metadata.- pub(super) async fn redirect_file_info(- &self,- path: &str,- name: &str,- redirect_entry: &RedirectEntry,- ) -> FileInfo {- for target_name in &redirect_entry.targets {- if let Some(be) = self.backup_by_name(target_name) {- if let Ok(mut info) = be.backend.stat(path).await {- info.name = name.to_string();- return info;- }- }- }- FileInfo::new_file(name.to_string(), 0, 0o644)- }-- /// Resolve the read backend for a path using the fallback chain.- pub(super) async fn resolve_read_backend(&self, path: &str) -> Option<Arc<dyn FileSystem>> {- let normalized = normalize_prefix_path(path);- let read_backups = self.read_backups_sorted();- let backup_exists = futures::future::join_all(read_backups.iter().map(|backup| async {- (- backup.name.clone(),- backup.backend.clone(),- backup.backend.exists(&normalized).await,- )- }))- .await;- for (_name, backend, exists) in backup_exists {- if exists {- self.record_read_route(ReadRouteSource::Backup);- return Some(backend);- }- }-- if self.primary().backend.exists(&normalized).await {- self.record_read_route(ReadRouteSource::Primary);- return Some(self.primary().backend.clone());- }-- if self.redirects.is_empty() {- self.record_read_route(ReadRouteSource::Miss);- return None;- }-- let dir = parent_dir(&normalized);- let name = file_name(&normalized).to_string();- let ctx = current_required_ctx()- .or_else(|_| self.meta_store.ctx_resolver().resolve(&dir))- .ok()?;- if let Ok(redirect_meta) = self.meta_store.get_redirect_meta(&dir, &ctx).await {- if let Some(entry) = redirect_meta.entries.get(&name) {- let redirect_targets: Vec<(String, Arc<dyn FileSystem>)> = entry- .targets- .iter()- .filter_map(|target_name| {- self.backup_by_name(target_name)- .map(|be| (be.name.clone(), be.backend.clone()))- })- .collect();- let redirect_exists = futures::future::join_all(redirect_targets.iter().map(- |(target_name, backend)| async {- (- target_name.clone(),- backend.clone(),- backend.exists(&normalized).await,- )- },- ))- .await;- for (_target_name, backend, exists) in redirect_exists {- if exists {- self.record_read_route(ReadRouteSource::Redirect);- return Some(backend);- }- }- }- }-- self.record_read_route(ReadRouteSource::Miss);- None- }-}diff --git a/crates/ragfs/src/core/multibackend_wrapper/tests.rs b/crates/ragfs/src/core/multibackend_wrapper/tests.rsdeleted file mode 100644index 1a6752d88..000000000--- a/crates/ragfs/src/core/multibackend_wrapper/tests.rs+++ /dev/null@@ -1,939 +0,0 @@-use std::sync::Arc;--use serde_json::Value;--use super::*;-use crate::core::context::FsContextInner;-use crate::plugins::memfs::MemFileSystem;--/// Create the shared test context used by multi-write tests.-fn test_ctx() -> FsContext {- Arc::new(FsContextInner::new("acct".to_string()))-}--/// Create a sync multi-write filesystem with one memfs backup.-fn test_multiwrite_fs(redirects: Vec<RedirectPolicy>) -> MultiWriteWrappedFS {- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let builder = MultiWriteWrappedFS::builder(primary.clone())- .with_primary_raw_backend(primary)- .with_backups(vec![BackendEntry {- name: "backup1".to_string(),- role: BackendRole::Backup,- backend: backup,- raw_backend: None,- operations: Vec::new(),- excludes: Vec::new(),- }])- .sync_mode(SyncMode::Sync {- ack_count: 1,- timeout_ms: 0,- });-- if redirects.is_empty() {- builder- } else {- builder.with_redirects(redirects)- }- .build()- .unwrap()-}--#[test]-fn test_file_policy_over_size() {- let policy = RedirectPolicy::FileOverSizePolicy {- max_size_mb: 1,- target: Some(vec!["backup1".to_string()]),- };- assert!(policy.matches("/a/big.bin", 2 * 1024 * 1024));- assert!(!policy.matches("/a/small.txt", 512));-}--#[test]-fn test_file_policy_extension() {- let policy = RedirectPolicy::FileExtensionPolicy {- extensions: vec!["(pdf|ppt)".to_string()],- target: Some(vec!["backup1".to_string()]),- };- assert!(policy.matches("/a/doc.pdf", 0));- assert!(policy.matches("/a/slides.ppt", 0));- assert!(!policy.matches("/a/text.txt", 0));-}--#[tokio::test]-async fn test_read_dir_redirect_entries_use_target_stat() {- let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {- extensions: vec!["\\.pdf$".to_string()],- target: Some(vec!["backup1".to_string()]),- }]);- let ctx = test_ctx();-- FS_CTX- .scope(ctx, async {- fs.ensure_parent_dirs("/local/acct/docs/report.pdf", 0o755)- .await?;- fs.write(- "/local/acct/docs/report.pdf",- b"pdf body",- 0,- WriteFlag::Create,- )- .await?;-- let entries = fs- .read_dir("/local/acct/docs", None, None, None, None)- .await?;- let report = entries- .iter()- .find(|entry| entry.name == "report.pdf")- .expect("redirected file should be visible in read_dir");- assert_eq!(report.size, 8);- assert_eq!(report.mode, 0o644);- Ok::<(), Error>(())- })- .await- .unwrap();-- let no_redirect_fs = test_multiwrite_fs(Vec::new());- let no_redirect_ctx = test_ctx();-- FS_CTX- .scope(no_redirect_ctx.clone(), async {- no_redirect_fs- .ensure_parent_dirs("/local/acct/docs/real.txt", 0o755)- .await?;- no_redirect_fs- .write(- "/local/acct/docs/real.txt",- b"real body",- 0,- WriteFlag::Create,- )- .await?;- no_redirect_fs- .inner- .meta_store- .update_dir_meta(- "/local/acct/docs",- &no_redirect_ctx,- |redirect, _sync_log| {- redirect.entries.insert(- "ghost.pdf".to_string(),- RedirectEntry {- targets: vec!["backup1".to_string()],- },- );- Ok(())- },- )- .await?;- let backup = no_redirect_fs- .inner- .backup_by_name("backup1")- .expect("backup exists")- .backend- .clone();- backup- .ensure_parent_dirs("/local/acct/docs/ghost.pdf", 0o755)- .await?;- backup- .write(- "/local/acct/docs/ghost.pdf",- b"GHOST_MARKER\n",- 0,- WriteFlag::Create,- )- .await?;-- let entries = no_redirect_fs- .read_dir("/local/acct/docs", None, None, None, None)- .await?;- let names: Vec<&str> = entries.iter().map(|entry| entry.name.as_str()).collect();- assert_eq!(names, vec!["real.txt"]);-- let tree = no_redirect_fs- .tree_directory(- "/local/acct",- false,- None,- Some(3),- None,- None,- None,- false,- )- .await?;- let paths: Vec<&str> = tree.iter().map(|entry| entry.path.as_str()).collect();- assert!(!paths.contains(&"/local/acct/docs/ghost.pdf"));-- let grep = no_redirect_fs- .grep(- "/local/acct/docs",- "GHOST_MARKER",- GrepOptions {- level_limit: Some(1),- ..Default::default()- },- )- .await?;- assert_eq!(grep.count, 0);-- let err = no_redirect_fs- .rename("/local/acct/docs/ghost.pdf", "/local/acct/docs/renamed.pdf")- .await- .expect_err("stale redirect metadata is ignored when redirects are disabled");- assert!(matches!(err, Error::NotFound(_)));- assert!(backup.exists("/local/acct/docs/ghost.pdf").await);- assert!(!backup.exists("/local/acct/docs/renamed.pdf").await);-- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_recursive_grep_finds_nested_redirected_files() {- let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {- extensions: vec!["\\.md$".to_string()],- target: Some(vec!["backup1".to_string()]),- }]);- let ctx = test_ctx();-- FS_CTX- .scope(ctx, async {- fs.ensure_parent_dirs("/local/acct/resources/doc-1/page.md", 0o755)- .await?;- fs.write(- "/local/acct/resources/doc-1/page.md",- b"MULTIWRITE_TEST_MARKER\n",- 0,- WriteFlag::Create,- )- .await?;-- let shallow = fs- .tree_directory(- "/local/acct/resources",- false,- None,- Some(1),- None,- None,- None,- false,- )- .await?;- let shallow_paths: Vec<&str> =- shallow.iter().map(|entry| entry.path.as_str()).collect();- assert_eq!(shallow_paths, vec!["/local/acct/resources/doc-1"]);-- let root_only = fs- .tree_directory(- "/local/acct/resources",- false,- None,- Some(0),- None,- None,- None,- false,- )- .await?;- assert!(root_only.is_empty());-- let deep = fs- .tree_directory(- "/local/acct/resources",- false,- None,- Some(2),- None,- None,- None,- false,- )- .await?;- let deep_paths: Vec<&str> = deep.iter().map(|entry| entry.path.as_str()).collect();- assert_eq!(- deep_paths,- vec![- "/local/acct/resources/doc-1",- "/local/acct/resources/doc-1/page.md",- ]- );-- let capped = fs- .tree_directory(- "/local/acct/resources",- false,- Some(1),- Some(2),- None,- None,- None,- false,- )- .await?;- let capped_paths: Vec<&str> = capped.iter().map(|entry| entry.path.as_str()).collect();- assert_eq!(capped_paths, vec!["/local/acct/resources/doc-1"]);-- let result = fs- .grep(- "/local/acct/resources",- "MULTIWRITE_TEST_MARKER",- GrepOptions {- recursive: true,- level_limit: Some(5),- ..Default::default()- },- )- .await?;- assert_eq!(result.count, 1);- assert_eq!(result.matches.len(), 1);- assert_eq!(result.matches[0].file, "doc-1/page.md");- assert_eq!(result.matches[0].line, 1);- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_redirect_write_marks_first_target_acked_before_returning() {- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup1: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup2: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup1_handle = backup1.clone();- let fs = MultiWriteWrappedFS::builder(primary.clone())- .with_primary_raw_backend(primary)- .with_backups(vec![- BackendEntry {- name: "backup1".to_string(),- role: BackendRole::Backup,- backend: backup1,- raw_backend: None,- operations: Vec::new(),- excludes: Vec::new(),- },- BackendEntry {- name: "backup2".to_string(),- role: BackendRole::Backup,- backend: backup2,- raw_backend: None,- operations: Vec::new(),- excludes: Vec::new(),- },- ])- .with_redirects(vec![RedirectPolicy::FileExtensionPolicy {- extensions: vec!["\\.pdf$".to_string()],- target: Some(vec!["backup1".to_string(), "backup2".to_string()]),- }])- .sync_mode(SyncMode::Async)- .build()- .unwrap();- let ctx = test_ctx();-- FS_CTX- .scope(ctx.clone(), async {- fs.ensure_parent_dirs("/local/acct/docs/report.pdf", 0o755)- .await?;- fs.write(- "/local/acct/docs/report.pdf",- b"pdf body",- 0,- WriteFlag::Create,- )- .await?;-- let redirect_meta = fs- .inner- .meta_store- .get_redirect_meta("/local/acct/docs", &ctx)- .await?;- let entry = redirect_meta- .entries- .get("report.pdf")- .expect("redirect metadata should only appear after one target succeeds");- assert_eq!(- entry.targets,- vec!["backup1".to_string(), "backup2".to_string()]- );-- let sync_log = fs- .inner- .meta_store- .get_sync_log_meta("/local/acct/docs", &ctx)- .await?;- let log_entry = sync_log- .entries- .get("report.pdf")- .expect("redirected file should be tracked in sync log");- assert!(log_entry.is_primary_committed());- assert!(- log_entry.is_in_sync("backup1"),- "the first redirect target must be acknowledged before returning"- );- assert!(- backup1_handle.exists("/local/acct/docs/report.pdf").await,- "the first redirect target must already contain the file"- );- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_redirect_write_does_not_publish_metadata_when_first_target_fails() {- let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {- extensions: vec!["\\.pdf$".to_string()],- target: Some(vec!["missing-backup".to_string()]),- }]);- let ctx = test_ctx();-- FS_CTX- .scope(ctx.clone(), async {- let err = fs- .write(- "/local/acct/docs/fail.pdf",- b"content",- 0,- WriteFlag::Create,- )- .await- .expect_err("redirect write should fail when the first target is not durable");-- assert!(- err.to_string().contains("resolved no writable targets"),- "unexpected error: {err}"- );-- let redirect_meta = fs- .inner- .meta_store- .get_redirect_meta("/local/acct/docs", &ctx)- .await?;- assert!(- !redirect_meta.entries.contains_key("fail.pdf"),- "redirect metadata must not be published before the first target succeeds"- );-- let sync_log = fs- .inner- .meta_store- .get_sync_log_meta("/local/acct/docs", &ctx)- .await?;- assert!(- !sync_log.entries.contains_key("fail.pdf"),- "sync log must not record a committed redirect write before durability"- );- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_cross_dir_rename_does_not_create_target_write_log() {- let fs = test_multiwrite_fs(Vec::new());- let ctx = test_ctx();-- FS_CTX- .scope(ctx.clone(), async {- fs.ensure_parent_dirs("/local/acct/src/file.txt", 0o755)- .await?;- fs.ensure_parent_dirs("/local/acct/dst/file.txt", 0o755)- .await?;- fs.write(- "/local/acct/src/file.txt",- b"rename me",- 0,- WriteFlag::Create,- )- .await?;- fs.rename("/local/acct/src/file.txt", "/local/acct/dst/file.txt")- .await?;-- let target_log = fs- .inner- .meta_store- .get_sync_log_meta("/local/acct/dst", &ctx)- .await?;- assert!(- !target_log.entries.contains_key("file.txt"),- "target directory must not record a fake write for cross-dir rename"- );- let source_log = fs- .inner- .meta_store- .get_sync_log_meta("/local/acct/src", &ctx)- .await?;- let rename_entry = source_log- .entries- .get("file.txt")- .expect("source directory should record rename command");- assert!(matches!(- &rename_entry.op,- SyncOp::Rename { to } if to == "/local/acct/dst/file.txt"- ));- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_rename_redirected_file_renames_original_redirect_target() {- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup_handle = backup.clone();- let fs = MultiWriteWrappedFS::builder(primary.clone())- .with_primary_raw_backend(primary)- .with_backups(vec![BackendEntry {- name: "backup1".to_string(),- role: BackendRole::Backup,- backend: backup,- raw_backend: None,- operations: vec![OperationItemConfig {- operation: "read".to_string(),- priority: 1,- }],- excludes: Vec::new(),- }])- .with_redirects(vec![RedirectPolicy::FileExtensionPolicy {- extensions: vec!["\\.pdf$".to_string()],- target: Some(vec!["backup1".to_string()]),- }])- .sync_mode(SyncMode::Sync {- ack_count: 1,- timeout_ms: 0,- })- .build()- .unwrap();- let ctx = test_ctx();-- FS_CTX- .scope(ctx, async {- fs.ensure_parent_dirs("/local/acct/src/a.pdf", 0o755)- .await?;- fs.ensure_parent_dirs("/local/acct/dst/b.pdf", 0o755)- .await?;- fs.write(- "/local/acct/src/a.pdf",- b"pdf payload",- 0,- WriteFlag::Create,- )- .await?;- assert!(backup_handle.exists("/local/acct/src/a.pdf").await);-- fs.rename("/local/acct/src/a.pdf", "/local/acct/dst/b.pdf")- .await?;-- assert!(- backup_handle.exists("/local/acct/dst/b.pdf").await,- "redirect target backend should receive rename fanout"- );- assert!(- !backup_handle.exists("/local/acct/src/a.pdf").await,- "redirect target backend should not retain the old path"- );- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_copy_within_primary_preserves_primary_bytes_and_fanouts_backups() {- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup_handle = backup.clone();- let fs = MultiWriteWrappedFS::builder(primary.clone())- .with_primary_raw_backend(primary)- .with_backups(vec![BackendEntry {- name: "backup1".to_string(),- role: BackendRole::Backup,- backend: backup,- raw_backend: None,- operations: Vec::new(),- excludes: Vec::new(),- }])- .sync_mode(SyncMode::Sync {- ack_count: 1,- timeout_ms: 0,- })- .build()- .unwrap();- let ctx = test_ctx();-- FS_CTX- .scope(ctx.clone(), async {- fs.ensure_parent_dirs("/local/acct/docs/src.bin", 0o755)- .await?;- fs.write("/local/acct/docs/src.bin", b"payload", 0, WriteFlag::Create)- .await?;-- let copied = fs- .copy_within_primary("/local/acct/docs/src.bin", "/local/acct/docs/dst.bin")- .await?;- assert!(- copied,- "copy_within_primary should use the primary fast-path"- );-- assert_eq!(- fs.inner- .primary()- .backend- .read("/local/acct/docs/dst.bin", 0, 0)- .await?,- b"payload"- );- assert_eq!(- backup_handle.read("/local/acct/docs/dst.bin", 0, 0).await?,- b"payload"- );-- let sync_log = fs- .inner- .meta_store- .get_sync_log_meta("/local/acct/docs", &ctx)- .await?;- let entry = sync_log- .entries- .get("dst.bin")- .expect("copied file should be tracked for backup fanout");- assert!(entry.is_primary_committed());- assert!(matches!(entry.op, SyncOp::SyncFile { size } if size == 7));- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_sync_log_write_entry_does_not_embed_large_payload() {- let fs = test_multiwrite_fs(Vec::new());- let ctx = test_ctx();- let payload = vec![b'x'; 256 * 1024];-- FS_CTX- .scope(ctx.clone(), async {- fs.ensure_parent_dirs("/local/acct/docs/large.bin", 0o755)- .await?;- fs.write("/local/acct/docs/large.bin", &payload, 0, WriteFlag::Create)- .await?;-- let sync_log = fs- .inner- .meta_store- .get_sync_log_meta("/local/acct/docs", &ctx)- .await?;- let encoded = serde_json::to_vec(&sync_log)?;- assert!(- encoded.len() < 16 * 1024,- "sync log should stay metadata-sized, got {} bytes",- encoded.len()- );- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_sync_log_truncate_entry_does_not_embed_snapshot() {- let fs = test_multiwrite_fs(Vec::new());- let ctx = test_ctx();- let payload = vec![b'y'; 256 * 1024];-- FS_CTX- .scope(ctx.clone(), async {- fs.ensure_parent_dirs("/local/acct/docs/large.bin", 0o755)- .await?;- fs.write("/local/acct/docs/large.bin", &payload, 0, WriteFlag::Create)- .await?;- fs.truncate("/local/acct/docs/large.bin", 128).await?;-- let sync_log = fs- .inner- .meta_store- .get_sync_log_meta("/local/acct/docs", &ctx)- .await?;- let encoded = serde_json::to_vec(&sync_log)?;- assert!(- encoded.len() < 16 * 1024,- "truncate sync log should stay metadata-sized, got {} bytes",- encoded.len()- );- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_fanout_sync_returns_structured_quorum_error() {- let fs = test_multiwrite_fs(Vec::new());- let ctx = test_ctx();- let targets = fs.inner.write_targets("/local/acct/docs/fail.txt", 0);-- let err = Inner::fanout_sync(- &fs.inner,- "/local/acct/docs/fail.txt",- &targets,- &ctx,- BackupWriteOp::Replay(SyncOp::SyncFile { size: 1 }),- )- .await- .expect_err("sync fanout should surface structured quorum failure");-- match err {- Error::SyncWriteQuorum {- succeeded,- required,- attempted,- failures,- } => {- assert_eq!(succeeded, 0);- assert_eq!(required, 1);- assert_eq!(attempted, 1);- assert_eq!(failures.len(), 1);- assert_eq!(failures[0].backend, "backup1");- assert_eq!(failures[0].kind, "not_found");- assert_eq!(failures[0].message, "not found: /local/acct/docs/fail.txt");- }- other => panic!("unexpected error: {other}"),- }-}--#[tokio::test]-async fn test_sync_write_errors_when_required_ack_has_no_targets() {- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let fs = MultiWriteWrappedFS::builder(primary.clone())- .with_primary_raw_backend(primary)- .with_backups(vec![BackendEntry {- name: "backup1".to_string(),- role: BackendRole::Backup,- backend: backup,- raw_backend: None,- operations: Vec::new(),- excludes: vec![RedirectPolicy::FileExtensionPolicy {- extensions: vec!["\\.txt$".to_string()],- target: None,- }],- }])- .sync_mode(SyncMode::Sync {- ack_count: 1,- timeout_ms: 0,- })- .build()- .unwrap();- let ctx = test_ctx();-- let err = FS_CTX- .scope(ctx, async {- fs.ensure_parent_dirs("/local/acct/docs/no-target.txt", 0o755)- .await?;- fs.write(- "/local/acct/docs/no-target.txt",- b"payload",- 0,- WriteFlag::Create,- )- .await- })- .await- .expect_err("sync mode must not silently pass when all backups are excluded");-- match err {- Error::SyncWriteQuorum {- succeeded,- required,- attempted,- failures,- } => {- assert_eq!(succeeded, 0);- assert_eq!(required, 1);- assert_eq!(attempted, 0);- assert!(failures.is_empty());- }- other => panic!("unexpected error: {other}"),- }-}--#[tokio::test]-async fn test_read_route_metrics_capture_backup_fallbacks() {- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let backup_handle = backup.clone();- let fs = MultiWriteWrappedFS::builder(primary)- .with_backups(vec![BackendEntry {- name: "backup1".to_string(),- role: BackendRole::Backup,- backend: backup,- raw_backend: None,- operations: vec![OperationItemConfig {- operation: "read".to_string(),- priority: 1,- }],- excludes: Vec::new(),- }])- .build()- .unwrap();- let ctx = test_ctx();-- FS_CTX- .scope(ctx, async {- backup_handle- .ensure_parent_dirs("/local/acct/hot/cache.txt", 0o755)- .await?;- backup_handle- .write("/local/acct/hot/cache.txt", b"hot", 0, WriteFlag::Create)- .await?;-- assert_eq!(fs.read("/local/acct/hot/cache.txt", 0, 0).await?, b"hot");- assert_eq!(fs.read("/local/acct/hot/cache.txt", 0, 0).await?, b"hot");-- let metrics = fs.inner.read_route_metrics();- assert_eq!(metrics.get("backup_hits").and_then(Value::as_u64), Some(2));- assert_eq!(metrics.get("cache_hits").and_then(Value::as_u64), Some(0));- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_shutdown_drains_background_retry_loop() {- let fs = test_multiwrite_fs(Vec::new());- fs.shutdown().await.unwrap();-}--#[tokio::test]-async fn test_system_sync_status_reports_corrupt_redirect_meta() {- let fs = test_multiwrite_fs(Vec::new());- let ctx = test_ctx();-- FS_CTX- .scope(ctx.clone(), async {- fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)- .await?;- fs.write("/local/acct/docs/a.txt", b"hello", 0, WriteFlag::Create)- .await?;- fs.inner- .primary()- .backend- .write(- "/local/acct/docs/.redirect.json",- b"{not-json",- 0,- WriteFlag::Create,- )- .await?;-- let err = fs- .system_sync_status("/local/acct/docs/a.txt")- .await- .expect_err("corrupt redirect metadata must be reported");- assert!(- err.to_string().contains("redirect")- || err.to_string().contains("json")- || err.to_string().contains("serialization"),- "unexpected error: {err}"- );- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_retry_pending_dir_reports_corrupt_redirect_meta() {- let fs = test_multiwrite_fs(Vec::new());- let ctx = test_ctx();-- FS_CTX- .scope(ctx.clone(), async {- fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)- .await?;- fs.write("/local/acct/docs/a.txt", b"hello", 0, WriteFlag::Create)- .await?;- fs.inner- .meta_store- .update_dir_meta("/local/acct/docs", &ctx, |_redirect, sync_log| {- sync_log- .entries- .insert("a.txt".to_string(), SyncLogEntry::new(1, SyncOp::Create));- Ok(())- })- .await?;- fs.inner- .primary()- .backend- .write(- "/local/acct/docs/.redirect.json",- b"{not-json",- 0,- WriteFlag::Create,- )- .await?;-- let err = fs- .inner- .retry_pending_dir("/local/acct/docs")- .await- .expect_err("corrupt redirect metadata must be reported to retry logic");- assert!(- err.to_string().contains("redirect")- || err.to_string().contains("json")- || err.to_string().contains("serialization"),- "unexpected error: {err}"- );- Ok::<(), Error>(())- })- .await- .unwrap();-}--#[tokio::test]-async fn test_uncommitted_sync_log_entry_is_visible_but_not_retried() {- let fs = test_multiwrite_fs(Vec::new());- let ctx = test_ctx();-- FS_CTX- .scope(ctx.clone(), async {- fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)- .await?;- fs.inner- .meta_store- .update_dir_meta("/local/acct/docs", &ctx, |_redirect, sync_log| {- sync_log- .entries- .insert("a.txt".to_string(), SyncLogEntry::new(1, SyncOp::Create));- Ok(())- })- .await?;-- let status = fs.system_sync_status("/local/acct/docs").await?;- assert_eq!(status["entry_count"], 1);- assert_eq!(status["pending_target_count"], 0);- assert_eq!(status["entries"][0]["primary_committed"], false);- assert_eq!(- status["entries"][0]["targets"][0]["primary_committed"],- false- );-- let retry = fs.system_sync_retry("/local/acct/docs").await?;- assert_eq!(retry["retried"], 0);- assert_eq!(retry["failed"], 0);- assert_eq!(retry["skipped"], 1);- assert_eq!(retry["results"][0]["status"], "awaiting_primary_commit");- Ok::<(), Error>(())- })- .await- .unwrap();-}diff --git a/crates/ragfs/src/core/types.rs b/crates/ragfs/src/core/types.rsindex fff95a76a..9038d4abf 100644--- a/crates/ragfs/src/core/types.rs+++ b/crates/ragfs/src/core/types.rs@@ -397,10 +397,6 @@ pub struct PluginConfig {/// Primary encryption enabled (follows global, not independently configurable)#[serde(default)]pub primary_encryption_enabled: bool,-- /// Primary redirect policies- #[serde(default)]- pub primary_redirects: Vec<RedirectPolicy>,}impl Default for PluginConfig {@@ -413,7 +409,6 @@ impl Default for PluginConfig {backups: None,server_encryption_enabled: false,primary_encryption_enabled: false,- primary_redirects: Vec::new(),}}}@@ -444,8 +439,6 @@ impl PluginConfig {Self::take_bool_with_default("server_encryption_enabled", &mut params, false)?;let primary_encryption_enabled =Self::take_bool_with_default("primary_encryption_enabled", &mut params, false)?;- let primary_redirects =- Self::take_json_with_default("primary_redirects", &mut params, Vec::new())?;Ok(Self {name: name.into(),@@ -454,7 +447,6 @@ impl PluginConfig {backups,server_encryption_enabled,primary_encryption_enabled,- primary_redirects,})}@@ -496,15 +488,6 @@ impl PluginConfig {}}- /// Remove one optional JSON config value and fall back to the provided default.- fn take_json_with_default<T: DeserializeOwned>(- field_name: &str,- params: &mut HashMap<String, ConfigValue>,- default: T,- ) -> crate::core::Result<T> {- Ok(Self::take_optional_json(field_name, params)?.unwrap_or(default))- }-/// Return a human-readable config value kind for error messages.fn config_value_kind(value: &ConfigValue) -> &'static str {match value {@@ -576,32 +559,41 @@ impl ConfigValue {/// Multi-write backends container configuration#[derive(Debug, Clone, Serialize, Deserialize)]pub struct BackendsConfig {- /// Sync type: "sync" or "async", default async- #[serde(default = "default_sync_type")]- pub sync_type: String,- /// Minimum backup ack count for sync mode- pub write_ack_count: Option<usize>,- /// Timeout for waiting backup ack in sync mode (ms)- pub write_ack_timeout_ms: Option<u64>,- /// Max concurrent async writes- pub write_concurrency: Option<usize>,- /// Retry loop interval in milliseconds- pub retry_interval_ms: Option<u64>,- /// Base backoff in milliseconds for retry attempts- pub retry_backoff_base_ms: Option<u64>,- /// Maximum retry attempts per file/target in one round- pub retry_max_retries_per_round: Option<usize>,- /// Failure threshold before quarantining one file/target pair- pub retry_quarantine_after_failures: Option<u32>,- /// Deprecated no-op retained for config compatibility.- #[serde(default)]- pub read_probe_cache_ttl_ms: Option<u64>,+ /// Cache key namespace used by cache-backed metadata.+ #[serde(default = "default_backups_namespace")]+ pub namespace: String,+ /// Initial number of metadata partitions.+ #[serde(default = "default_initial_partitions")]+ pub initial_partitions: u32,+ /// Checkpoint interval in seconds.+ #[serde(default = "default_checkpoint_interval_secs")]+ pub checkpoint_interval_secs: u64,+ /// Metadata storage provider.+ #[serde(default = "default_backups_provider")]+ pub provider: String,/// Backup items+ #[serde(default)]pub items: Vec<BackendItemConfig>,}-fn default_sync_type() -> String {- "async".to_string()+/// Return the default cache-backed metadata namespace.+fn default_backups_namespace() -> String {+ "default".to_string()+}++/// Return the default metadata partition count.+fn default_initial_partitions() -> u32 {+ 16+}++/// Return the default checkpoint interval in seconds.+fn default_checkpoint_interval_secs() -> u64 {+ 86_400+}++/// Return the default metadata provider.+fn default_backups_provider() -> String {+ "filesystem".to_string()}/// Single backup backend item configuration@@ -614,23 +606,8 @@ pub struct BackendItemConfig {/// Plugin-specific params (nested JSON)#[serde(default)]pub params: serde_json::Value,- /// Timeout in seconds- pub timeout: Option<u64>,/// Encryption config for this backuppub encryption: Option<EncryptionConfig>,- /// Operations this backup participates in- pub operations: Option<Vec<OperationItemConfig>>,- /// Exclude policies- pub excludes: Option<Vec<RedirectPolicy>>,-}--/// Per-operation priority config-#[derive(Debug, Clone, Serialize, Deserialize)]-pub struct OperationItemConfig {- /// Operation type: "read" | "write"- pub operation: String,- /// Priority (smaller = higher priority)- pub priority: u32,}/// Encryption on/off config for a backend@@ -640,211 +617,6 @@ pub struct EncryptionConfig {pub enabled: bool,}-/// Redirect / exclude policy (shared trait)-#[derive(Debug, Clone, Serialize, Deserialize)]-#[serde(tag = "type")]-pub enum RedirectPolicy {- /// Redirect/exclude files exceeding a size threshold- #[serde(rename = "FileOverSizePolicy")]- FileOverSizePolicy {- /// Max file size in MB- max_size_mb: u64,- /// Target backend names (redirect only)- target: Option<Vec<String>>,- },- /// Redirect/exclude files matching extension patterns- #[serde(rename = "FileExtensionPolicy")]- FileExtensionPolicy {- /// Regex patterns for file extensions- extensions: Vec<String>,- /// Target backend names (redirect only)- target: Option<Vec<String>>,- },-}--/// Strongly typed multi-write operation stored in `.sync_log.json`.-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]-#[serde(tag = "type")]-pub enum SyncOp {- /// Create an empty file.- Create,- /// Synchronize file content by copying the current primary state to the backup.- SyncFile {- /// File size used by retry/exclude policy decisions.- size: u64,- },- /// Create a directory.- Mkdir {- /// Directory mode.- mode: u32,- },- /// Change file mode.- Chmod {- /// File mode.- mode: u32,- },- /// Remove one file.- Remove,- /// Remove a file tree.- RemoveAll,- /// Rename to another path.- Rename {- /// Rename target path.- to: String,- },-}--/// Per-backend sync state-#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]-pub struct BackendSyncState {- /// Acknowledged sequence number- pub acked_seq: u64,- /// Consecutive retry failures for the current operation.- #[serde(default)]- pub retry_failures: u32,- /// Whether this backend/path pair is quarantined for manual intervention.- #[serde(default)]- pub quarantined: bool,-}--impl BackendSyncState {- /// Create an acknowledged backend sync state.- pub fn acked(acked_seq: u64) -> Self {- Self {- acked_seq,- retry_failures: 0,- quarantined: false,- }- }-- /// Mark this backend as acknowledged and clear retry state.- pub fn mark_acked(&mut self, acked_seq: u64) {- self.acked_seq = acked_seq;- self.retry_failures = 0;- self.quarantined = false;- }-- /// Record one retry failure and quarantine after the configured threshold.- pub fn mark_retry_failed(&mut self, quarantine_after_failures: u32) {- self.retry_failures = self.retry_failures.saturating_add(1);- if self.retry_failures >= quarantine_after_failures {- self.quarantined = true;- }- }-}--/// Sync log entry for a single file path.-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]-pub struct SyncLogEntry {- /// Latest monotonic sequence number.- pub latest_seq: u64,- /// Whether the primary backend has durably completed this operation.- #[serde(default = "default_primary_committed")]- pub primary_committed: bool,- /// Strongly typed operation payload.- pub op: SyncOp,- /// Per-backend acked sequence numbers.- pub backends: std::collections::HashMap<String, BackendSyncState>,-}--fn default_primary_committed() -> bool {- true-}--impl SyncLogEntry {- /// Create a new prepared sync log entry for a sequenced operation.- pub fn new(latest_seq: u64, op: SyncOp) -> Self {- Self {- latest_seq,- primary_committed: false,- op,- backends: std::collections::HashMap::new(),- }- }-- /// Create a committed sync log entry for an operation already applied on primary.- pub fn committed(latest_seq: u64, op: SyncOp) -> Self {- Self {- latest_seq,- primary_committed: true,- op,- backends: std::collections::HashMap::new(),- }- }-- /// Mark this entry as committed on primary.- pub fn mark_primary_committed(&mut self) {- self.primary_committed = true;- }-- /// Return whether this entry is committed on primary.- pub fn is_primary_committed(&self) -> bool {- self.primary_committed- }-- /// Return the backend sync state if present.- pub fn backend_state(&self, backend_name: &str) -> Option<&BackendSyncState> {- self.backends.get(backend_name)- }-- /// Return the acknowledged sequence for a backend, or 0 if unknown.- pub fn acked_seq(&self, backend_name: &str) -> u64 {- self.backend_state(backend_name)- .map(|state| state.acked_seq)- .unwrap_or(0)- }-- /// Return whether the backend is quarantined.- pub fn is_quarantined(&self, backend_name: &str) -> bool {- self.backend_state(backend_name)- .map(|state| state.quarantined)- .unwrap_or(false)- }-- /// Return whether the backend has acknowledged the latest operation.- pub fn is_in_sync(&self, backend_name: &str) -> bool {- self.acked_seq(backend_name) >= self.latest_seq- }-}--/// Sync log file content-#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]-pub struct SyncLogMeta {- /// File entries keyed by file name (current directory)- pub entries: std::collections::HashMap<String, SyncLogEntry>,-}--/// Redirect metadata file content-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]-pub struct RedirectMeta {- /// Schema version- #[serde(default = "default_redirect_version")]- pub version: u32,- /// Redirect entries keyed by file name- #[serde(default)]- pub entries: std::collections::HashMap<String, RedirectEntry>,-}--fn default_redirect_version() -> u32 {- 1-}--impl Default for RedirectMeta {- fn default() -> Self {- Self {- version: 1,- entries: HashMap::new(),- }- }-}--/// Single redirect entry-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]-pub struct RedirectEntry {- /// Target backend names- pub targets: Vec<String>,-}-/// Backend role#[derive(Debug, Clone, Copy, PartialEq, Eq)]pub enum BackendRole {@@ -854,15 +626,6 @@ pub enum BackendRole {Backup,}-/// Sync type enum-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-pub enum SyncType {- /// Synchronous: wait for backup ack before returning- Sync,- /// Asynchronous: return after primary write, sync in background- Async,-}-/// Custom serde module for SystemTimemod systemtime_serde {use serde::{Deserialize, Deserializer, Serialize, Serializer};@@ -922,21 +685,6 @@ mod tests {assert_eq!(param.param_type, "string");}- #[test]- fn test_sync_log_entry_legacy_payload_defaults_primary_committed_true() {- let entry: SyncLogEntry = serde_json::from_value(serde_json::json!({- "latest_seq": 7,- "op": {- "type": "Create"- },- "backends": {}- }))- .unwrap();-- assert!(entry.is_primary_committed());- assert_eq!(entry.latest_seq, 7);- }-#[test]fn test_plugin_config_from_raw_parts_handles_multiwrite_shapes() {let mut valid_params = HashMap::new();@@ -958,22 +706,10 @@ mod tests {"primary_encryption_enabled".to_string(),ConfigValue::Bool(true),);- valid_params.insert(- "primary_redirects".to_string(),- ConfigValue::Json(serde_json::json!([- {- "type": "FileExtensionPolicy",- "extensions": [".bin"],- "target": ["backup1"]- }- ])),- );-let config = PluginConfig::from_raw_parts("localfs", "/local", valid_params).unwrap();assert!(config.backups.is_some());assert!(config.server_encryption_enabled);assert!(config.primary_encryption_enabled);- assert_eq!(config.primary_redirects.len(), 1);assert_eq!(config.params.get("root_path"),Some(&ConfigValue::String("/tmp/data".to_string()))@@ -981,7 +717,6 @@ mod tests {assert!(!config.params.contains_key("backups"));assert!(!config.params.contains_key("server_encryption_enabled"));assert!(!config.params.contains_key("primary_encryption_enabled"));- assert!(!config.params.contains_key("primary_redirects"));for (params, expected_message) in [(diff --git a/crates/ragfs/src/git/service.rs b/crates/ragfs/src/git/service.rsindex 896e7941b..0f1548ae6 100644--- a/crates/ragfs/src/git/service.rs+++ b/crates/ragfs/src/git/service.rs@@ -6635,7 +6635,9 @@ mod fast_path1_tests {).unwrap();- let stack = build_default_stack(RagfsConfig::default()).await.unwrap();+ let mut ragfs_config = RagfsConfig::default();+ ragfs_config.pathlock.provider = "memory".to_string();+ let stack = build_default_stack(ragfs_config).await.unwrap();let mut params = HashMap::new();params.insert("local_dir".to_string(),@@ -6648,25 +6650,17 @@ mod fast_path1_tests {mount_path: "/local".to_string(),params,backups: Some(BackendsConfig {- sync_type: "async".to_string(),- write_ack_count: None,- write_ack_timeout_ms: None,- write_concurrency: None,- retry_interval_ms: None,- retry_backoff_base_ms: None,- retry_max_retries_per_round: None,- retry_quarantine_after_failures: None,- read_probe_cache_ttl_ms: None,+ namespace: "default".to_string(),+ initial_partitions: 16,+ checkpoint_interval_secs: 86_400,+ provider: "filesystem".to_string(),items: vec![BackendItemConfig {name: "backup1".to_string(),backend: "localfs".to_string(),params: serde_json::json!({"local_dir": backup_dir.path().to_str().unwrap(),}),- timeout: None,encryption: None,- operations: None,- excludes: None,}],}),..PluginConfig::default()diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rsindex 668263a61..cef2bdb7e 100644--- a/crates/ragfs/src/lock/manager.rs+++ b/crates/ragfs/src/lock/manager.rs@@ -875,7 +875,7 @@ impl PathLockManager {}if waiting.0.is_none() {- info!(owner_id = %owner_id, requests = ?sorted, timeout_ms = timeout.as_millis() as u64, error = %err, "pathlock acquire batch entered wait state");+ debug!(owner_id = %owner_id, requests = ?sorted, timeout_ms = timeout.as_millis() as u64, error = %err, "pathlock acquire batch entered wait state");waiting.0 = Some(&self.waiting_lock_count);self.waiting_lock_count.fetch_add(1, AtomicOrdering::Relaxed);diff --git a/crates/ragfs/src/multibackend/admin.rs b/crates/ragfs/src/multibackend/admin.rsdeleted file mode 100644index d0450a1b3..000000000--- a/crates/ragfs/src/multibackend/admin.rs+++ /dev/null@@ -1,234 +0,0 @@-use std::collections::HashSet;-use std::time::Duration;--use serde_json::{json, Value};--use crate::core::errors::Result;-use crate::core::filesystem::{normalize_prefix_path, FileSystem};-use crate::core::multibackend_wrapper::{MultiWriteWrappedFS, SyncWorkEntry};-use crate::multibackend::meta::{current_required_ctx, parent_dir};--impl MultiWriteWrappedFS {- /// Collect effective sync work entries under a path using the current request context.- async fn collect_sync_work(&self, path: &str) -> Result<Vec<SyncWorkEntry>> {- let ctx = current_required_ctx()?;- let inner = &self.inner;- let normalized = normalize_prefix_path(path);- let path_info = <Self as FileSystem>::stat(self, &normalized).await?;- let mut dirs = Vec::new();- let mut seen_dirs = HashSet::new();-- let add_dir = |dirs: &mut Vec<String>, seen_dirs: &mut HashSet<String>, dir: String| {- if seen_dirs.insert(dir.clone()) {- dirs.push(dir);- }- };-- if path_info.is_dir {- add_dir(&mut dirs, &mut seen_dirs, normalized.clone());- for dir in inner.pending_dirs_snapshot().await {- if dir == normalized- || dir- .strip_prefix(&normalized)- .is_some_and(|suffix| suffix.starts_with('/'))- {- add_dir(&mut dirs, &mut seen_dirs, dir);- }- }- if dirs.len() == 1 {- for entry in inner- .primary()- .backend- .tree_directory(&normalized, true, None, None, None, None, None, false)- .await?- {- if entry.info.is_dir {- add_dir(- &mut dirs,- &mut seen_dirs,- normalize_prefix_path(&entry.path),- );- }- }- }- } else {- add_dir(- &mut dirs,- &mut seen_dirs,- normalize_prefix_path(&parent_dir(&normalized)),- );- }-- let mut work = Vec::new();- for dir in dirs {- work.extend(- inner- .sync_work_in_dir(&dir, &ctx)- .await?- .into_iter()- .filter(|work| path_info.is_dir || work.file_path == normalized),- );- }-- Ok(work)- }-- /// Query effective multi-write sync status under a file or directory path.- pub async fn system_sync_status(&self, path: &str) -> Result<Value> {- let work = self.collect_sync_work(path).await?;- let mut entries = Vec::new();- let mut pending_target_count = 0usize;-- for work in work {- let mut targets = Vec::new();- let primary_committed = work.entry.is_primary_committed();- let mut all_synced = primary_committed;-- for backend_name in work.targets {- let acked_seq = if primary_committed {- work.entry.acked_seq(&backend_name)- } else {- 0- };- let in_sync = primary_committed && work.entry.is_in_sync(&backend_name);- if primary_committed && !in_sync {- pending_target_count += 1;- all_synced = false;- }- let state = work.entry.backend_state(&backend_name);- targets.push(json!({- "name": backend_name,- "acked_seq": acked_seq,- "retry_failures": state.map(|state| state.retry_failures).unwrap_or(0),- "quarantined": state.map(|state| state.quarantined).unwrap_or(false),- "primary_committed": primary_committed,- "in_sync": in_sync,- }));- }-- entries.push(json!({- "path": work.file_path,- "latest_seq": work.entry.latest_seq,- "primary_committed": primary_committed,- "op": serde_json::to_value(&work.entry.op)?,- "all_synced": all_synced,- "targets": targets,- }));- }-- entries.sort_by(|a, b| {- let ap = a.get("path").and_then(Value::as_str).unwrap_or_default();- let bp = b.get("path").and_then(Value::as_str).unwrap_or_default();- ap.cmp(bp)- });-- Ok(json!({- "path": normalize_prefix_path(path),- "entry_count": entries.len(),- "pending_target_count": pending_target_count,- "capabilities": {- "multi_instance_safe": false- },- "read_route_metrics": self.inner.read_route_metrics(),- "entries": entries,- }))- }-- /// Manually retry lagging multi-write targets under a file or directory path.- pub async fn system_sync_retry(&self, path: &str) -> Result<Value> {- let ctx = current_required_ctx()?;- let work = self.collect_sync_work(path).await?;- let mut results = Vec::new();- let mut retried = 0usize;- let mut failed = 0usize;- let mut skipped = 0usize;-- for work in work {- if !work.entry.is_primary_committed() {- skipped += work.targets.len();- for backend_name in work.targets {- results.push(json!({- "path": work.file_path,- "target": backend_name,- "status": "awaiting_primary_commit",- "latest_seq": work.entry.latest_seq,- "primary_committed": false,- "acked_seq": 0,- }));- }- continue;- }-- for backend_name in work.targets {- let acked_seq = work.entry.acked_seq(&backend_name);- let was_quarantined = work.entry.is_quarantined(&backend_name);- if work.entry.is_in_sync(&backend_name) {- skipped += 1;- results.push(json!({- "path": work.file_path,- "target": backend_name,- "status": "skipped",- "latest_seq": work.entry.latest_seq,- "acked_seq": acked_seq,- }));- continue;- }-- let mut last_error = None;- let mut success = false;- for _attempt in 0..self.inner.max_retry_per_round {- match self- .inner- .replay_operation(&work.file_path, &backend_name, &ctx)- .await- {- Ok(()) => {- success = true;- break;- }- Err(err) => {- last_error = Some(err.to_string());- tokio::time::sleep(Duration::from_millis(- self.inner.retry_backoff_base_ms,- ))- .await;- }- }- }-- if success {- retried += 1;- results.push(json!({- "path": work.file_path,- "target": backend_name,- "status": "retried",- "latest_seq": work.entry.latest_seq,- "acked_seq": work.entry.latest_seq,- }));- } else {- self.inner- .record_backup_retry_failure(&work.file_path, &backend_name, &ctx)- .await?;- failed += 1;- results.push(json!({- "path": work.file_path,- "target": backend_name,- "status": "failed",- "latest_seq": work.entry.latest_seq,- "acked_seq": acked_seq,- "was_quarantined": was_quarantined,- "error": last_error.unwrap_or_else(|| "unknown replay error".to_string()),- }));- }- }- }-- Ok(json!({- "path": normalize_prefix_path(path),- "retried": retried,- "failed": failed,- "skipped": skipped,- "results": results,- }))- }-}diff --git a/crates/ragfs/src/multibackend/catch_up.rs b/crates/ragfs/src/multibackend/catch_up.rsnew file mode 100644index 000000000..0ed190b74--- /dev/null+++ b/crates/ragfs/src/multibackend/catch_up.rs@@ -0,0 +1,1058 @@+//! Backup catch-up workers and current-state reconciliation primitives.++use std::collections::{BTreeMap, BTreeSet, HashSet};+use std::future::Future;+use std::path::Path;+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::Arc;+use std::time::Duration;++use async_trait::async_trait;+use rand::Rng;+use serde_json::Value;+use tokio::sync::watch;+use tracing::{debug, warn};++use crate::core::context::{FsContext, FsContextInner, FS_CTX};+use crate::core::errors::{Error, Result};+use crate::core::filesystem::FileSystem;+use crate::core::internal_names::is_multiwrite_internal_path;+use crate::core::types::{FileInfo, ListSortBy, SortOrder, WriteFlag};+use crate::lock::OwnedPathLockLease;+use crate::multibackend::constants::{MULTIWRITE_MOUNT_PREFIX, SYSTEM_DIR};+use crate::multibackend::meta::{MetadataStore, MultiWriteWorker};+use crate::multibackend::model::{+ FileState, PartitionState, PartitionsManifest, ScopeKey, SegmentEventType, SegmentRecord,+};+use crate::multibackend::provider::MultiWriteProvider;++#[cfg(test)]+#[path = "tests/test_catch_up.rs"]+mod tests;++/// One configured backup handle owned by its catch-up worker.+pub struct CatchUpTarget {+ /// Stable backend id stored in partition progress.+ pub backend_id: String,+ /// Encrypted business filesystem handle.+ pub backend: Arc<dyn FileSystem>,+}++/// Parsed checkpoint state accepted by CatchUpWorker from the future reader.+#[derive(Clone)]+pub struct CheckpointSnapshot {+ /// Last sequence represented by this complete checkpoint.+ pub checkpoint_to_seq: u64,+ /// Materialized file states already decoded by the checkpoint reader.+ pub file_states: Vec<FileState>,+ /// Directory prefixes requiring current-state reconciliation.+ pub directory_prefixes: Vec<String>,+}++/// Typed outcome of reading the latest checkpoint publication.+pub enum CheckpointReadResult {+ /// No published checkpoint fits the requested sequence window.+ NoCheckpoint,+ /// One complete validated checkpoint snapshot.+ Snapshot(CheckpointSnapshot),+ /// Immutable files changed during a reader/GC race; reread latest.+ RetryLatestCheckpoint,+}++/// Consumer boundary implemented by the future checkpoint reader.+#[async_trait]+pub trait CheckpointConsumer: Send + Sync {+ /// Return the latest usable parsed checkpoint inside the fixed tail.+ async fn read_latest(+ &self,+ scope: &ScopeKey,+ synced_seq: u64,+ tail_seq: u64,+ ) -> Result<CheckpointReadResult>;+}++struct EmptyCheckpointConsumer;++#[async_trait]+impl CheckpointConsumer for EmptyCheckpointConsumer {+ /// Return no checkpoint until T-011 supplies a reader.+ async fn read_latest(+ &self,+ _scope: &ScopeKey,+ _synced_seq: u64,+ _tail_seq: u64,+ ) -> Result<CheckpointReadResult> {+ Ok(CheckpointReadResult::NoCheckpoint)+ }+}++/// One backup worker that consumes V2 metadata without blocking primary writes.+pub struct CatchUpWorker {+ backend_id: String,+ primary: Arc<dyn FileSystem>,+ backup: Arc<dyn FileSystem>,+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+ checkpoint_consumer: Arc<dyn CheckpointConsumer>,+ cancellation: Option<watch::Receiver<bool>>,+}++/// One Exact PathLock lease protecting a backup catch-up worker.+pub struct CatchUpLease {+ store: Arc<MetadataStore>,+ lease: OwnedPathLockLease,+ valid: AtomicBool,+}++enum CatchUpFailure {+ Cancelled,+ Failed(Error),+}++impl From<Error> for CatchUpFailure {+ /// Preserve ordinary failures while propagating explicit cancellation separately.+ fn from(error: Error) -> Self {+ Self::Failed(error)+ }+}++type CatchUpResult<T> = std::result::Result<T, CatchUpFailure>;++impl CatchUpLease {+ /// Acquire the mount-level Exact lease for one logical backup id.+ async fn acquire(store: Arc<MetadataStore>, backend_id: &str) -> Result<Self> {+ let path =+ format!("{MULTIWRITE_MOUNT_PREFIX}/{SYSTEM_DIR}/.multiwrite.catch-up.{backend_id}");+ let manager = store.pathlock_manager();+ let lease = manager+ .acquire_exact(+ &path,+ manager.default_lock_timeout().max(Duration::from_secs(5)),+ None,+ )+ .await?;+ Ok(Self {+ store,+ lease,+ valid: AtomicBool::new(true),+ })+ }++ /// Refresh the lease and return false for every non-refreshed result.+ async fn refresh(&self) -> Result<bool> {+ let result = self.store.pathlock_manager().refresh(&self.lease).await;+ let refreshed = matches!(result.as_deref(), Ok("refreshed"));+ if !refreshed {+ self.valid.store(false, Ordering::SeqCst);+ }+ result.map(|_| refreshed).map_err(Error::from)+ }++ /// Return whether every explicit refresh has retained the lease.+ fn is_valid(&self) -> bool {+ self.valid.load(Ordering::SeqCst)+ }++ /// Release this worker's Exact lease.+ async fn release(&self) -> Result<()> {+ self.store+ .pathlock_manager()+ .release(&self.lease)+ .await+ .map_err(Error::from)+ }+}++impl CatchUpWorker {+ /// Create one worker for an encrypted primary and backup handle.+ pub fn new(+ backend_id: String,+ primary: Arc<dyn FileSystem>,+ backup: Arc<dyn FileSystem>,+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+ ) -> Self {+ Self {+ backend_id,+ primary,+ backup,+ store,+ provider,+ checkpoint_consumer: Arc::new(EmptyCheckpointConsumer),+ cancellation: None,+ }+ }++ /// Attach the parsed-checkpoint consumer supplied by T-011.+ pub fn with_checkpoint_consumer(+ mut self,+ checkpoint_consumer: Arc<dyn CheckpointConsumer>,+ ) -> Self {+ self.checkpoint_consumer = checkpoint_consumer;+ self+ }++ /// Attach a runtime cancellation receiver for interruptible catch-up rounds.+ pub(crate) fn with_cancellation(mut self, cancellation: watch::Receiver<bool>) -> Self {+ self.cancellation = Some(cancellation);+ self+ }++ /// Acquire the backup lease, execute one bounded round, and release it.+ pub async fn run_once(&self) -> Result<()> {+ let outcome: CatchUpResult<()> = async {+ let lease =+ Arc::new(CatchUpLease::acquire(self.store.clone(), &self.backend_id).await?);+ debug!(backend = %self.backend_id, "multi-write catch-up lease acquired");+ let operation = self.run_round(&lease).await;+ let release = lease.release().await;+ match (operation, release) {+ (Ok(()), Ok(())) => Ok(()),+ (Err(CatchUpFailure::Cancelled), Ok(())) => Err(CatchUpFailure::Cancelled),+ (Err(CatchUpFailure::Failed(error)), _) => Err(CatchUpFailure::Failed(error)),+ (_, Err(error)) => Err(CatchUpFailure::Failed(error)),+ }+ }+ .await;+ match outcome {+ Ok(()) => Ok(()),+ Err(CatchUpFailure::Cancelled) => Err(Error::internal("catch-up was cancelled")),+ Err(CatchUpFailure::Failed(error)) => {+ self.store.record_worker_error(MultiWriteWorker::CatchUp);+ Err(error)+ }+ }+ }++ /// Wait until runtime cancellation or remain pending for standalone workers.+ async fn cancelled(&self) {+ let Some(mut cancellation) = self.cancellation.clone() else {+ std::future::pending::<()>().await;+ return;+ };+ while !*cancellation.borrow() {+ if cancellation.changed().await.is_err() {+ return;+ }+ }+ }++ /// Reconcile registry membership and one fixed partition window per scope.+ async fn run_round(&self, lease: &Arc<CatchUpLease>) -> CatchUpResult<()> {+ let guarded = LeaseGuardedFileSystem {+ inner: self.backup.clone(),+ lease: lease.clone(),+ };+ let mut accounts = self.read_accounts().await?;+ self.remove_extra_accounts(&guarded, &accounts).await?;+ if !accounts.iter().any(|account| account == SYSTEM_DIR) {+ accounts.insert(0, SYSTEM_DIR.to_string());+ }++ for account in accounts {+ let path = self.store.paths().account_manifest(&account)?.0;+ let manifest: PartitionsManifest = match self.store.read_json(&path).await {+ Ok(manifest) => manifest,+ Err(Error::NotFound(_)) => continue,+ Err(error) => {+ self.store.record_worker_error(MultiWriteWorker::CatchUp);+ warn!(backend = %self.backend_id, account = %account, error = %error,+ "multi-write catch-up account metadata unavailable");+ continue;+ }+ };+ if let Err(error) = manifest.validate() {+ self.store.record_worker_error(MultiWriteWorker::CatchUp);+ warn!(backend = %self.backend_id, account = %account, error = %error,+ "multi-write catch-up account metadata invalid");+ continue;+ }+ for (&partition_id, partition) in &manifest.partitions {+ if partition.state != PartitionState::Stable {+ continue;+ }+ let scope = ScopeKey {+ account_id: account.clone(),+ partition_id,+ epoch: manifest.epoch,+ };+ if let Err(failure) = self.reconcile_scope(&guarded, lease, &scope).await {+ let error = match failure {+ CatchUpFailure::Cancelled => return Err(CatchUpFailure::Cancelled),+ CatchUpFailure::Failed(Error::NotFound(_)) => continue,+ CatchUpFailure::Failed(error) => error,+ };+ if !lease.is_valid() {+ return Err(error.into());+ }+ self.store.record_worker_error(MultiWriteWorker::CatchUp);+ warn!(+ backend = %self.backend_id,+ account = %account,+ partition = partition_id,+ error = %error,+ "multi-write catch-up partition failed"+ );+ }+ }+ }+ Ok(())+ }++ /// Read sorted account ids from the encrypted mount-level registry.+ async fn read_accounts(&self) -> Result<Vec<String>> {+ let bytes = self+ .with_context(+ SYSTEM_DIR,+ self.primary.read("/_system/accounts.json", 0, 0),+ )+ .await?;+ let value: Value = serde_json::from_slice(&bytes)?;+ let accounts = value+ .get("accounts")+ .and_then(Value::as_object)+ .ok_or_else(|| Error::Serialization("accounts.json lacks an accounts object".into()))?;+ let mut ids = accounts.keys().cloned().collect::<Vec<_>>();+ ids.sort();+ Ok(ids)+ }++ /// Remove backup account roots that are absent from the primary registry.+ async fn remove_extra_accounts(+ &self,+ backup: &LeaseGuardedFileSystem,+ accounts: &[String],+ ) -> Result<()> {+ let expected = accounts.iter().map(String::as_str).collect::<HashSet<_>>();+ let entries = self+ .with_context(SYSTEM_DIR, backup.read_internal_dir("/"))+ .await?;+ for entry in entries {+ if entry.is_dir && entry.name != SYSTEM_DIR && !expected.contains(entry.name.as_str()) {+ let path = format!("/{}", entry.name);+ self.with_context(&entry.name, backup.remove_all(&path))+ .await?;+ }+ }+ Ok(())+ }++ /// Consume one fixed `(synced_seq, tail]` window and advance after convergence.+ async fn reconcile_scope(+ &self,+ backup: &LeaseGuardedFileSystem,+ lease: &CatchUpLease,+ scope: &ScopeKey,+ ) -> CatchUpResult<()> {+ let manifest = self.provider.read_manifest(scope).await?;+ let state = manifest+ .backend_states+ .get(&self.backend_id)+ .ok_or_else(|| {+ Error::invalid_operation(format!("unknown backend id: {}", self.backend_id))+ })?;+ let tail = manifest.next_seq - 1;+ debug!(+ backend = %self.backend_id,+ account = %scope.account_id,+ partition = scope.partition_id,+ synced_seq = state.synced_seq,+ tail_seq = tail,+ lag = tail - state.synced_seq,+ "multi-write catch-up partition snapshot"+ );+ if state.synced_seq == tail {+ return Ok(());+ }+ let records = self+ .provider+ .read_committed_head(scope, state.synced_seq + 1, tail + 1)+ .await;+ let records = match records {+ Ok(records) if continuous_records(&records, state.synced_seq + 1, tail + 1) => {+ Some(records)+ }+ _ => self+ .provider+ .read_committed_range(scope, state.synced_seq + 1, tail + 1)+ .await+ .ok()+ .filter(|records| continuous_records(records, state.synced_seq + 1, tail + 1)),+ };+ let Some(records) = records else {+ return self+ .reconcile_checkpoint(backup, lease, scope, state.synced_seq, tail)+ .await;+ };+ let progress = self+ .reconcile_records(backup, lease, scope, state.synced_seq, tail, records)+ .await?;+ self.advance_progress(lease, scope, state.synced_seq, progress)+ .await?;+ Ok(())+ }++ /// Reconcile ordinary paths first, then delayed exact and prefix directory paths.+ async fn reconcile_records(+ &self,+ backup: &LeaseGuardedFileSystem,+ lease: &CatchUpLease,+ scope: &ScopeKey,+ synced_seq: u64,+ tail: u64,+ records: Vec<SegmentRecord>,+ ) -> CatchUpResult<u64> {+ let (mut ordinary, mut delayed_exact, mut prefixes, mut scope_deletes) = (+ BTreeMap::new(),+ BTreeMap::new(),+ BTreeMap::new(),+ BTreeMap::new(),+ );+ for record in records {+ match record.event_type {+ SegmentEventType::Write | SegmentEventType::Remove => {+ let path = self+ .store+ .paths()+ .backend_path(&scope.account_id, &record.path)?;+ insert_earliest(&mut ordinary, path, record.seq);+ }+ SegmentEventType::RemoveTree => {+ let path = self+ .store+ .paths()+ .backend_path(&scope.account_id, &record.path)?;+ insert_earliest(&mut prefixes, path, record.seq);+ }+ SegmentEventType::MoveTree => {+ let source = self+ .store+ .paths()+ .backend_path(&scope.account_id, &record.path)?;+ let destination = self.store.paths().backend_path(+ &scope.account_id,+ record.destination_path.as_deref().ok_or_else(|| {+ Error::Serialization("move marker lacks destination".to_string())+ })?,+ )?;+ let (exact, dirty) = self+ .with_context(+ &scope.account_id,+ classify_move_paths(backup, &source, &destination),+ )+ .await?;+ for path in exact {+ insert_earliest(&mut delayed_exact, path, record.seq);+ }+ for path in dirty {+ insert_earliest(&mut prefixes, path, record.seq);+ }+ }+ SegmentEventType::DeleteScope => {+ if scope.account_id != SYSTEM_DIR {+ return Err(Error::invalid_operation(+ "DeleteScope is valid only in the system scope",+ )+ .into());+ }+ let (account_id, path) =+ delete_scope_target(self.store.as_ref(), &record.path)?;+ insert_earliest_scope_delete(&mut scope_deletes, path, account_id, record.seq);+ }+ }+ }+ let mut failed_from = None;+ for (path, (account_id, seq)) in scope_deletes {+ match self+ .remove_scope_with_retry(backup, lease, &account_id, &path)+ .await+ {+ Ok(()) => {+ if let Err(error) = self+ .reset_account_progress_after_scope_delete(lease, &account_id)+ .await+ {+ if !lease.is_valid() {+ return Err(error.into());+ }+ warn!(+ backend = %self.backend_id,+ account = %account_id,+ error = %error,+ "multi-write catch-up scope-delete progress reset failed"+ );+ failed_from = Some(failed_from.map_or(seq, |failed: u64| failed.min(seq)));+ }+ }+ Err(CatchUpFailure::Cancelled) => return Err(CatchUpFailure::Cancelled),+ Err(CatchUpFailure::Failed(_)) if !lease.is_valid() => {+ return Err(Error::internal("catch-up lease was not refreshed").into());+ }+ Err(CatchUpFailure::Failed(_)) => {+ failed_from = Some(failed_from.map_or(seq, |failed: u64| failed.min(seq)));+ }+ }+ }+ for (paths, prefix) in [(ordinary, false), (delayed_exact, false), (prefixes, true)] {+ for (path, seq) in paths {+ match self+ .reconcile_with_retry(backup, lease, &scope.account_id, &path, prefix)+ .await+ {+ Ok(()) => {}+ Err(CatchUpFailure::Cancelled) => return Err(CatchUpFailure::Cancelled),+ Err(CatchUpFailure::Failed(_)) if !lease.is_valid() => {+ return Err(Error::internal("catch-up lease was not refreshed").into());+ }+ Err(CatchUpFailure::Failed(_)) => {+ failed_from = Some(failed_from.map_or(seq, |failed: u64| failed.min(seq)));+ }+ }+ }+ }+ Ok(failed_from.map_or(tail, |seq| seq.saturating_sub(1).max(synced_seq)))+ }++ /// Remove one backup scope root with the same bounded retry policy as path sync.+ async fn remove_scope_with_retry(+ &self,+ backup: &LeaseGuardedFileSystem,+ lease: &CatchUpLease,+ account_id: &str,+ path: &str,+ ) -> CatchUpResult<()> {+ let mut last_error = None;+ for attempt in 0..=3 {+ match self+ .with_context(account_id, remove_if_present(backup, path))+ .await+ {+ Ok(()) => return Ok(()),+ Err(error) => last_error = Some(error),+ }+ if !lease.is_valid() || attempt == 3 {+ break;+ }+ let jitter = rand::thread_rng().gen_range(0..=50);+ let delay_ms = (1u64 << attempt) * 1_000 + jitter;+ warn!(+ backend = %self.backend_id,+ account = %account_id,+ path = %path,+ retry = attempt + 1,+ delay_ms,+ error = %last_error.as_ref().unwrap(),+ "multi-write catch-up scope-delete retry"+ );+ tokio::select! {+ _ = tokio::time::sleep(Duration::from_millis(delay_ms)) => {}+ _ = self.cancelled() => return Err(CatchUpFailure::Cancelled),+ }+ }+ Err(CatchUpFailure::Failed(last_error.unwrap()))+ }++ /// Reset this backup's progress for a recreated account after scope delete.+ async fn reset_account_progress_after_scope_delete(+ &self,+ lease: &CatchUpLease,+ account_id: &str,+ ) -> Result<()> {+ let path = self.store.paths().account_manifest(account_id)?.0;+ let manifest: PartitionsManifest = match self.store.read_json(&path).await {+ Ok(manifest) => manifest,+ Err(Error::NotFound(_)) => return Ok(()),+ Err(error) => return Err(error),+ };+ manifest.validate()?;+ if !lease.refresh().await? {+ return Err(Error::internal("catch-up lease was not refreshed"));+ }+ for (&partition_id, _) in &manifest.partitions {+ let scope = ScopeKey {+ account_id: account_id.to_string(),+ partition_id,+ epoch: manifest.epoch,+ };+ self.provider+ .advance_backend_state(&scope, &self.backend_id, 0)+ .await?;+ }+ Ok(())+ }++ /// Refresh the lease immediately before a monotonic progress update.+ async fn advance_progress(+ &self,+ lease: &CatchUpLease,+ scope: &ScopeKey,+ previous: u64,+ progress: u64,+ ) -> Result<()> {+ if progress == previous {+ return Ok(());+ }+ if !lease.refresh().await? {+ return Err(Error::internal("catch-up lease was not refreshed"));+ }+ self.provider+ .advance_backend_state(scope, &self.backend_id, progress)+ .await+ }++ /// Reconcile one path with fixed exponential delays and bounded jitter.+ async fn reconcile_with_retry(+ &self,+ backup: &LeaseGuardedFileSystem,+ lease: &CatchUpLease,+ account_id: &str,+ path: &str,+ prefix: bool,+ ) -> CatchUpResult<()> {+ let mut last_error = None;+ for attempt in 0..=3 {+ let operation = if prefix {+ self.with_context(+ account_id,+ reconcile_prefix(self.primary.as_ref(), backup, path),+ )+ .await+ } else {+ self.with_context(+ account_id,+ reconcile_file(self.primary.as_ref(), backup, path),+ )+ .await+ };+ match operation {+ Ok(()) => return Ok(()),+ Err(error) => last_error = Some(error),+ }+ if !lease.is_valid() || attempt == 3 {+ break;+ }+ let jitter = rand::thread_rng().gen_range(0..=50);+ let delay_ms = (1u64 << attempt) * 1_000 + jitter;+ warn!(+ backend = %self.backend_id,+ account = %account_id,+ path = %path,+ retry = attempt + 1,+ delay_ms,+ error = %last_error.as_ref().unwrap(),+ "multi-write catch-up path retry"+ );+ tokio::select! {+ _ = tokio::time::sleep(Duration::from_millis(delay_ms)) => {}+ _ = self.cancelled() => return Err(CatchUpFailure::Cancelled),+ }+ }+ Err(CatchUpFailure::Failed(last_error.unwrap()))+ }++ /// Consume one already parsed checkpoint and advance only after full convergence.+ async fn reconcile_checkpoint(+ &self,+ backup: &LeaseGuardedFileSystem,+ lease: &CatchUpLease,+ scope: &ScopeKey,+ synced_seq: u64,+ tail: u64,+ ) -> CatchUpResult<()> {+ let snapshot = match self+ .checkpoint_consumer+ .read_latest(scope, synced_seq, tail)+ .await?+ {+ CheckpointReadResult::Snapshot(snapshot) => snapshot,+ CheckpointReadResult::NoCheckpoint => {+ return Err(Error::invalid_operation("segment window is not continuous").into());+ }+ CheckpointReadResult::RetryLatestCheckpoint => {+ return Err(Error::invalid_operation("retry latest checkpoint").into());+ }+ };+ if snapshot.checkpoint_to_seq <= synced_seq || snapshot.checkpoint_to_seq > tail {+ return Err(Error::Serialization(+ "checkpoint sequence is outside the fixed catch-up window".to_string(),+ )+ .into());+ }+ let mut paths = BTreeSet::new();+ for state in snapshot.file_states {+ state.validate(scope)?;+ if state.latest_seq.seq > snapshot.checkpoint_to_seq {+ return Err(Error::Serialization(+ "checkpoint file state exceeds checkpoint sequence".to_string(),+ )+ .into());+ }+ paths.insert(+ self.store+ .paths()+ .backend_path(&scope.account_id, &state.path)?,+ );+ }+ for path in paths {+ self.reconcile_with_retry(backup, lease, &scope.account_id, &path, false)+ .await?;+ }+ let mut prefixes = BTreeSet::new();+ for prefix in snapshot.directory_prefixes {+ prefixes.insert(+ self.store+ .paths()+ .backend_path(&scope.account_id, &prefix)?,+ );+ }+ for prefix in prefixes {+ self.reconcile_with_retry(backup, lease, &scope.account_id, &prefix, true)+ .await?;+ }+ self.advance_progress(lease, scope, synced_seq, snapshot.checkpoint_to_seq)+ .await?;+ if snapshot.checkpoint_to_seq == tail {+ return Ok(());+ }+ let records = self+ .provider+ .read_committed_range(scope, snapshot.checkpoint_to_seq + 1, tail + 1)+ .await?;+ if !continuous_records(&records, snapshot.checkpoint_to_seq + 1, tail + 1) {+ return Err(Error::invalid_operation(+ "post-checkpoint segment window is not continuous",+ )+ .into());+ }+ let progress = self+ .reconcile_records(+ backup,+ lease,+ scope,+ snapshot.checkpoint_to_seq,+ tail,+ records,+ )+ .await?;+ self.advance_progress(lease, scope, snapshot.checkpoint_to_seq, progress)+ .await?;+ Ok(())+ }++ /// Run one filesystem future with cache bypass and automatic PathLock disabled.+ async fn with_context<T>(+ &self,+ account_id: &str,+ future: impl Future<Output = Result<T>>,+ ) -> Result<T> {+ let context: FsContext = Arc::new(+ FsContextInner::new(account_id)+ .with_bypass_cache(true)+ .with_auto_pathlock_disabled(),+ );+ FS_CTX.scope(context, future).await+ }+}++/// Insert one path while retaining its earliest dependent sequence.+fn insert_earliest(paths: &mut BTreeMap<String, u64>, path: String, seq: u64) {+ paths+ .entry(path)+ .and_modify(|current| *current = (*current).min(seq))+ .or_insert(seq);+}++/// Insert one scope delete while retaining its earliest dependent sequence.+fn insert_earliest_scope_delete(+ paths: &mut BTreeMap<String, (String, u64)>,+ path: String,+ account_id: String,+ seq: u64,+) {+ paths+ .entry(path)+ .and_modify(|(_, current)| *current = (*current).min(seq))+ .or_insert((account_id, seq));+}++/// Convert a logical `/local/{scope}` deletion target into backup path and context.+fn delete_scope_target(store: &MetadataStore, logical_path: &str) -> Result<(String, String)> {+ let path = store.paths().raw_backend_path(logical_path)?;+ let account_id = path+ .trim_start_matches('/')+ .split('/')+ .next()+ .filter(|value| !value.is_empty())+ .ok_or_else(|| Error::invalid_path(format!("invalid DeleteScope path: {logical_path}")))?;+ Ok((account_id.to_string(), path))+}++struct LeaseGuardedFileSystem {+ inner: Arc<dyn FileSystem>,+ lease: Arc<CatchUpLease>,+}++impl LeaseGuardedFileSystem {+ /// Reject the next backup mutation unless the lease refreshes literally.+ async fn before_write(&self) -> Result<()> {+ if self.lease.refresh().await? {+ Ok(())+ } else {+ Err(Error::internal("catch-up lease was not refreshed"))+ }+ }+}++#[async_trait]+impl FileSystem for LeaseGuardedFileSystem {+ /// Refresh the catch-up lease before creating a backup file.+ async fn create(&self, path: &str) -> Result<()> {+ self.before_write().await?;+ self.inner.create(path).await+ }++ /// Refresh the catch-up lease before creating a backup directory.+ async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {+ self.before_write().await?;+ self.inner.mkdir(path, mode).await+ }++ /// Refresh the catch-up lease before removing a backup file.+ async fn remove(&self, path: &str) -> Result<()> {+ self.before_write().await?;+ self.inner.remove(path).await+ }++ /// Refresh the catch-up lease before removing a backup subtree.+ async fn remove_all(&self, path: &str) -> Result<()> {+ self.before_write().await?;+ self.inner.remove_all(path).await+ }++ /// Read bytes from the guarded backup without mutating it.+ async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {+ self.inner.read(path, offset, size).await+ }++ /// Refresh the catch-up lease before writing backup bytes.+ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {+ self.before_write().await?;+ self.inner.write(path, data, offset, flags).await+ }++ /// List guarded backup entries without mutating them.+ async fn read_dir(+ &self,+ path: &str,+ offset: Option<usize>,+ limit: Option<usize>,+ sort_by: Option<ListSortBy>,+ sort_order: Option<SortOrder>,+ ) -> Result<Vec<FileInfo>> {+ self.inner+ .read_dir(path, offset, limit, sort_by, sort_order)+ .await+ }++ /// Read guarded backup metadata without mutating it.+ async fn stat(&self, path: &str) -> Result<FileInfo> {+ self.inner.stat(path).await+ }++ /// Refresh the catch-up lease before renaming a backup path.+ async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {+ self.before_write().await?;+ self.inner.rename(old_path, new_path).await+ }++ /// Refresh the catch-up lease before changing a backup mode.+ async fn chmod(&self, path: &str, mode: u32) -> Result<()> {+ self.before_write().await?;+ self.inner.chmod(path, mode).await+ }+}++/// Return whether records exactly cover one requested half-open sequence range.+fn continuous_records(records: &[SegmentRecord], from: u64, to: u64) -> bool {+ records.len() as u64 == to - from+ && records+ .iter()+ .enumerate()+ .all(|(index, record)| record.seq == from + index as u64)+}++/// Classify move source and destination by the backup's current source type.+pub async fn classify_move_paths(+ backup: &dyn FileSystem,+ source: &str,+ destination: &str,+) -> Result<(HashSet<String>, HashSet<String>)> {+ let paths = HashSet::from([source.to_string(), destination.to_string()]);+ match backup.stat(source).await {+ Ok(info) if !info.is_dir => Ok((paths, HashSet::new())),+ Ok(_) | Err(Error::NotFound(_)) => Ok((HashSet::new(), paths)),+ Err(error) => Err(error),+ }+}++/// Reconcile one file or directory prefix from primary state into a backup.+pub async fn reconcile_prefix(+ primary: &dyn FileSystem,+ backup: &dyn FileSystem,+ prefix: &str,+) -> Result<()> {+ if is_catch_up_internal_path(prefix) {+ return Ok(());+ }+ let root = match primary.stat(prefix).await {+ Ok(root) => root,+ Err(Error::NotFound(_)) => {+ return remove_stale_paths(backup, prefix, &HashSet::new()).await;+ }+ Err(error) => return Err(error),+ };+ if !root.is_dir {+ return copy_file_state(primary, backup, prefix).await;+ }++ let entries = primary+ .tree_directory(prefix, true, None, None, None, None, None, false)+ .await?;+ ensure_directory(backup, prefix, root.mode).await?;+ let mut current = HashSet::from([prefix.to_string()]);+ for entry in entries+ .iter()+ .filter(|entry| !is_catch_up_internal_path(&entry.path))+ {+ current.insert(entry.path.clone());+ if entry.info.is_dir {+ ensure_directory(backup, &entry.path, entry.info.mode).await?;+ } else {+ copy_file_state(primary, backup, &entry.path).await?;+ }+ }++ remove_stale_paths(backup, prefix, ¤t).await+}++/// Reconcile one exact path from the primary's current state.+pub async fn reconcile_file(+ primary: &dyn FileSystem,+ backup: &dyn FileSystem,+ path: &str,+) -> Result<()> {+ if is_catch_up_internal_path(path) {+ return Ok(());+ }+ match primary.stat(path).await {+ Ok(info) if info.is_dir => reconcile_prefix(primary, backup, path).await,+ Ok(_) => copy_file_state(primary, backup, path).await,+ Err(Error::NotFound(_)) => remove_if_present(backup, path).await,+ Err(error) => Err(error),+ }+}++/// Remove stale business paths while preserving internal files and their ancestors.+async fn remove_stale_paths(+ backup: &dyn FileSystem,+ prefix: &str,+ current: &HashSet<String>,+) -> Result<()> {+ match backup.stat(prefix).await {+ Ok(info) if !info.is_dir => {+ return if current.contains(prefix) || is_multiwrite_internal_path(prefix) {+ Ok(())+ } else {+ remove_if_present(backup, prefix).await+ };+ }+ Ok(_) => {}+ Err(Error::NotFound(_)) => return Ok(()),+ Err(error) => return Err(error),+ }+ let entries = match backup+ .tree_directory(prefix, true, None, None, None, None, None, false)+ .await+ {+ Ok(entries) => entries,+ Err(Error::NotFound(_)) => return Ok(()),+ Err(error) => return Err(error),+ };+ let protected = entries+ .iter()+ .filter(|entry| is_catch_up_internal_path(&entry.path))+ .flat_map(|entry| Path::new(&entry.path).ancestors().skip(1))+ .filter_map(Path::to_str)+ .map(str::to_string)+ .collect::<HashSet<_>>();+ let mut stale = std::iter::once(prefix.to_string())+ .chain(entries.into_iter().map(|entry| entry.path))+ .filter(|path| {+ !is_catch_up_internal_path(path) && !current.contains(path) && !protected.contains(path)+ })+ .collect::<Vec<_>>();+ stale.sort_by_key(|path| std::cmp::Reverse(path.matches('/').count()));+ for path in stale {+ remove_if_present(backup, &path).await?;+ }+ Ok(())+}++/// Recognize internal paths in either logical or mount-relative path space.+fn is_catch_up_internal_path(path: &str) -> bool {+ is_multiwrite_internal_path(path)+ || is_multiwrite_internal_path(&format!(+ "{MULTIWRITE_MOUNT_PREFIX}/{}",+ path.trim_start_matches('/')+ ))+}++/// Ensure one backup path is a directory with the requested mode.+async fn ensure_directory(backup: &dyn FileSystem, path: &str, mode: u32) -> Result<()> {+ match backup.stat(path).await {+ Ok(info) if info.is_dir => {}+ Ok(_) => {+ backup.remove(path).await?;+ backup.mkdir(path, mode).await?;+ }+ Err(Error::NotFound(_)) => {+ backup.ensure_parent_dirs(path, 0o755).await?;+ backup.mkdir(path, mode).await?;+ }+ Err(error) => return Err(error),+ }+ Ok(())+}++/// Replace one backup file with the primary bytes.+async fn copy_file_state(+ primary: &dyn FileSystem,+ backup: &dyn FileSystem,+ path: &str,+) -> Result<()> {+ if backup.stat(path).await.is_ok_and(|current| current.is_dir) {+ backup.remove_all(path).await?;+ }+ let bytes = match primary.read(path, 0, 0).await {+ Ok(bytes) => bytes,+ Err(Error::NotFound(_)) => return remove_if_present(backup, path).await,+ Err(error) => return Err(error),+ };+ backup.ensure_parent_dirs(path, 0o755).await?;+ backup.write(path, &bytes, 0, WriteFlag::Create).await?;+ Ok(())+}++/// Remove one backup path while accepting an already converged absence.+async fn remove_if_present(backup: &dyn FileSystem, path: &str) -> Result<()> {+ let result = match backup.stat(path).await {+ Ok(info) if info.is_dir => backup.remove_all(path).await,+ Ok(_) => backup.remove(path).await,+ Err(Error::NotFound(_)) => return Ok(()),+ Err(error) => return Err(error),+ };+ match result {+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),+ Err(error) => Err(error),+ }+}diff --git a/crates/ragfs/src/multibackend/checkpoint.rs b/crates/ragfs/src/multibackend/checkpoint.rsnew file mode 100644index 000000000..85f0987b3--- /dev/null+++ b/crates/ragfs/src/multibackend/checkpoint.rs@@ -0,0 +1,1751 @@+//! Checkpoint construction, validation, publication, and reading.++use std::collections::{BTreeMap, BTreeSet};+use std::sync::Arc;+use std::time::{Duration, SystemTime, UNIX_EPOCH};++use async_trait::async_trait;+use chrono::{DateTime, Utc};+use tokio::sync::watch;+use tracing::warn;++use crate::core::context::{FsContextInner, FS_CTX};+use crate::core::errors::{Error, Result};+use crate::core::filesystem::FileSystem;+use crate::core::internal_names::is_multiwrite_internal_path;+use crate::multibackend::catch_up::{CheckpointConsumer, CheckpointReadResult, CheckpointSnapshot};+use crate::multibackend::codec::{decode_checkpoint_chunk, encode_checkpoint_chunk, sha256_hex};+use crate::multibackend::constants::{+ CHECKPOINT_CHUNK_FILE_EXTENSION, CHECKPOINT_CHUNK_FILE_PREFIX, CHECKPOINT_DIR_PREFIX,+ CHECKPOINT_DIR_TIME_FORMAT, MANIFEST_FILE, MAX_CHECKPOINT_NODES, MULTIWRITE_MOUNT_PREFIX,+};+use crate::multibackend::gc::MetadataGc;+use crate::multibackend::meta::{MetadataStore, MultiWriteWorker};+use crate::multibackend::model::{+ is_checkpoint_directory_name, CheckpointManifest, CheckpointNode, CheckpointsManifest,+ ChunkDescriptor, DirectoryOperation, FileState, LatestCheckpoint, PartitionsManifest,+ ProtocolStatus, ScopeKey, ScopedSeq, SegmentEventType, SegmentRecord,+};+use crate::multibackend::provider::MultiWriteProvider;+use crate::multibackend::router::AccountRouter;++/// Periodically builds checkpoints for every Stable account partition.+pub struct CheckpointWorker {+ store: Arc<MetadataStore>,+ builder: CheckpointBuilder,+ gc: MetadataGc,+}+impl CheckpointWorker {+ /// Create a worker over the production primary, store, and provider.+ pub fn new(+ primary: Arc<dyn FileSystem>,+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+ ) -> Self {+ let builder = CheckpointBuilder::new(primary, store.clone(), provider.clone());+ Self {+ gc: MetadataGc::new(store.clone(), provider),+ builder,+ store,+ }+ }++ /// Run checkpoint rounds at the configured interval until cancellation.+ pub async fn run(&self, interval: Duration, mut cancellation: watch::Receiver<bool>) {+ let mut ticker = tokio::time::interval(interval);+ ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);+ loop {+ tokio::select! {+ _ = ticker.tick() => if let Err(error) = self.run_once(interval).await {+ warn!(error = %error, "multi-write checkpoint discovery failed");+ },+ changed = cancellation.changed() => {+ if changed.is_err() || *cancellation.borrow() {+ break;+ }+ }+ }+ }+ }++ /// Build one checkpoint round while isolating account and scope failures.+ pub async fn run_once(&self, interval: Duration) -> Result<()> {+ let result = self.run_once_inner(interval).await;+ if result.is_err() {+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);+ }+ result+ }++ /// Execute one checkpoint round and return discovery failures.+ async fn run_once_inner(&self, interval: Duration) -> Result<()> {+ if self.store.protocol_status().await? != ProtocolStatus::Stable {+ return Ok(());+ }+ let now = now_ns()?;+ let interval_ns = u64::try_from(interval.as_nanos()).unwrap_or(u64::MAX);+ for account in self.store.initialized_accounts().await? {+ let path = self.store.paths().account_manifest(&account)?.0;+ let manifest: PartitionsManifest = match self.store.read_json(&path).await {+ Ok(manifest) => manifest,+ Err(Error::NotFound(_)) => continue,+ Err(error) => {+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);+ warn!(account = %account, error = %error,+ "multi-write checkpoint account metadata unavailable");+ continue;+ }+ };+ if let Err(error) = manifest.validate() {+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);+ warn!(account = %account, error = %error,+ "multi-write checkpoint account metadata invalid");+ continue;+ }+ for (&partition_id, partition) in &manifest.partitions {+ if partition.state != crate::multibackend::model::PartitionState::Stable {+ continue;+ }+ let scope = ScopeKey {+ account_id: account.clone(),+ partition_id,+ epoch: manifest.epoch,+ };+ let pointer_path = self+ .store+ .paths()+ .checkpoints_manifest(&account, partition_id)?+ .0;+ let checkpoints: CheckpointsManifest =+ match self.store.read_json(&pointer_path).await {+ Ok(checkpoints) => checkpoints,+ Err(Error::NotFound(_)) => continue,+ Err(error) => {+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);+ warn!(account = %account, partition = partition_id, error = %error,+ "multi-write checkpoint pointer unavailable");+ continue;+ }+ };+ if let Err(error) = checkpoints.validate() {+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);+ warn!(account = %account, partition = partition_id, error = %error,+ "multi-write checkpoint pointer invalid");+ continue;+ }+ if checkpoints+ .latest_checkpoint+ .as_ref()+ .is_some_and(|latest| now < latest.updated_at_ns.saturating_add(interval_ns))+ {+ continue;+ }+ match self.builder.build_scope(&scope).await {+ Ok(CheckpointBuildResult::Published(_)) => {+ if let Err(error) = self.gc.collect_scope(&scope).await {+ warn!(account = %account, partition = partition_id, error = %error,+ "multi-write metadata garbage collection failed");+ }+ }+ Ok(_) => {}+ Err(Error::NotFound(_)) => {}+ Err(error) => {+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);+ warn!(account = %account, partition = partition_id, error = %error,+ "multi-write checkpoint build failed");+ }+ }+ }+ }+ Ok(())+ }+}++/// Result of one bounded checkpoint build attempt.+pub enum CheckpointBuildResult {+ /// No new sealed sequence exists for this scope.+ NoSealedChanges,+ /// A validated checkpoint became the latest publication.+ Published(LatestCheckpoint),+ /// Another builder published first and this candidate was discarded.+ Retry,+}++#[derive(Default)]+struct SegmentDelta {+ states: BTreeMap<String, FileState>,+ exact_paths: BTreeSet<String>,+ prefixes: BTreeSet<String>,+}++impl SegmentDelta {+ /// Return whether this segment delta cannot change checkpoint contents.+ fn is_empty(&self) -> bool {+ self.states.is_empty() && self.exact_paths.is_empty() && self.prefixes.is_empty()+ }+}++enum ChunkMergeTarget<'a> {+ All,+ Range(&'a str, Option<&'a str>),+ RootOnly(&'a str),+}++/// Builds immutable checkpoints from sealed records and current primary state.+pub struct CheckpointBuilder {+ primary: Arc<dyn FileSystem>,+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+ router: AccountRouter,+}+impl CheckpointBuilder {+ /// Create a builder over the encrypted primary and raw metadata store.+ pub fn new(+ primary: Arc<dyn FileSystem>,+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+ ) -> Self {+ Self {+ primary,+ router: AccountRouter::new(store.clone()),+ store,+ provider,+ }+ }++ /// Build and compare-before-publish one Stable partition checkpoint.+ pub async fn build_scope(&self, scope: &ScopeKey) -> Result<CheckpointBuildResult> {+ scope.validate()?;+ let pointer_path = self+ .store+ .paths()+ .checkpoints_manifest(&scope.account_id, scope.partition_id)?+ .0;+ let lease = self+ .store+ .pathlock_manager()+ .acquire_exact(+ &pointer_path,+ self.store+ .pathlock_manager()+ .default_lock_timeout()+ .max(Duration::from_secs(5)),+ None,+ )+ .await?;+ let read_pointer = async {+ let start: CheckpointsManifest = self.store.read_json(&pointer_path).await?;+ start.validate()?;+ Ok(start)+ }+ .await;+ let release = self+ .store+ .pathlock_manager()+ .release(&lease)+ .await+ .map_err(Error::from);+ let start = finish_with_release(read_pointer, release)?;+ let segments = self.provider.read_manifest(scope).await?;+ segments.validate()?;+ let Some(tail) = segments+ .segments+ .iter()+ .filter_map(|segment| segment.segment_to_seq)+ .last()+ else {+ return Ok(CheckpointBuildResult::NoSealedChanges);+ };+ let old_to = start+ .latest_checkpoint+ .as_ref()+ .map_or(0, |latest| latest.checkpoint_to_seq);+ if tail <= old_to {+ return Ok(CheckpointBuildResult::NoSealedChanges);+ }+ let min_synced = segments+ .backend_states+ .values()+ .map(|state| state.synced_seq)+ .min();+ let created_at_ns = now_ns()?;+ let candidate = checkpoint_candidate_name(created_at_ns)?;+ let directory = self+ .store+ .paths()+ .checkpoints_dir(&scope.account_id, scope.partition_id)?+ .0;+ let candidate_path = format!("{directory}/{}", candidate.trim_end_matches('/'));+ let candidate_lease = self+ .store+ .pathlock_manager()+ .acquire_exact(+ &candidate_path,+ self.store+ .pathlock_manager()+ .default_lock_timeout()+ .max(Duration::from_secs(5)),+ None,+ )+ .await?;+ let operation = async {+ let mut manifest = self+ .prepare_candidate_checkpoint(scope, &start, &candidate, created_at_ns, old_to)+ .await?;+ let mut chunk_index = checkpoint_chunk_count(&manifest.root);+ for descriptor in segments+ .segments+ .iter()+ .filter(|descriptor| descriptor.segment_to_seq.is_some_and(|to| to > old_to))+ {+ let records = self.provider.read_sealed_segment(scope, descriptor).await?;+ let records = records+ .into_iter()+ .filter(|record| record.seq > old_to && record.seq <= tail)+ .collect::<Vec<_>>();+ if records.is_empty() {+ continue;+ }+ let delta = self.segment_delta(scope, records).await?;+ self.apply_checkpoint_delta(+ scope,+ &candidate,+ &mut manifest,+ &delta,+ &mut chunk_index,+ )+ .await?;+ }+ self.prune_checkpoint_tombstones(+ scope,+ &candidate,+ &mut manifest,+ min_synced,+ &mut chunk_index,+ )+ .await?;+ manifest.version = 1;+ manifest.partition_id = scope.partition_id;+ manifest.epoch = scope.epoch;+ manifest.checkpoint_from_seq = old_to + 1;+ manifest.checkpoint_to_seq = tail;+ manifest.created_at_ns = created_at_ns;+ manifest.checksum = tree_checksum(&manifest.root)?;+ manifest.validate_for_scope(scope)?;+ let manifest_bytes = serde_json::to_vec(&manifest)?;+ self.store+ .write_bytes(+ &format!("{directory}/{candidate}{MANIFEST_FILE}"),+ &manifest_bytes,+ )+ .await?;+ verify_checkpoint(self.store.as_ref(), scope, &manifest, &candidate, tail).await?;+ let latest = LatestCheckpoint {+ path: candidate,+ checkpoint_to_seq: tail,+ updated_at_ns: created_at_ns,+ };+ let lease = self+ .store+ .pathlock_manager()+ .acquire_exact(+ &pointer_path,+ self.store+ .pathlock_manager()+ .default_lock_timeout()+ .max(Duration::from_secs(5)),+ None,+ )+ .await?;+ let publish = async {+ let current: CheckpointsManifest = self.store.read_json(&pointer_path).await?;+ if current.latest_checkpoint != start.latest_checkpoint {+ return Ok(CheckpointBuildResult::Retry);+ }+ self.store+ .publish_json(+ &pointer_path,+ &CheckpointsManifest {+ version: 1,+ latest_checkpoint: Some(latest.clone()),+ },+ CheckpointsManifest::validate,+ )+ .await?;+ Ok(CheckpointBuildResult::Published(latest))+ }+ .await;+ let release = self+ .store+ .pathlock_manager()+ .release(&lease)+ .await+ .map_err(Error::from);+ finish_with_release(publish, release)+ }+ .await;+ let release = self+ .store+ .pathlock_manager()+ .release(&candidate_lease)+ .await+ .map_err(Error::from);+ finish_with_release(operation, release)+ }++ /// Prepare a candidate checkpoint directory from the latest checkpoint or an empty base.+ async fn prepare_candidate_checkpoint(+ &self,+ scope: &ScopeKey,+ start: &CheckpointsManifest,+ candidate: &str,+ created_at_ns: u64,+ old_to: u64,+ ) -> Result<CheckpointManifest> {+ if let Some(latest) = &start.latest_checkpoint {+ let manifest = read_manifest(self.store.as_ref(), scope, &latest.path).await?;+ if manifest.checkpoint_to_seq != latest.checkpoint_to_seq {+ return Err(Error::Serialization(+ "checkpoint pointer sequence mismatch".into(),+ ));+ }+ let manifest = copy_checkpoint_files(+ self.store.as_ref(),+ scope,+ &latest.path,+ candidate,+ &manifest,+ )+ .await?;+ return Ok(manifest);+ }++ let mut chunk_index = 0;+ let (root, chunks) = encode_states_with_index(scope, Vec::new(), &mut chunk_index)?;+ let checksum = tree_checksum(&root)?;+ let manifest = CheckpointManifest {+ version: 1,+ partition_id: scope.partition_id,+ epoch: scope.epoch,+ checkpoint_from_seq: 1,+ checkpoint_to_seq: old_to,+ created_at_ns,+ root,+ checksum,+ };+ manifest.validate_for_scope(scope)?;+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;+ let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?;+ self.store+ .write_bytes(+ &format!("{directory}/{MANIFEST_FILE}"),+ &serde_json::to_vec(&manifest)?,+ )+ .await?;+ Ok(manifest)+ }++ /// Convert one sealed segment into checkpoint path replacements and prefix invalidations.+ async fn segment_delta(+ &self,+ scope: &ScopeKey,+ records: Vec<SegmentRecord>,+ ) -> Result<SegmentDelta> {+ let mut delta = SegmentDelta::default();+ for record in records {+ match record.event_type {+ SegmentEventType::Write | SegmentEventType::Remove => {+ let backend_path = self+ .store+ .paths()+ .backend_path(&scope.account_id, &record.path)?;+ let deleted = match FS_CTX+ .scope(+ Arc::new(+ FsContextInner::new(&scope.account_id)+ .with_bypass_cache(true)+ .with_auto_pathlock_disabled(),+ ),+ self.primary.stat(&backend_path),+ )+ .await+ {+ Ok(_) => None,+ Err(Error::NotFound(_)) => Some(true),+ Err(error) => return Err(error),+ };+ delta.exact_paths.insert(record.path.clone());+ delta.states.insert(+ record.path.clone(),+ FileState {+ path: record.path,+ latest_seq: ScopedSeq {+ scope: scope.clone(),+ seq: record.seq,+ },+ deleted,+ },+ );+ }+ SegmentEventType::RemoveTree | SegmentEventType::MoveTree => {+ let mut prefixes = vec![record.path.clone()];+ if let Some(destination) = record.destination_path {+ prefixes.push(destination);+ }+ for prefix in prefixes {+ delta.prefixes.insert(prefix.clone());+ delta+ .states+ .retain(|path, _| !is_path_within(path, &prefix));+ for path in self.scan_prefix(scope, &prefix).await? {+ delta.states.insert(+ path.clone(),+ FileState {+ path,+ latest_seq: ScopedSeq {+ scope: scope.clone(),+ seq: record.seq,+ },+ deleted: None,+ },+ );+ }+ }+ }+ SegmentEventType::DeleteScope => {}+ }+ }+ Ok(delta)+ }++ /// Apply one segment delta to the candidate manifest by rewriting only needed chunks.+ async fn apply_checkpoint_delta(+ &self,+ scope: &ScopeKey,+ candidate: &str,+ manifest: &mut CheckpointManifest,+ delta: &SegmentDelta,+ chunk_index: &mut usize,+ ) -> Result<()> {+ if delta.is_empty() {+ return Ok(());+ }+ if manifest.root.chunks.is_empty() {+ self.rewrite_full_checkpoint_chunk(scope, candidate, manifest, delta, chunk_index)+ .await?;+ } else {+ self.rewrite_split_checkpoint_chunks(scope, candidate, manifest, delta, chunk_index)+ .await?;+ }+ manifest.root.validate()?;+ Ok(())+ }++ /// Rewrite a checkpoint whose complete state still lives in the root chunk.+ async fn rewrite_full_checkpoint_chunk(+ &self,+ scope: &ScopeKey,+ candidate: &str,+ manifest: &mut CheckpointManifest,+ delta: &SegmentDelta,+ chunk_index: &mut usize,+ ) -> Result<()> {+ let old_root = manifest.root.clone();+ let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?;+ let states =+ read_checkpoint_chunk_states(self.store.as_ref(), scope, &directory, &old_root, true)+ .await?;+ let mut remaining = delta.states.clone();+ let states = merge_checkpoint_states(states, delta, &mut remaining, ChunkMergeTarget::All);+ let (root, chunks) = encode_states_with_index(scope, states, chunk_index)?;+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;+ manifest.root = root;+ remove_replaced_checkpoint_files(+ self.store.as_ref(),+ scope,+ candidate,+ checkpoint_chunk_files(&old_root),+ &manifest.root,+ )+ .await;+ Ok(())+ }++ /// Rewrite affected child chunks and insert new states not covered by old ranges.+ async fn rewrite_split_checkpoint_chunks(+ &self,+ scope: &ScopeKey,+ candidate: &str,+ manifest: &mut CheckpointManifest,+ delta: &SegmentDelta,+ chunk_index: &mut usize,+ ) -> Result<()> {+ let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?;+ let account_root = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id);+ let mut remaining = delta.states.clone();++ if checkpoint_state_is_dirty(&account_root, delta) || remaining.contains_key(&account_root)+ {+ let old_root = manifest.root.clone();+ let children = std::mem::take(&mut manifest.root.chunks);+ let states = read_checkpoint_chunk_states(+ self.store.as_ref(),+ scope,+ &directory,+ &old_root,+ true,+ )+ .await?;+ let states = merge_checkpoint_states(+ states,+ delta,+ &mut remaining,+ ChunkMergeTarget::RootOnly(&account_root),+ );+ let (mut root, chunks) = encode_states_with_index(scope, states, chunk_index)?;+ root.chunks = children;+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;+ manifest.root = root;+ remove_replaced_checkpoint_files(+ self.store.as_ref(),+ scope,+ candidate,+ vec![old_root.file],+ &manifest.root,+ )+ .await;+ }++ let mut next_children = Vec::new();+ let children = std::mem::take(&mut manifest.root.chunks);+ for descriptor in children {+ if checkpoint_descriptor_is_dirty(&descriptor, delta)? {+ let old_files = checkpoint_chunk_files(&descriptor);+ let states = read_checkpoint_chunk_states(+ self.store.as_ref(),+ scope,+ &directory,+ &descriptor,+ false,+ )+ .await?;+ let (start, end) = checkpoint_descriptor_range(&descriptor)?;+ let states = merge_checkpoint_states(+ states,+ delta,+ &mut remaining,+ ChunkMergeTarget::Range(start, end),+ );+ let (mut descriptors, chunks) =+ encode_child_states_with_index(scope, states, start, end, chunk_index)?;+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;+ let replacement_root = ChunkDescriptor {+ file: String::new(),+ root_path: account_root.clone(),+ start_path: None,+ end_path: None,+ file_state_count: 0,+ checksum: String::new(),+ chunks: descriptors.clone(),+ };+ remove_replaced_checkpoint_files(+ self.store.as_ref(),+ scope,+ candidate,+ old_files,+ &replacement_root,+ )+ .await;+ next_children.append(&mut descriptors);+ } else {+ next_children.push(descriptor);+ }+ }++ if !remaining.is_empty() {+ return Err(Error::Serialization(+ "checkpoint delta paths were not assigned to a chunk".into(),+ ));+ }+ sort_checkpoint_children(&mut next_children, &account_root)?;+ manifest.root.chunks = next_children;+ Ok(())+ }++ /// Remove checkpoint tombstones that no current backend still needs.+ async fn prune_checkpoint_tombstones(+ &self,+ scope: &ScopeKey,+ candidate: &str,+ manifest: &mut CheckpointManifest,+ min_synced: Option<u64>,+ chunk_index: &mut usize,+ ) -> Result<()> {+ if manifest.root.chunks.is_empty() {+ let old_root = manifest.root.clone();+ let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?;+ let states = read_checkpoint_chunk_states(+ self.store.as_ref(),+ scope,+ &directory,+ &old_root,+ true,+ )+ .await?;+ let retained = retain_checkpoint_states(states.clone(), min_synced);+ if retained != states {+ let (root, chunks) = encode_states_with_index(scope, retained, chunk_index)?;+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;+ manifest.root = root;+ remove_replaced_checkpoint_files(+ self.store.as_ref(),+ scope,+ candidate,+ checkpoint_chunk_files(&old_root),+ &manifest.root,+ )+ .await;+ }+ return Ok(());+ }++ let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?;+ let account_root = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id);+ let old_root = manifest.root.clone();+ let children = std::mem::take(&mut manifest.root.chunks);+ let states =+ read_checkpoint_chunk_states(self.store.as_ref(), scope, &directory, &old_root, true)+ .await?;+ let retained = retain_checkpoint_states(states.clone(), min_synced);+ if retained != states {+ let (mut root, chunks) = encode_states_with_index(scope, retained, chunk_index)?;+ root.chunks = children;+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;+ manifest.root = root;+ remove_replaced_checkpoint_files(+ self.store.as_ref(),+ scope,+ candidate,+ vec![old_root.file],+ &manifest.root,+ )+ .await;+ } else {+ manifest.root.chunks = children;+ }++ let mut next_children = Vec::new();+ for descriptor in std::mem::take(&mut manifest.root.chunks) {+ let old_files = checkpoint_chunk_files(&descriptor);+ let states = read_checkpoint_chunk_states(+ self.store.as_ref(),+ scope,+ &directory,+ &descriptor,+ false,+ )+ .await?;+ let retained = retain_checkpoint_states(states.clone(), min_synced);+ if retained == states {+ next_children.push(descriptor);+ continue;+ }+ let (start, end) = checkpoint_descriptor_range(&descriptor)?;+ let (mut descriptors, chunks) =+ encode_child_states_with_index(scope, retained, start, end, chunk_index)?;+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;+ let replacement_root = ChunkDescriptor {+ file: String::new(),+ root_path: account_root.clone(),+ start_path: None,+ end_path: None,+ file_state_count: 0,+ checksum: String::new(),+ chunks: descriptors.clone(),+ };+ remove_replaced_checkpoint_files(+ self.store.as_ref(),+ scope,+ candidate,+ old_files,+ &replacement_root,+ )+ .await;+ next_children.append(&mut descriptors);+ }+ sort_checkpoint_children(&mut next_children, &account_root)?;+ manifest.root.chunks = next_children;+ Ok(())+ }++ /// Scan one current primary prefix and retain paths owned by this scope.+ async fn scan_prefix(&self, scope: &ScopeKey, logical_prefix: &str) -> Result<Vec<String>> {+ let backend_prefix = self+ .store+ .paths()+ .backend_path(&scope.account_id, logical_prefix)?;+ let context = Arc::new(+ FsContextInner::new(&scope.account_id)+ .with_bypass_cache(true)+ .with_auto_pathlock_disabled(),+ );+ let paths = FS_CTX+ .scope(context, async {+ let root = match self.primary.stat(&backend_prefix).await {+ Ok(root) => root,+ Err(Error::NotFound(_)) => return Ok(Vec::new()),+ Err(error) => return Err(error),+ };+ let mut paths = vec![backend_prefix.clone()];+ if root.is_dir {+ paths.extend(+ self.primary+ .tree_directory(+ &backend_prefix,+ true,+ None,+ None,+ None,+ None,+ None,+ false,+ )+ .await?+ .into_iter()+ .map(|entry| entry.path),+ );+ }+ Ok(paths)+ })+ .await?;+ let mut selected = Vec::new();+ for path in paths {+ let logical = format!("{MULTIWRITE_MOUNT_PREFIX}{path}");+ if !is_multiwrite_internal_path(&logical)+ && self.router.route(&scope.account_id, &logical).await?.scope == *scope+ {+ selected.push(logical);+ }+ }+ selected.sort();+ selected.dedup();+ Ok(selected)+ }+}++/// Reads complete validated checkpoint snapshots under the publication lease.+pub struct CheckpointReader {+ store: Arc<MetadataStore>,+}+impl CheckpointReader {+ /// Create a reader over the raw metadata store.+ pub fn new(store: Arc<MetadataStore>) -> Self {+ Self { store }+ }+}++#[async_trait]+impl CheckpointConsumer for CheckpointReader {+ /// Read the latest checkpoint or return a typed retry for immutable races.+ async fn read_latest(+ &self,+ scope: &ScopeKey,+ synced_seq: u64,+ tail_seq: u64,+ ) -> Result<CheckpointReadResult> {+ let pointer_path = self+ .store+ .paths()+ .checkpoints_manifest(&scope.account_id, scope.partition_id)?+ .0;+ let lease = self+ .store+ .pathlock_manager()+ .acquire_exact(+ &pointer_path,+ self.store+ .pathlock_manager()+ .default_lock_timeout()+ .max(Duration::from_secs(5)),+ None,+ )+ .await?;+ let pointer = async {+ let pointer: CheckpointsManifest = self.store.read_json(&pointer_path).await?;+ pointer.validate()?;+ let Some(latest) = pointer.latest_checkpoint else {+ return Ok(None);+ };+ if latest.checkpoint_to_seq <= synced_seq || latest.checkpoint_to_seq > tail_seq {+ return Ok(None);+ }+ Ok(Some(latest))+ }+ .await;+ let release = self+ .store+ .pathlock_manager()+ .release(&lease)+ .await+ .map_err(Error::from);+ let latest = finish_with_release(pointer, release)?;+ let Some(latest) = latest else {+ return Ok(CheckpointReadResult::NoCheckpoint);+ };+ let states = match read_published(self.store.as_ref(), scope, &latest).await {+ Ok(states) => states,+ Err(Error::NotFound(_)) | Err(Error::Serialization(_)) => {+ return Ok(CheckpointReadResult::RetryLatestCheckpoint)+ }+ Err(error) => return Err(error),+ };+ let account_manifest_path = self.store.paths().account_manifest(&scope.account_id)?.0;+ let account_manifest: PartitionsManifest =+ self.store.read_json(&account_manifest_path).await?;+ account_manifest.validate()?;+ let mut directory_prefixes = BTreeSet::new();+ for event in account_manifest.directory_events {+ if event.positions.iter().any(|position| {+ position.partition_id == scope.partition_id+ && position.epoch == scope.epoch+ && position.seq <= latest.checkpoint_to_seq+ }) {+ directory_prefixes.insert(event.source_path);+ if event.operation == DirectoryOperation::MoveTree {+ directory_prefixes.insert(event.destination_path.ok_or_else(|| {+ Error::Serialization("move directory event lacks destination".into())+ })?);+ }+ }+ }+ Ok(CheckpointReadResult::Snapshot(CheckpointSnapshot {+ checkpoint_to_seq: latest.checkpoint_to_seq,+ file_states: states,+ directory_prefixes: directory_prefixes.into_iter().collect(),+ }))+ }+}++/// Encode a deterministic checkpoint tree while continuing the caller's chunk index.+fn encode_states_with_index(+ scope: &ScopeKey,+ mut states: Vec<FileState>,+ index: &mut usize,+) -> Result<(ChunkDescriptor, Vec<(String, Vec<u8>)>)> {+ let (node, root_path) = checkpoint_node_from_states(scope, &mut states)?;+ encode_checkpoint_tree(node, &root_path, index)+}++/// Encode states as one or more non-root checkpoint descriptors.+fn encode_child_states_with_index(+ scope: &ScopeKey,+ mut states: Vec<FileState>,+ start_path: &str,+ end_path: Option<&str>,+ index: &mut usize,+) -> Result<(Vec<ChunkDescriptor>, Vec<(String, Vec<u8>)>)> {+ if states.is_empty() {+ return Ok((Vec::new(), Vec::new()));+ }+ let (mut node, root_path) = checkpoint_node_from_states(scope, &mut states)?;+ let mut files = Vec::new();+ let mut descriptors = Vec::new();+ let children = std::mem::take(&mut node.children);+ if node.latest_seq.is_some() {+ descriptors.push(encode_node(node, &root_path, false, index, &mut files)?);+ }+ partition_children(children, &root_path, index, &mut files, &mut descriptors)?;+ assign_checkpoint_child_ranges(&mut descriptors, start_path, end_path)?;+ Ok((descriptors, files))+}++/// Build the account-relative checkpoint radix tree from sorted file states.+fn checkpoint_node_from_states(+ scope: &ScopeKey,+ states: &mut Vec<FileState>,+) -> Result<(CheckpointNode, String)> {+ for state in states.iter() {+ state.validate(scope)?;+ }+ states.sort_by(|left, right| left.path.cmp(&right.path));+ if states.windows(2).any(|pair| pair[0].path == pair[1].path) {+ return Err(Error::Serialization(+ "checkpoint contains duplicate paths".into(),+ ));+ }+ let root_path = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id);+ let mut node = CheckpointNode {+ path_fragment: String::new(),+ latest_seq: None,+ deleted: None,+ children: Vec::new(),+ };+ for state in states.iter() {+ if !is_canonical_account_path(&state.path, &root_path) {+ return Err(Error::Serialization(+ "checkpoint path crosses account root".into(),+ ));+ }+ let relative = state+ .path+ .strip_prefix(&root_path)+ .ok_or_else(|| Error::Serialization("checkpoint path crosses account root".into()))?;+ insert_state(&mut node, relative, state)?;+ }+ Ok((node, root_path))+}++/// Encode one preconstructed radix tree through the production chunk splitter.+fn encode_checkpoint_tree(+ mut node: CheckpointNode,+ root_path: &str,+ index: &mut usize,+) -> Result<(ChunkDescriptor, Vec<(String, Vec<u8>)>)> {+ node.validate()?;+ let mut files = Vec::new();+ let root = if checkpoint_node_count(&node) <= MAX_CHECKPOINT_NODES {+ encode_node(node, root_path, true, index, &mut files)?+ } else {+ let children = std::mem::take(&mut node.children);+ let mut descriptor = encode_node(node, root_path, true, index, &mut files)?;+ partition_children(+ children,+ root_path,+ index,+ &mut files,+ &mut descriptor.chunks,+ )?;+ assign_checkpoint_child_ranges(&mut descriptor.chunks, root_path, None)?;+ descriptor+ };+ Ok((root, files))+}++/// Insert one file state into the account-relative component radix tree.+fn insert_state(root: &mut CheckpointNode, relative: &str, state: &FileState) -> Result<()> {+ let mut node = root;+ let mut remaining = relative;+ while !remaining.is_empty() {+ let Some(index) = node+ .children+ .iter()+ .position(|child| common_prefix_len(&child.path_fragment, remaining) > 0)+ else {+ node.children.push(CheckpointNode {+ path_fragment: remaining.to_string(),+ latest_seq: Some(state.latest_seq.seq),+ deleted: state.deleted,+ children: Vec::new(),+ });+ node.children+ .sort_by(|left, right| left.path_fragment.cmp(&right.path_fragment));+ return Ok(());+ };+ let common = common_prefix_len(&node.children[index].path_fragment, remaining);+ if common == node.children[index].path_fragment.len() {+ remaining = &remaining[common..];+ node = &mut node.children[index];+ } else {+ let mut old = node.children.remove(index);+ let suffix = old.path_fragment.split_off(common);+ let prefix = std::mem::replace(&mut old.path_fragment, suffix);+ let mut parent = CheckpointNode {+ path_fragment: prefix,+ latest_seq: None,+ deleted: None,+ children: vec![old],+ };+ remaining = &remaining[common..];+ if remaining.is_empty() {+ parent.latest_seq = Some(state.latest_seq.seq);+ parent.deleted = state.deleted;+ } else {+ parent.children.push(CheckpointNode {+ path_fragment: remaining.to_string(),+ latest_seq: Some(state.latest_seq.seq),+ deleted: state.deleted,+ children: Vec::new(),+ });+ parent+ .children+ .sort_by(|left, right| left.path_fragment.cmp(&right.path_fragment));+ }+ node.children.insert(index, parent);+ return Ok(());+ }+ }+ if node.latest_seq.is_some() {+ return Err(Error::Serialization(+ "checkpoint contains duplicate paths".into(),+ ));+ }+ node.latest_seq = Some(state.latest_seq.seq);+ node.deleted = state.deleted;+ Ok(())+}++/// Return the longest UTF-8-safe common prefix length.+fn common_prefix_len(left: &str, right: &str) -> usize {+ left.char_indices()+ .zip(right.char_indices())+ .take_while(|((_, left), (_, right))| left == right)+ .last()+ .map_or(0, |((index, ch), _)| index + ch.len_utf8())+}++/// Split ordered direct child ranges into bounded immutable chunks.+fn partition_children(+ children: Vec<CheckpointNode>,+ parent_path: &str,+ index: &mut usize,+ files: &mut Vec<(String, Vec<u8>)>,+ descriptors: &mut Vec<ChunkDescriptor>,+) -> Result<()> {+ let mut group = Vec::new();+ let mut group_nodes = 1;+ for mut child in children {+ let child_nodes = checkpoint_node_count(&child);+ if child_nodes + 1 > MAX_CHECKPOINT_NODES {+ flush_child_group(&mut group, parent_path, index, files, descriptors)?;+ let fragment = child.path_fragment.clone();+ let mut grandchildren = std::mem::take(&mut child.children);+ if child.latest_seq.is_some() {+ descriptors.push(encode_node(child, parent_path, false, index, files)?);+ }+ for grandchild in &mut grandchildren {+ grandchild.path_fragment.insert_str(0, &fragment);+ }+ partition_children(grandchildren, parent_path, index, files, descriptors)?;+ } else {+ if group_nodes + child_nodes > MAX_CHECKPOINT_NODES {+ flush_child_group(&mut group, parent_path, index, files, descriptors)?;+ group_nodes = 1;+ }+ group_nodes += child_nodes;+ group.push(child);+ }+ }+ flush_child_group(&mut group, parent_path, index, files, descriptors)+}++/// Encode one non-empty direct child range as a synthetic-root chunk.+fn flush_child_group(+ children: &mut Vec<CheckpointNode>,+ parent_path: &str,+ index: &mut usize,+ files: &mut Vec<(String, Vec<u8>)>,+ descriptors: &mut Vec<ChunkDescriptor>,+) -> Result<()> {+ if children.is_empty() {+ return Ok(());+ }+ let node = CheckpointNode {+ path_fragment: String::new(),+ latest_seq: None,+ deleted: None,+ children: std::mem::take(children),+ };+ descriptors.push(encode_node(node, parent_path, false, index, files)?);+ Ok(())+}++/// Encode one bounded radix subtree and derive its exact path range.+fn encode_node(+ node: CheckpointNode,+ root_path: &str,+ root: bool,+ index: &mut usize,+ files: &mut Vec<(String, Vec<u8>)>,+) -> Result<ChunkDescriptor> {+ let (count, first_path) = checkpoint_node_range(&node, root_path);+ let start_path = if root {+ None+ } else {+ Some(+ first_path+ .or_else(|| checkpoint_node_start_path(&node, root_path))+ .ok_or_else(|| {+ Error::Serialization("checkpoint child chunk lacks start path".into())+ })?,+ )+ };+ let bytes = encode_checkpoint_chunk(&node)?;+ let checksum = sha256_hex(&bytes);+ let file = checkpoint_chunk_file_name(*index, &checksum);+ *index += 1;+ files.push((file.clone(), bytes));+ Ok(ChunkDescriptor {+ file,+ root_path: root_path.to_string(),+ start_path,+ end_path: None,+ file_state_count: count as u32,+ checksum,+ chunks: Vec::new(),+ })+}++/// Count nodes in one radix subtree.+fn checkpoint_node_count(root: &CheckpointNode) -> usize {+ let mut nodes = vec![root];+ let mut count = 0;+ while let Some(node) = nodes.pop() {+ count += 1;+ nodes.extend(&node.children);+ }+ count+}++/// Return the state count and first ordered path encoded in one chunk.+fn checkpoint_node_range(root: &CheckpointNode, root_path: &str) -> (usize, Option<String>) {+ let mut paths = Vec::new();+ collect_node_paths(root, root_path, &mut paths);+ (paths.len(), paths.first().cloned())+}++/// Return the lowest owned path prefix for one encoded checkpoint subtree.+fn checkpoint_node_start_path(node: &CheckpointNode, root_path: &str) -> Option<String> {+ let path = format!("{root_path}{}", node.path_fragment);+ if node.latest_seq.is_some() || !node.path_fragment.is_empty() {+ return Some(path);+ }+ node.children+ .first()+ .map(|child| format!("{path}{}", child.path_fragment))+}++/// Assign contiguous left-closed and right-open ownership ranges to siblings.+fn assign_checkpoint_child_ranges(+ children: &mut Vec<ChunkDescriptor>,+ start_path: &str,+ end_path: Option<&str>,+) -> Result<()> {+ if children.is_empty() {+ return Ok(());+ }+ children.sort_by(|left, right| left.start_path.cmp(&right.start_path));+ let starts = children+ .iter()+ .map(|child| {+ child.start_path.clone().ok_or_else(|| {+ Error::Serialization("checkpoint child chunk lacks start path".into())+ })+ })+ .collect::<Result<Vec<_>>>()?;+ for index in 0..children.len() {+ children[index].start_path = Some(if index == 0 {+ start_path.to_string()+ } else {+ starts[index].clone()+ });+ children[index].end_path = if index + 1 < starts.len() {+ Some(starts[index + 1].clone())+ } else {+ end_path.map(str::to_string)+ };+ }+ Ok(())+}++/// Collect state paths from one radix subtree in canonical order.+fn collect_node_paths(node: &CheckpointNode, parent: &str, paths: &mut Vec<String>) {+ let path = format!("{parent}{}", node.path_fragment);+ if node.latest_seq.is_some() {+ paths.push(path.clone());+ }+ for child in &node.children {+ collect_node_paths(child, &path, paths);+ }+}++/// Read and validate one published checkpoint reference.+async fn read_published(+ store: &MetadataStore,+ scope: &ScopeKey,+ latest: &LatestCheckpoint,+) -> Result<Vec<FileState>> {+ let manifest = read_manifest(store, scope, &latest.path).await?;+ if manifest.checkpoint_to_seq != latest.checkpoint_to_seq {+ return Err(Error::Serialization(+ "checkpoint pointer sequence mismatch".into(),+ ));+ }+ read_chunks(store, scope, &manifest, &latest.path).await+}++/// Read and validate one immutable checkpoint manifest.+async fn read_manifest(+ store: &MetadataStore,+ scope: &ScopeKey,+ candidate: &str,+) -> Result<CheckpointManifest> {+ let directory = checkpoint_directory(store, scope, candidate)?;+ let manifest: CheckpointManifest = store+ .read_json(&format!("{directory}/{MANIFEST_FILE}"))+ .await?;+ manifest.validate_for_scope(scope)?;+ if tree_checksum(&manifest.root)? != manifest.checksum {+ return Err(Error::Serialization(+ "checkpoint tree checksum mismatch".into(),+ ));+ }+ Ok(manifest)+}++/// Build the second-resolution checkpoint directory name used by new publications.+fn checkpoint_candidate_name(created_at_ns: u64) -> Result<String> {+ let seconds = i64::try_from(created_at_ns / 1_000_000_000)+ .map_err(|_| Error::internal("checkpoint timestamp overflow"))?;+ let datetime = DateTime::<Utc>::from_timestamp(seconds, 0)+ .ok_or_else(|| Error::internal("checkpoint timestamp is out of range"))?;+ Ok(format!(+ "{CHECKPOINT_DIR_PREFIX}{}/",+ datetime.format(CHECKPOINT_DIR_TIME_FORMAT)+ ))+}++/// Build a checkpoint chunk filename without embedding any sequence number.+fn checkpoint_chunk_file_name(index: usize, checksum: &str) -> String {+ format!("{CHECKPOINT_CHUNK_FILE_PREFIX}{index:06}-{checksum}{CHECKPOINT_CHUNK_FILE_EXTENSION}")+}++/// Rename descriptor file references while recording old-to-new chunk copies.+fn rename_checkpoint_chunk_files(+ descriptor: &mut ChunkDescriptor,+ index: &mut usize,+ copies: &mut Vec<(String, String, String)>,+) {+ let source = descriptor.file.clone();+ let candidate = checkpoint_chunk_file_name(*index, &descriptor.checksum);+ *index += 1;+ descriptor.file = candidate.clone();+ copies.push((source, candidate, descriptor.checksum.clone()));+ for child in &mut descriptor.chunks {+ rename_checkpoint_chunk_files(child, index, copies);+ }+}++/// Copy a published checkpoint manifest and referenced chunks into a candidate directory.+async fn copy_checkpoint_files(+ store: &MetadataStore,+ scope: &ScopeKey,+ source: &str,+ candidate: &str,+ manifest: &CheckpointManifest,+) -> Result<CheckpointManifest> {+ let source_directory = checkpoint_directory(store, scope, source)?;+ let candidate_directory = checkpoint_directory(store, scope, candidate)?;+ let mut candidate_manifest = manifest.clone();+ let mut copies = Vec::new();+ let mut index = 0;+ rename_checkpoint_chunk_files(&mut candidate_manifest.root, &mut index, &mut copies);+ candidate_manifest.checksum = tree_checksum(&candidate_manifest.root)?;+ candidate_manifest.validate_for_scope(scope)?;+ for (source_file, candidate_file, checksum) in copies {+ let bytes = store+ .read_bytes(&format!("{source_directory}/{source_file}"))+ .await?;+ if sha256_hex(&bytes) != checksum {+ return Err(Error::Serialization(+ "checkpoint chunk checksum mismatch".into(),+ ));+ }+ store+ .write_bytes(&format!("{candidate_directory}/{candidate_file}"), &bytes)+ .await?;+ }+ store+ .write_bytes(+ &format!("{candidate_directory}/{MANIFEST_FILE}"),+ &serde_json::to_vec(&candidate_manifest)?,+ )+ .await?;+ Ok(candidate_manifest)+}++/// Write checkpoint chunk bytes into one candidate directory.+async fn write_checkpoint_chunks(+ store: &MetadataStore,+ scope: &ScopeKey,+ candidate: &str,+ chunks: &[(String, Vec<u8>)],+) -> Result<()> {+ let directory = checkpoint_directory(store, scope, candidate)?;+ for (file, bytes) in chunks {+ store+ .write_bytes(&format!("{directory}/{file}"), bytes)+ .await?;+ }+ Ok(())+}++/// Read and validate one checkpoint chunk without reading its child descriptors.+async fn read_checkpoint_chunk_states(+ store: &MetadataStore,+ scope: &ScopeKey,+ directory: &str,+ descriptor: &ChunkDescriptor,+ root: bool,+) -> Result<Vec<FileState>> {+ let account_root = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id);+ if descriptor.root_path != account_root {+ return Err(Error::Serialization(+ "checkpoint descriptor root mismatch".into(),+ ));+ }+ let bytes = store+ .read_bytes(&format!("{directory}/{}", descriptor.file))+ .await?;+ if sha256_hex(&bytes) != descriptor.checksum {+ return Err(Error::Serialization(+ "checkpoint chunk checksum mismatch".into(),+ ));+ }+ let node = decode_checkpoint_chunk(&bytes)?;+ if encode_checkpoint_chunk(&node)? != bytes {+ return Err(Error::Serialization(+ "checkpoint chunk is not canonical".into(),+ ));+ }+ let mut states = Vec::new();+ flatten_node(scope, &node, &descriptor.root_path, true, &mut states)?;+ if states.len() != descriptor.file_state_count as usize {+ return Err(Error::Serialization(+ "checkpoint chunk state count mismatch".into(),+ ));+ }+ if states.windows(2).any(|pair| pair[0].path >= pair[1].path)+ || states+ .iter()+ .any(|state| !is_canonical_account_path(&state.path, &account_root))+ {+ return Err(Error::Serialization(+ "checkpoint chunk paths are invalid or unordered".into(),+ ));+ }+ if !root {+ let (start, end) = checkpoint_descriptor_range(descriptor)?;+ if states+ .iter()+ .any(|state| !checkpoint_range_contains(start, end, &state.path))+ {+ return Err(Error::Serialization(+ "checkpoint chunk range mismatch".into(),+ ));+ }+ }+ Ok(states)+}++/// Merge old chunk states with replacement states that belong to one target range.+fn merge_checkpoint_states(+ states: Vec<FileState>,+ delta: &SegmentDelta,+ remaining: &mut BTreeMap<String, FileState>,+ target: ChunkMergeTarget<'_>,+) -> Vec<FileState> {+ let mut merged = states+ .into_iter()+ .filter(|state| !checkpoint_state_is_dirty(&state.path, delta))+ .map(|state| (state.path.clone(), state))+ .collect::<BTreeMap<_, _>>();+ let selected = remaining+ .keys()+ .filter(|path| checkpoint_target_contains(&target, path))+ .cloned()+ .collect::<Vec<_>>();+ for path in selected {+ if let Some(state) = remaining.remove(&path) {+ merged.insert(path, state);+ }+ }+ merged.into_values().collect()+}++/// Return whether one merge target owns a logical checkpoint path.+fn checkpoint_target_contains(target: &ChunkMergeTarget<'_>, path: &str) -> bool {+ match target {+ ChunkMergeTarget::All => true,+ ChunkMergeTarget::Range(start, end) => checkpoint_range_contains(start, *end, path),+ ChunkMergeTarget::RootOnly(root) => path == *root,+ }+}++/// Return whether a checkpoint state path is replaced or invalidated by a delta.+fn checkpoint_state_is_dirty(path: &str, delta: &SegmentDelta) -> bool {+ delta.exact_paths.contains(path)+ || delta+ .prefixes+ .iter()+ .any(|prefix| is_path_within(path, prefix))+}++/// Return whether a non-root chunk range intersects one segment delta.+fn checkpoint_descriptor_is_dirty(+ descriptor: &ChunkDescriptor,+ delta: &SegmentDelta,+) -> Result<bool> {+ let (start, end) = checkpoint_descriptor_range(descriptor)?;+ Ok(delta+ .exact_paths+ .iter()+ .any(|path| checkpoint_range_contains(start, end, path))+ || delta+ .states+ .keys()+ .any(|path| checkpoint_range_contains(start, end, path))+ || delta+ .prefixes+ .iter()+ .any(|prefix| checkpoint_range_overlaps_prefix(start, end, prefix)))+}++/// Return the validated left-closed and right-open range for a non-root descriptor.+fn checkpoint_descriptor_range(descriptor: &ChunkDescriptor) -> Result<(&str, Option<&str>)> {+ let start = descriptor+ .start_path+ .as_deref()+ .ok_or_else(|| Error::Serialization("checkpoint child chunk lacks start path".into()))?;+ let end = descriptor.end_path.as_deref();+ if let Some(end) = end {+ if start >= end {+ return Err(Error::Serialization(+ "checkpoint chunk range is reversed".into(),+ ));+ }+ }+ Ok((start, end))+}++/// Return whether a left-closed range contains one logical path.+fn checkpoint_range_contains(start: &str, end: Option<&str>, path: &str) -> bool {+ start <= path && end.is_none_or(|end| path < end)+}++/// Return whether one ownership range can contain paths under a dirty prefix.+fn checkpoint_range_overlaps_prefix(start: &str, end: Option<&str>, prefix: &str) -> bool {+ checkpoint_range_contains(start, end, prefix) || is_path_within(start, prefix)+}++/// Sort child descriptors and normalize contiguous ownership ranges.+fn sort_checkpoint_children(children: &mut Vec<ChunkDescriptor>, root_path: &str) -> Result<()> {+ assign_checkpoint_child_ranges(children, root_path, None)+}++/// Retain live states and tombstones still needed by at least one backend.+fn retain_checkpoint_states(states: Vec<FileState>, min_synced: Option<u64>) -> Vec<FileState> {+ states+ .into_iter()+ .filter(|state| {+ state.deleted.is_none()+ || min_synced.is_some_and(|synced| state.latest_seq.seq >= synced)+ })+ .collect()+}++/// Return every chunk descriptor reachable from a root descriptor.+fn checkpoint_chunk_descriptors(root: &ChunkDescriptor) -> Vec<&ChunkDescriptor> {+ let mut descriptors = Vec::new();+ let mut stack = vec![root];+ while let Some(descriptor) = stack.pop() {+ descriptors.push(descriptor);+ stack.extend(descriptor.chunks.iter());+ }+ descriptors+}++/// Return every non-empty chunk filename reachable from one descriptor tree.+fn checkpoint_chunk_files(root: &ChunkDescriptor) -> Vec<String> {+ checkpoint_chunk_descriptors(root)+ .into_iter()+ .filter_map(|descriptor| (!descriptor.file.is_empty()).then(|| descriptor.file.clone()))+ .collect()+}++/// Count chunk files already referenced by one descriptor tree.+fn checkpoint_chunk_count(root: &ChunkDescriptor) -> usize {+ checkpoint_chunk_files(root).len()+}++/// Best-effort delete candidate chunk files no longer referenced after a rewrite.+async fn remove_replaced_checkpoint_files(+ store: &MetadataStore,+ scope: &ScopeKey,+ candidate: &str,+ old_files: Vec<String>,+ new_root: &ChunkDescriptor,+) {+ let Ok(directory) = checkpoint_directory(store, scope, candidate) else {+ return;+ };+ let retained = checkpoint_chunk_files(new_root)+ .into_iter()+ .collect::<BTreeSet<_>>();+ for file in old_files {+ if retained.contains(&file) {+ continue;+ }+ if let Err(error) = store.remove_file(&format!("{directory}/{file}")).await {+ warn!(file = %file, error = %error, "checkpoint chunk cleanup failed");+ }+ }+}++/// Stream-validate a candidate checkpoint without materializing all file states.+async fn verify_checkpoint(+ store: &MetadataStore,+ scope: &ScopeKey,+ manifest: &CheckpointManifest,+ candidate: &str,+ tail: u64,+) -> Result<()> {+ if tree_checksum(&manifest.root)? != manifest.checksum {+ return Err(Error::Serialization(+ "checkpoint tree checksum mismatch".into(),+ ));+ }+ let directory = checkpoint_directory(store, scope, candidate)?;+ let mut descriptors = vec![(&manifest.root, true)];+ let mut previous_path: Option<String> = None;+ while let Some((descriptor, root)) = descriptors.pop() {+ let states =+ read_checkpoint_chunk_states(store, scope, &directory, descriptor, root).await?;+ for state in states {+ if state.latest_seq.seq > tail+ || previous_path+ .as_ref()+ .is_some_and(|previous| previous >= &state.path)+ {+ return Err(Error::Serialization(+ "checkpoint candidate verification failed".into(),+ ));+ }+ previous_path = Some(state.path);+ }+ descriptors.extend(descriptor.chunks.iter().rev().map(|child| (child, false)));+ }+ Ok(())+}++/// Read every referenced chunk and reconstruct sorted file states.+async fn read_chunks(+ store: &MetadataStore,+ scope: &ScopeKey,+ manifest: &CheckpointManifest,+ candidate: &str,+) -> Result<Vec<FileState>> {+ let directory = checkpoint_directory(store, scope, candidate)?;+ let mut descriptors = vec![(&manifest.root, true)];+ let mut states = Vec::new();+ while let Some((descriptor, root)) = descriptors.pop() {+ states.extend(+ read_checkpoint_chunk_states(store, scope, &directory, descriptor, root).await?,+ );+ descriptors.extend(descriptor.chunks.iter().rev().map(|child| (child, false)));+ }+ if states.windows(2).any(|pair| pair[0].path >= pair[1].path) {+ return Err(Error::Serialization(+ "checkpoint paths are not strictly ordered".into(),+ ));+ }+ Ok(states)+}++/// Flatten one decoded checkpoint node into validated file states.+fn flatten_node(+ scope: &ScopeKey,+ node: &CheckpointNode,+ parent: &str,+ root: bool,+ states: &mut Vec<FileState>,+) -> Result<()> {+ if (!root || !node.path_fragment.is_empty()) && !valid_path_fragment(&node.path_fragment) {+ return Err(Error::Serialization(+ "checkpoint path fragment is not relative".into(),+ ));+ }+ let path = format!("{parent}{}", node.path_fragment);+ if let Some(seq) = node.latest_seq {+ let state = FileState {+ path: path.clone(),+ latest_seq: ScopedSeq {+ scope: scope.clone(),+ seq,+ },+ deleted: node.deleted,+ };+ state.validate(scope)?;+ states.push(state);+ }+ for child in &node.children {+ flatten_node(scope, child, &path, false, states)?;+ }+ Ok(())+}++/// Resolve one isolated checkpoint directory from its relative latest pointer.+fn checkpoint_directory(+ store: &MetadataStore,+ scope: &ScopeKey,+ candidate: &str,+) -> Result<String> {+ let name = candidate.strip_suffix('/').ok_or_else(|| {+ Error::Serialization("checkpoint pointer must reference a directory".into())+ })?;+ if !is_checkpoint_directory_name(name) {+ return Err(Error::Serialization(+ "checkpoint pointer directory is invalid".into(),+ ));+ }+ let root = store+ .paths()+ .checkpoints_dir(&scope.account_id, scope.partition_id)?+ .0;+ Ok(format!("{root}/{name}"))+}++/// Return whether one radix edge is a non-empty relative fragment.+fn valid_path_fragment(fragment: &str) -> bool {+ !fragment.is_empty() && !fragment.contains('\0')+}++/// Return whether a path is canonical and belongs to one account root.+fn is_canonical_account_path(path: &str, root: &str) -> bool {+ is_path_within(path, root)+ && !path.ends_with('/')+ && path+ .split('/')+ .skip(1)+ .all(|part| !part.is_empty() && part != "." && part != "..")+}++/// Hash the canonical serialized descriptor tree.+fn tree_checksum(root: &ChunkDescriptor) -> Result<String> {+ Ok(sha256_hex(&serde_json::to_vec(root)?))+}++/// Return whether a path equals a prefix or is one of its descendants.+fn is_path_within(path: &str, prefix: &str) -> bool {+ path == prefix+ || path+ .strip_prefix(prefix)+ .is_some_and(|suffix| suffix.starts_with('/'))+}++/// Return the current Unix timestamp in nanoseconds.+fn now_ns() -> Result<u64> {+ let nanos = SystemTime::now()+ .duration_since(UNIX_EPOCH)+ .map_err(|error| Error::internal(error.to_string()))?+ .as_nanos();+ u64::try_from(nanos).map_err(|_| Error::internal("checkpoint timestamp overflow"))+}++/// Preserve an operation result while requiring lease release to succeed.+fn finish_with_release<T>(operation: Result<T>, release: Result<()>) -> Result<T> {+ match (operation, release) {+ (Ok(value), Ok(())) => Ok(value),+ (Err(error), _) => Err(error),+ (Ok(_), Err(error)) => Err(error),+ }+}diff --git a/crates/ragfs/src/multibackend/codec.rs b/crates/ragfs/src/multibackend/codec.rsnew file mode 100644index 000000000..bd0e527a5--- /dev/null+++ b/crates/ragfs/src/multibackend/codec.rs@@ -0,0 +1,359 @@+//! Stable binary codecs for V2 segment and checkpoint files.++use sha2::{Digest, Sha256};+use uuid::Uuid;++use crate::core::errors::{Error, Result};+use crate::multibackend::constants::{MAX_CHECKPOINT_NODES, MAX_SEGMENT_RECORDS};+use crate::multibackend::model::{CheckpointNode, DecodedSegment, SegmentEventType, SegmentRecord};++const SEGMENT_MAGIC: &[u8; 4] = b"OVSG";+const CHECKPOINT_MAGIC: &[u8; 4] = b"OVCP";+const FORMAT_VERSION: u16 = 1;+const MAX_STRING_LEN: usize = 1024 * 1024;+const MIN_RECORD_LEN: usize = 14;+const MAX_RECORD_LEN: usize = 2 * 1024 * 1024;+const MAX_CHECKPOINT_DEPTH: usize = 256;++/// Encode segment records in the stable OVSG binary format.+pub fn encode_segment(records: &[SegmentRecord]) -> Result<Vec<u8>> {+ if records.len() > MAX_SEGMENT_RECORDS {+ return Err(serialization("segment has too many records"));+ }++ let mut bytes = Vec::new();+ bytes.extend_from_slice(SEGMENT_MAGIC);+ bytes.extend_from_slice(&FORMAT_VERSION.to_le_bytes());+ for record in records {+ record.validate()?;+ let frame = encode_record(record)?;+ if frame.len() > MAX_RECORD_LEN {+ return Err(serialization("segment record is too large"));+ }+ bytes.extend_from_slice(&(frame.len() as u32).to_le_bytes());+ bytes.extend_from_slice(&frame);+ }+ Ok(bytes)+}++/// Decode complete segment records and recover a truncated final frame.+pub fn decode_segment(bytes: &[u8]) -> Result<DecodedSegment> {+ let mut reader = Reader::new(bytes);+ read_header(&mut reader, SEGMENT_MAGIC)?;+ let mut records = Vec::new();+ let mut valid_bytes = reader.position();+ let mut truncated_tail = false;++ while reader.remaining() > 0 {+ if records.len() == MAX_SEGMENT_RECORDS {+ return Err(serialization("segment has too many records"));+ }+ let frame_start = reader.position();+ if reader.remaining() < 4 {+ truncated_tail = true;+ break;+ }+ let frame_len = reader.read_u32()? as usize;+ if frame_len < MIN_RECORD_LEN {+ return Err(serialization("segment record is too small"));+ }+ if frame_len > MAX_RECORD_LEN {+ return Err(serialization("segment record is too large"));+ }+ if frame_len > reader.remaining() {+ truncated_tail = true;+ break;+ }+ let mut frame = Reader::new(reader.read_exact(frame_len)?);+ let record = decode_record(&mut frame)?;+ if frame.remaining() != 0 {+ return Err(serialization("segment record has trailing bytes"));+ }+ records.push(record);+ valid_bytes = reader.position();+ debug_assert!(valid_bytes > frame_start);+ }++ Ok(DecodedSegment {+ records,+ valid_bytes: valid_bytes as u64,+ truncated_tail,+ })+}++/// Encode one checkpoint tree in the stable OVCP binary format.+pub fn encode_checkpoint_chunk(root: &CheckpointNode) -> Result<Vec<u8>> {+ let node_count = validate_checkpoint_shape(root)?;+ root.validate()?;++ let mut bytes = Vec::new();+ bytes.extend_from_slice(CHECKPOINT_MAGIC);+ bytes.extend_from_slice(&FORMAT_VERSION.to_le_bytes());+ bytes.extend_from_slice(&(node_count as u32).to_le_bytes());+ encode_checkpoint_node(root, &mut bytes)?;+ Ok(bytes)+}++/// Decode one complete OVCP checkpoint tree.+pub fn decode_checkpoint_chunk(bytes: &[u8]) -> Result<CheckpointNode> {+ let mut reader = Reader::new(bytes);+ read_header(&mut reader, CHECKPOINT_MAGIC)?;+ let node_count = reader.read_u32()? as usize;+ if node_count == 0 || node_count > MAX_CHECKPOINT_NODES {+ return Err(serialization("invalid checkpoint node count"));+ }++ let mut remaining_nodes = node_count;+ let root = decode_checkpoint_node(&mut reader, &mut remaining_nodes, 1)?;+ if remaining_nodes != 0 {+ return Err(serialization(+ "checkpoint node count does not match payload",+ ));+ }+ if reader.remaining() != 0 {+ return Err(serialization("checkpoint has trailing bytes"));+ }+ root.validate()?;+ Ok(root)+}++/// Return a lowercase hexadecimal SHA-256 digest.+pub fn sha256_hex(bytes: &[u8]) -> String {+ format!("{:x}", Sha256::digest(bytes))+}++/// Encode one validated segment record payload.+fn encode_record(record: &SegmentRecord) -> Result<Vec<u8>> {+ let mut bytes = Vec::new();+ bytes.extend_from_slice(&record.seq.to_le_bytes());+ bytes.push(record.event_type as u8);+ let flags =+ u8::from(record.op_id.is_some()) | (u8::from(record.destination_path.is_some()) << 1);+ bytes.push(flags);+ write_string(&mut bytes, &record.path)?;+ if let Some(op_id) = record.op_id {+ bytes.extend_from_slice(op_id.as_bytes());+ }+ if let Some(destination) = &record.destination_path {+ write_string(&mut bytes, destination)?;+ }+ Ok(bytes)+}++/// Decode and validate one complete segment record payload.+fn decode_record(reader: &mut Reader<'_>) -> Result<SegmentRecord> {+ let seq = reader.read_u64()?;+ let event_type = match reader.read_u8()? {+ 1 => SegmentEventType::Write,+ 2 => SegmentEventType::Remove,+ 3 => SegmentEventType::RemoveTree,+ 4 => SegmentEventType::MoveTree,+ 5 => SegmentEventType::DeleteScope,+ _ => return Err(serialization("unknown segment event type")),+ };+ let flags = reader.read_u8()?;+ if flags & !0b11 != 0 {+ return Err(serialization("unknown segment record flags"));+ }+ let path = reader.read_string()?;+ let op_id = if flags & 1 != 0 {+ let raw: [u8; 16] = reader+ .read_exact(16)?+ .try_into()+ .map_err(|_| serialization("invalid operation id"))?;+ Some(Uuid::from_bytes(raw))+ } else {+ None+ };+ let destination_path = if flags & 2 != 0 {+ Some(reader.read_string()?)+ } else {+ None+ };+ let record = SegmentRecord {+ seq,+ event_type,+ path,+ op_id,+ destination_path,+ };+ record.validate()?;+ Ok(record)+}++/// Validate checkpoint size and depth without recursive traversal.+fn validate_checkpoint_shape(root: &CheckpointNode) -> Result<usize> {+ let mut stack = vec![(root, 1_usize)];+ let mut node_count = 0_usize;+ while let Some((node, depth)) = stack.pop() {+ if depth > MAX_CHECKPOINT_DEPTH {+ return Err(serialization("checkpoint exceeds maximum depth"));+ }+ node_count += 1;+ if node_count > MAX_CHECKPOINT_NODES {+ return Err(serialization("checkpoint has too many nodes"));+ }+ if node.path_fragment.len() > MAX_STRING_LEN {+ return Err(serialization("checkpoint path fragment is too large"));+ }+ stack.extend(node.children.iter().map(|child| (child, depth + 1)));+ }+ Ok(node_count)+}++/// Encode one checkpoint node after iterative shape validation.+fn encode_checkpoint_node(node: &CheckpointNode, bytes: &mut Vec<u8>) -> Result<()> {+ write_string(bytes, &node.path_fragment)?;+ let flags = u8::from(node.latest_seq.is_some()) | (u8::from(node.deleted.is_some()) << 1);+ bytes.push(flags);+ if let Some(seq) = node.latest_seq {+ bytes.extend_from_slice(&seq.to_le_bytes());+ }+ bytes.extend_from_slice(&(node.children.len() as u32).to_le_bytes());+ for child in &node.children {+ encode_checkpoint_node(child, bytes)?;+ }+ Ok(())+}++/// Decode one node while enforcing global node and depth budgets before allocation.+fn decode_checkpoint_node(+ reader: &mut Reader<'_>,+ remaining_nodes: &mut usize,+ depth: usize,+) -> Result<CheckpointNode> {+ if depth > MAX_CHECKPOINT_DEPTH {+ return Err(serialization("checkpoint exceeds maximum depth"));+ }+ if *remaining_nodes == 0 {+ return Err(serialization("checkpoint contains too many nodes"));+ }+ *remaining_nodes -= 1;++ let path_fragment = reader.read_string()?;+ let flags = reader.read_u8()?;+ if flags & !0b11 != 0 {+ return Err(serialization("unknown checkpoint node flags"));+ }+ let latest_seq = if flags & 1 != 0 {+ Some(reader.read_u64()?)+ } else {+ None+ };+ let deleted = if flags & 2 != 0 { Some(true) } else { None };+ let child_count = reader.read_u32()? as usize;+ if child_count > *remaining_nodes {+ return Err(serialization(+ "checkpoint child count exceeds remaining node budget",+ ));+ }++ let mut children = Vec::new();+ for _ in 0..child_count {+ children.push(decode_checkpoint_node(reader, remaining_nodes, depth + 1)?);+ }+ Ok(CheckpointNode {+ path_fragment,+ latest_seq,+ deleted,+ children,+ })+}++/// Append one bounded length-prefixed UTF-8 string.+fn write_string(bytes: &mut Vec<u8>, value: &str) -> Result<()> {+ if value.len() > MAX_STRING_LEN {+ return Err(serialization("string is too large"));+ }+ bytes.extend_from_slice(&(value.len() as u32).to_le_bytes());+ bytes.extend_from_slice(value.as_bytes());+ Ok(())+}++/// Read and validate a format magic and version.+fn read_header(reader: &mut Reader<'_>, magic: &[u8; 4]) -> Result<()> {+ if reader.read_exact(4)? != magic {+ return Err(serialization("invalid binary format magic"));+ }+ if reader.read_u16()? != FORMAT_VERSION {+ return Err(serialization("unsupported binary format version"));+ }+ Ok(())+}++/// Build a serialization error with a concise message.+fn serialization(message: impl Into<String>) -> Error {+ Error::Serialization(message.into())+}++struct Reader<'a> {+ bytes: &'a [u8],+ position: usize,+}++impl<'a> Reader<'a> {+ /// Create a reader positioned at the start of a byte slice.+ fn new(bytes: &'a [u8]) -> Self {+ Self { bytes, position: 0 }+ }++ /// Return the current byte offset.+ fn position(&self) -> usize {+ self.position+ }++ /// Return the number of unread bytes.+ fn remaining(&self) -> usize {+ self.bytes.len() - self.position+ }++ /// Read an exact byte range or reject truncated input.+ fn read_exact(&mut self, length: usize) -> Result<&'a [u8]> {+ let end = self+ .position+ .checked_add(length)+ .ok_or_else(|| serialization("binary length overflow"))?;+ let value = self+ .bytes+ .get(self.position..end)+ .ok_or_else(|| serialization("truncated binary data"))?;+ self.position = end;+ Ok(value)+ }++ /// Read one unsigned byte.+ fn read_u8(&mut self) -> Result<u8> {+ Ok(self.read_exact(1)?[0])+ }++ /// Read one little-endian u16.+ fn read_u16(&mut self) -> Result<u16> {+ Ok(u16::from_le_bytes(self.read_exact(2)?.try_into().unwrap()))+ }++ /// Read one little-endian u32.+ fn read_u32(&mut self) -> Result<u32> {+ Ok(u32::from_le_bytes(self.read_exact(4)?.try_into().unwrap()))+ }++ /// Read one little-endian u64.+ fn read_u64(&mut self) -> Result<u64> {+ Ok(u64::from_le_bytes(self.read_exact(8)?.try_into().unwrap()))+ }++ /// Read one bounded length-prefixed UTF-8 string.+ fn read_string(&mut self) -> Result<String> {+ let length = self.read_u32()? as usize;+ if length > MAX_STRING_LEN {+ return Err(serialization("string is too large"));+ }+ let raw = self.read_exact(length)?;+ let value =+ std::str::from_utf8(raw).map_err(|_| serialization("string is not valid UTF-8"))?;+ Ok(value.to_owned())+ }+}++#[cfg(test)]+#[path = "tests/test_codec.rs"]+mod tests;diff --git a/crates/ragfs/src/multibackend/config.rs b/crates/ragfs/src/multibackend/config.rsindex be305f73c..f2d0f581c 100644--- a/crates/ragfs/src/multibackend/config.rs+++ b/crates/ragfs/src/multibackend/config.rs@@ -1,12 +1,11 @@//! Multi-backend config validation and normalization helpers.-use std::collections::HashMap;+use std::collections::{HashMap, HashSet};use serde_json::Value;use crate::core::errors::{Error, Result};-use crate::core::multibackend_wrapper::{BackendEntry, SyncMode};-use crate::core::types::{BackendsConfig, ConfigValue, PluginConfig, RedirectPolicy};+use crate::core::types::{BackendsConfig, ConfigValue, PluginConfig};/// Convert one nested backup `params` object into plugin config values.pub fn item_params_to_config_values(value: &Value) -> Result<HashMap<String, ConfigValue>> {@@ -66,62 +65,32 @@ pub fn validate_primary_encryption_flags(Ok(())}-/// Validate redirect targets against the assembled backup entries.-pub fn validate_redirect_targets(- redirects: &[RedirectPolicy],- backup_entries: &[BackendEntry],-) -> Result<()> {- for policy in redirects {- let targets = match policy {- RedirectPolicy::FileOverSizePolicy { target, .. } => target.as_ref(),- RedirectPolicy::FileExtensionPolicy { target, .. } => target.as_ref(),- };- let target_names = targets- .ok_or_else(|| Error::config("redirect policy target must not be empty".to_string()))?;- if target_names.is_empty() {+/// Validate V2 backup settings.+pub fn validate_backups_config(bc: &BackendsConfig) -> Result<()> {+ if !matches!(bc.provider.as_str(), "filesystem" | "cache") {+ return Err(Error::config(+ "backups.provider must be 'filesystem' or 'cache'".to_string(),+ ));+ }++ let mut names = HashSet::new();+ for item in &bc.items {+ let name = item.name.trim();+ if name.is_empty() {+ return Err(Error::config("backup name must not be empty".to_string()));+ }+ if name == "primary" {return Err(Error::config(- "redirect policy target must not be empty".to_string(),+ "backup backend name 'primary' is reserved".to_string(),));}- for name in target_names {- if !backup_entries.iter().any(|be| &be.name == name) {- return Err(Error::config(format!(- "redirect target '{}' not found in backup entries",- name- )));- }+ if !names.insert(name) {+ return Err(Error::config(format!(+ "duplicate backup name '{}'",+ item.name+ )));}}- Ok(())-}-/// Validate that exclude policies do not silently carry redirect targets.-pub fn validate_backup_excludes(bc: &BackendsConfig) -> Result<()> {- for item in &bc.items {- for policy in item.excludes.as_deref().unwrap_or(&[]) {- match policy {- RedirectPolicy::FileOverSizePolicy { target, .. }- | RedirectPolicy::FileExtensionPolicy { target, .. } => {- if target.is_some() {- return Err(Error::config(format!(- "exclude policy for backup '{}' must not contain target",- item.name- )));- }- }- }- }- }Ok(())}--/// Convert config sync settings into the runtime sync mode.-pub fn sync_mode_from_config(bc: &BackendsConfig) -> SyncMode {- match bc.sync_type.as_str() {- "sync" => SyncMode::Sync {- ack_count: bc.write_ack_count.unwrap_or(usize::MAX),- timeout_ms: bc.write_ack_timeout_ms.unwrap_or(0),- },- _ => SyncMode::Async,- }-}diff --git a/crates/ragfs/src/multibackend/constants.rs b/crates/ragfs/src/multibackend/constants.rsnew file mode 100644index 000000000..160f71154--- /dev/null+++ b/crates/ragfs/src/multibackend/constants.rs@@ -0,0 +1,38 @@+//! Shared constants for V2 multi-write metadata.++/// Canonical logical mount prefix used by V2 multi-write metadata.+pub(crate) const MULTIWRITE_MOUNT_PREFIX: &str = "/local";+/// Internal metadata directory name.+pub(crate) const SYSTEM_DIR: &str = "_system";+/// Mount-level protocol manifest file name.+pub(crate) const MULTIWRITE_PROTOCOL_FILE: &str = ".multiwrite.json";+/// Account partition directory name.+pub(crate) const PARTITIONS_DIR: &str = "partitions";+/// Standard manifest file name.+pub(crate) const MANIFEST_FILE: &str = "manifest.json";+/// Segment blob directory name.+pub(crate) const SEGMENTS_DIR: &str = "segments";+/// Checkpoint directory name.+pub(crate) const CHECKPOINTS_DIR: &str = "checkpoints";++/// Mutable head segment file prefix.+pub(crate) const HEAD_SEGMENT_FILE_PREFIX: &str = "head-";+/// Immutable sealed segment file prefix.+pub(crate) const SEALED_SEGMENT_FILE_PREFIX: &str = "segment-";+/// Segment blob file extension.+pub(crate) const SEGMENT_FILE_EXTENSION: &str = ".ovsg";+/// Checkpoint candidate directory prefix.+pub(crate) const CHECKPOINT_DIR_PREFIX: &str = "cp-";+/// Checkpoint candidate timestamp format.+pub(crate) const CHECKPOINT_DIR_TIME_FORMAT: &str = "%Y%m%d%H%M%S";+/// Checkpoint chunk file prefix.+pub(crate) const CHECKPOINT_CHUNK_FILE_PREFIX: &str = "chunk-";+/// Checkpoint chunk file extension.+pub(crate) const CHECKPOINT_CHUNK_FILE_EXTENSION: &str = ".ovcp";++/// Total number of fixed routing buckets.+pub const VBUCKETS: u32 = 65_536;+/// Maximum records allowed in one segment.+pub(crate) const MAX_SEGMENT_RECORDS: usize = 8_192;+/// Maximum file states allowed in one checkpoint chunk.+pub(crate) const MAX_CHECKPOINT_NODES: usize = 500_000;\ No newline at end of filediff --git a/crates/ragfs/src/multibackend/factory.rs b/crates/ragfs/src/multibackend/factory.rsindex 0f657d850..da9d8d4bc 100644--- a/crates/ragfs/src/multibackend/factory.rs+++ b/crates/ragfs/src/multibackend/factory.rs@@ -1,6 +1,6 @@//! Multi-backend runtime assembly from validated mount configuration.-use std::collections::{HashMap, HashSet};+use std::collections::HashMap;use std::sync::Arc;use std::time::Duration;@@ -13,10 +13,12 @@ use crate::core::multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS};use crate::core::plugin::ServicePlugin;use crate::core::types::{BackendRole, BackendsConfig, ConfigValue, PluginConfig};use crate::multibackend::config::{- item_params_to_config_values, sync_mode_from_config, validate_backup_excludes,- validate_primary_encryption_flags, validate_redirect_targets,+ item_params_to_config_values, validate_backups_config, validate_primary_encryption_flags,};-use crate::multibackend::meta::RelativePathFsContextResolver;+use crate::multibackend::meta::{MetadataStore, RelativePathFsContextResolver};+#[cfg(feature = "cache")]+use crate::multibackend::provider::CacheProvider;+use crate::multibackend::provider::{FilesystemProvider, MultiWriteProvider};use crate::multibackend::types::MultiBackendBuildContext;use crate::shape::validate::ensure_backend_shape;@@ -51,14 +53,15 @@ pub async fn init_backend_plugin(Ok(Arc::from(fs))}-/// Build the multi-backend wrapper from primary and backup configuration.-pub async fn build_multi_write_fs(+/// Build the multi-backend wrapper without starting background work.+pub(crate) async fn build_inactive_multi_write_fs(registry: &Arc<RwLock<HashMap<String, Arc<dyn ServicePlugin>>>>,config: &PluginConfig,bc: &BackendsConfig,build_ctx: MultiBackendBuildContext,) -> Result<MultiWriteWrappedFS> {let global_encryption_enabled = build_ctx.global_encryption_enabled();+ validate_backups_config(bc)?;validate_primary_encryption_flags(config, global_encryption_enabled)?;let primary_raw = init_backend_plugin(registry, &config.name, &config.params).await?;@@ -92,22 +95,37 @@ pub async fn build_multi_write_fs(primary_raw.clone()};- let mut backup_entries: Vec<BackendEntry> = Vec::new();- let mut seen_names: HashSet<String> = HashSet::new();-- for item in &bc.items {- if item.name == "primary" {- return Err(Error::config(- "backup backend name 'primary' is reserved".to_string(),- ));- }- if !seen_names.insert(item.name.clone()) {- return Err(Error::config(format!(- "duplicate backup name '{}'",- item.name- )));+ let metadata_store = Arc::new(MetadataStore::new(+ primary_backend.clone(),+ build_ctx.pathlock_manager.clone(),+ &build_ctx.backend_prefix,+ )?);+ let metadata_provider: Arc<dyn MultiWriteProvider> = match bc.provider.as_str() {+ "filesystem" => Arc::new(FilesystemProvider::new(metadata_store.clone())),+ "cache" => {+ #[cfg(feature = "cache")]+ {+ let runtime = build_ctx.cache_runtime.clone().ok_or_else(|| {+ Error::config("backups.provider = 'cache' requires a top-level CacheRuntime")+ })?;+ Arc::new(CacheProvider::new(+ metadata_store.clone(),+ runtime,+ bc.namespace.clone(),+ )?)+ }+ #[cfg(not(feature = "cache"))]+ {+ return Err(Error::config(+ "backups.provider = 'cache' requires the ragfs cache feature",+ ));+ }}+ _ => unreachable!("backup provider validated before construction"),+ };+ let mut backup_entries: Vec<BackendEntry> = Vec::new();+ for item in &bc.items {let backup_params = item_params_to_config_values(&item.params)?;let backup_raw = init_backend_plugin(registry, &item.backend, &backup_params).await?;let backup_encrypted = global_encryption_enabled@@ -160,26 +178,17 @@ pub async fn build_multi_write_fs(role: BackendRole::Backup,backend: backup_backend,raw_backend: None,- operations: item.operations.clone().unwrap_or_default(),- excludes: item.excludes.clone().unwrap_or_default(),});}- validate_redirect_targets(&config.primary_redirects, &backup_entries)?;- validate_backup_excludes(bc)?;-MultiWriteWrappedFS::builder(primary_backend).with_primary_raw_backend(primary_raw)+ .with_metadata_store(metadata_store)+ .with_metadata_provider(metadata_provider).with_backups(backup_entries)- .with_redirects(config.primary_redirects.clone())- .sync_mode(sync_mode_from_config(bc))- .write_concurrency(bc.write_concurrency)- .retry_interval(Duration::from_millis(- bc.retry_interval_ms.unwrap_or(30_000),- ))- .retry_backoff_base_ms(bc.retry_backoff_base_ms.unwrap_or(1_000))- .max_retry_per_round(bc.retry_max_retries_per_round.unwrap_or(3))- .quarantine_after_failures(bc.retry_quarantine_after_failures.unwrap_or(9))+ .initial_partitions(bc.initial_partitions)+ .checkpoint_interval(Duration::from_secs(bc.checkpoint_interval_secs)).ctx_resolver(Arc::new(RelativePathFsContextResolver))- .build()+ .build_inactive()+ .await}diff --git a/crates/ragfs/src/multibackend/gc.rs b/crates/ragfs/src/multibackend/gc.rsnew file mode 100644index 000000000..64507ca22--- /dev/null+++ b/crates/ragfs/src/multibackend/gc.rs@@ -0,0 +1,250 @@+//! Garbage collection for versioned multi-write metadata.++use std::collections::BTreeMap;+use std::sync::Arc;+use std::time::Duration;++use crate::core::errors::{Error, Result};+use crate::multibackend::constants::SEGMENT_FILE_EXTENSION;+use crate::multibackend::meta::{MetadataStore, MultiWriteWorker};+use crate::multibackend::model::{is_checkpoint_directory_name, CheckpointsManifest, DirectoryEvent, LatestCheckpoint,PartitionState, PartitionsManifest, ProtocolStatus, ScopeKey, SegmentManifest,};+use crate::multibackend::provider::MultiWriteProvider;++struct ScopeSnapshot { latest: Option<LatestCheckpoint>, segments: SegmentManifest}++/// Collects obsolete segment and checkpoint metadata after publication.+pub struct MetadataGc {+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+}++impl MetadataGc {+ /// Create a collector over the shared metadata store and segment provider.+ pub fn new(store: Arc<MetadataStore>, provider: Arc<dyn MultiWriteProvider>) -> Self {+ Self { store, provider }+ }++ /// Collect one scope while treating concurrent account deletion as complete.+ pub async fn collect_scope(&self, scope: &ScopeKey) -> Result<()> {+ match self.collect_scope_inner(scope).await {+ Err(Error::NotFound(_)) => Ok(()),+ Err(error) => {+ self.store.record_worker_error(MultiWriteWorker::Gc);+ Err(error)+ }+ result => result,+ }+ }++ /// Apply account safety checks before collecting one Stable scope.+ async fn collect_scope_inner(&self, scope: &ScopeKey) -> Result<()> {+ if self.store.protocol_status().await? != ProtocolStatus::Stable {+ return Ok(());+ }+ scope.validate()?;+ let account = self.store.update_partitions_manifest(&scope.account_id,+ |manifest| Ok(manifest.clone())).await?;+ if account.partitions.values().any(|entry| entry.state != PartitionState::Stable)+ || account.epoch != scope.epoch+ || !account.partitions.contains_key(&scope.partition_id)+ {+ return Ok(());+ }+ let stable = stable_partitions(&account);+ let positions_complete =+ directory_positions_complete(&account.directory_events, &stable, account.epoch);+ let retained = if positions_complete {+ self.collect_completed_events(&scope.account_id, &account, &stable).await?+ } else {+ Some(account.directory_events.clone())+ };+ let Some(retained) = retained else {+ return Ok(());+ };+ let allow_segments =+ positions_complete && directory_positions_complete(&retained, &stable, account.epoch);+ let latest = self.read_latest(scope).await?;+ if allow_segments {+ self.collect_segments(scope, latest.as_ref(), &retained).await?;+ }+ self.collect_checkpoints(scope).await+ }++ /// Remove completed events only if their account identity and content remain unchanged.+ async fn collect_completed_events(&self, account_id: &str,+ account: &PartitionsManifest, stable: &[u32]) -> Result<Option<Vec<DirectoryEvent>>> {+ let mut snapshots = BTreeMap::new();+ for &partition_id in stable {+ let scope = ScopeKey { account_id: account_id.to_string(),+ partition_id, epoch: account.epoch };+ snapshots.insert(partition_id, self.read_scope_snapshot(&scope).await?);+ }+ let completed = account.directory_events.iter()+ .filter(|event| event_is_complete(event, &snapshots))+ .cloned().collect::<Vec<_>>();+ let account_id = account_id.to_string();+ let expected_epoch = account.epoch;+ let expected_stable = stable.to_vec();+ self.store.update_partitions_manifest(&account_id, move |current| {+ if current.epoch != expected_epoch+ || stable_partitions(current) != expected_stable+ {+ return Ok(None);+ }+ current.directory_events.retain(|event|+ !completed.iter().any(|candidate| candidate == event));+ Ok(Some(current.directory_events.clone()))+ }).await+ }++ /// Read a checkpoint pointer briefly and pair it with segment progress.+ async fn read_scope_snapshot(&self, scope: &ScopeKey) -> Result<ScopeSnapshot> {+ let latest = self.read_latest(scope).await?;+ let segments = self.provider.read_manifest(scope).await?;+ segments.validate()?;+ Ok(ScopeSnapshot { latest, segments })+ }++ /// Read the latest checkpoint while holding only its pointer Exact lock.+ async fn read_latest(&self, scope: &ScopeKey) -> Result<Option<LatestCheckpoint>> {+ let path = self.store.paths().checkpoints_manifest(+ &scope.account_id, scope.partition_id)?.0;+ let lease = self.store.pathlock_manager().acquire_exact(+ &path,+ self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)),+ None,+ ).await?;+ let operation = async {+ let pointer: CheckpointsManifest = self.store.read_json(&path).await?;+ pointer.validate()?;+ Ok(pointer.latest_checkpoint)+ }.await;+ let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from);+ finish_with_release(operation, release)+ }++ /// Prune one eligible sealed prefix and remove every unreferenced segment blob.+ async fn collect_segments(&self, scope: &ScopeKey, latest: Option<&LatestCheckpoint>,+ retained: &[DirectoryEvent]) -> Result<()> {+ let manifest = self.provider.read_manifest(scope).await?;+ manifest.validate()?;+ let removable = manifest.segments.iter().take_while(|descriptor| {+ descriptor.segment_to_seq.is_some_and(|to| {+ latest.is_some_and(|checkpoint| checkpoint.checkpoint_to_seq >= to)+ && manifest.backend_states.values().all(|state| state.synced_seq >= to)+ && !retained.iter().any(|event| event.positions.iter().any(|position| {+ position.partition_id == scope.partition_id+ && position.epoch == scope.epoch+ && (descriptor.segment_from_seq..=to).contains(&position.seq)+ }))+ })+ }).map(|descriptor| descriptor.path.clone()).collect::<Vec<_>>();+ if !removable.is_empty() {+ for descriptor in self.provider.prune_sealed(scope, &removable).await? {+ self.remove_segment(scope, &descriptor.path).await?;+ }+ }+ let directory = self.store.paths().segments_dir(+ &scope.account_id, scope.partition_id)?.0;+ for entry in self.store.list_directory(&directory).await? {+ if !entry.is_dir && entry.name.ends_with(SEGMENT_FILE_EXTENSION) {+ self.remove_segment(scope, &entry.name).await?;+ }+ }+ Ok(())+ }++ /// Delete one segment blob while holding its Exact path lock.+ async fn remove_segment(&self, scope: &ScopeKey, name: &str) -> Result<()> {+ let directory = self.store.paths().segments_dir(+ &scope.account_id, scope.partition_id)?.0;+ let path = format!("{directory}/{name}");+ let lease = self.store.pathlock_manager().acquire_exact(+ &path,+ self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)),+ None,+ ).await?;+ let operation = async {+ let manifest = self.provider.read_manifest(scope).await?;+ if manifest.segments.iter().any(|descriptor| descriptor.path == name) {+ return Ok(());+ }+ self.store.remove_file(&path).await+ }.await;+ let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from);+ finish_with_release(operation, release)+ }++ /// Delete non-latest checkpoint candidates in manifest, chunks, directory order.+ async fn collect_checkpoints(&self, scope: &ScopeKey) -> Result<()> {+ let directory = self.store.paths().checkpoints_dir(+ &scope.account_id, scope.partition_id)?.0;+ for entry in self.store.list_directory(&directory).await? {+ if !entry.is_dir || !is_checkpoint_directory_name(&entry.name) {+ continue;+ }+ let candidate = format!("{directory}/{}", entry.name);+ let lease = self.store.pathlock_manager().acquire_exact(+ &candidate,+ self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)),+ None,+ ).await?;+ let operation = async {+ if self.read_latest(scope).await?.as_ref().is_some_and(+ |latest| latest.path.trim_end_matches('/') == entry.name)+ {+ return Ok(());+ }+ self.store.remove_all(&candidate).await+ }.await;+ let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from);+ finish_with_release(operation, release)?;+ }+ Ok(())+ }+}++/// Return Stable partition identifiers in deterministic order.+fn stable_partitions(manifest: &PartitionsManifest) -> Vec<u32> {+ manifest.partitions.iter().filter_map(|(&partition_id, entry)| {+ (entry.state == PartitionState::Stable).then_some(partition_id)+ }).collect()+}++/// Return whether every event has one current-epoch position per Stable partition.+fn directory_positions_complete(+ events: &[DirectoryEvent],+ stable: &[u32],+ epoch: u64,+) -> bool {+ events.iter().all(|event| stable.iter().all(|partition_id| {+ event.positions.iter().any(|position| {+ position.partition_id == *partition_id && position.epoch == epoch+ })+ }))+}++/// Return whether every marker is absorbed and applied by every current backend.+fn event_is_complete(+ event: &DirectoryEvent,+ snapshots: &BTreeMap<u32, ScopeSnapshot>,+) -> bool {+ event.positions.iter().all(|position| snapshots.get(&position.partition_id)+ .is_some_and(|snapshot| {+ snapshot.latest.as_ref().is_some_and(+ |latest| latest.checkpoint_to_seq >= position.seq)+ && snapshot.segments.backend_states.values().all(+ |state| state.synced_seq >= position.seq)+ }))+}++/// Preserve an operation failure while requiring lock release to succeed.+fn finish_with_release<T>(operation: Result<T>, release: Result<()>) -> Result<T> {+ match (operation, release) {+ (Ok(value), Ok(())) => Ok(value),+ (Ok(_), Err(error)) | (Err(error), Ok(())) => Err(error),+ (Err(error), Err(release_error)) => Err(Error::internal(format!(+ "{error}; lock release failed: {release_error}"+ ))),+ }+}diff --git a/crates/ragfs/src/multibackend/meta.rs b/crates/ragfs/src/multibackend/meta.rsindex 777a3cdcf..6c347df44 100644--- a/crates/ragfs/src/multibackend/meta.rs+++ b/crates/ragfs/src/multibackend/meta.rs@@ -1,26 +1,26 @@//! Multi-write metadata management.-//!-//! Provides `MetaStateStore` for serialized read-modify-write of `.redirect.json` and-//! `.sync_log.json` through `primary_backend`, and `FsContextResolver` for recovering-//! `FsContext` from paths in background tasks.-use std::collections::HashMap;+use std::collections::BTreeMap;+use std::sync::atomic::{AtomicU64, Ordering};use std::sync::Arc;-use std::time::Duration;-use tokio::sync::Mutex;-use serde::{de::DeserializeOwned, Deserialize, Serialize};+use serde::{de::DeserializeOwned, Serialize};use crate::core::context::{FsContext, FsContextInner, FS_CTX};use crate::core::errors::{Error, Result};use crate::core::filesystem::FileSystem;-use crate::core::types::{RedirectMeta, SyncLogMeta, WriteFlag};+use crate::core::types::{FileInfo, WriteFlag};+use crate::lock::PathLockManager;+use crate::multibackend::constants::{MULTIWRITE_MOUNT_PREFIX, VBUCKETS};+use crate::multibackend::model::{+ BackendState, CheckpointsManifest, PartitionEntry, PartitionManifest, PartitionState,+ PartitionsManifest, ProtocolState, ProtocolStatus, ScopeKey, SegmentManifest,+};+use crate::multibackend::router::{build_initial_routes, MultiWritePaths};/// Trait for resolving `FsContext` from a filesystem path.///-/// Used by background tasks (retry_loop, backfill, system_sync_retry) that lack a-/// foreground request context. Implementations extract `account_id` from the path-/// (e.g. `/local/{account_id}/...`).+/// Implementations extract `account_id` from the path (e.g. `/local/{account_id}/...`).pub trait FsContextResolver: Send + Sync {/// Recover `FsContext` from a normalized path./// Returns an error if the path cannot be resolved to a valid context.@@ -34,7 +34,10 @@ impl FsContextResolver for DefaultFsContextResolver {fn resolve(&self, path: &str) -> Result<FsContext> {let parts: Vec<&str> = path.trim_start_matches('/').split('/').collect();// Path format: /local/{account_id}/...- if parts.len() >= 2 && parts[0] == "local" && !parts[1].is_empty() {+ if parts.len() >= 2+ && parts[0] == MULTIWRITE_MOUNT_PREFIX.trim_start_matches('/')+ && !parts[1].is_empty()+ {Ok(Arc::new(FsContextInner::new(parts[1].to_string())))} else {Err(Error::internal(format!(@@ -61,429 +64,461 @@ impl FsContextResolver for RelativePathFsContextResolver {}}-/// Internal file name for redirect metadata.-pub(crate) const REDIRECT_FILE: &str = ".redirect.json";-/// Internal file name for sync-log metadata.-pub(crate) const SYNC_LOG_FILE: &str = ".sync_log.json";-/// Hidden multi-write internal file names (redirect / sync-log metadata).-pub(crate) const MULTIWRITE_INTERNAL_NAMES: &[&str] = &[SYNC_LOG_FILE, REDIRECT_FILE];-const GLOBAL_STATE_FILE: &str = "/_system/.multiwrite.global.json";-const GLOBAL_STATE_VERSION: u32 = 1;--type TrackedLock = Mutex<()>;--#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]-struct GlobalMultiWriteState {- version: u32,- next_seq: u64,+/// Closed worker labels used by the multi-write error counter.+#[derive(Clone, Copy)]+pub(crate) enum MultiWriteWorker {+ Flush,+ Checkpoint,+ CatchUp,+ Gc,}-impl Default for GlobalMultiWriteState {- /// Create the default persisted global state for multi-write sequencing.- fn default() -> Self {- Self {- version: GLOBAL_STATE_VERSION,- next_seq: 1,- }- }+/// Primary filesystem store for V2 multi-write metadata.+pub struct MetadataStore {+ primary: Arc<dyn FileSystem>,+ pathlock_manager: Arc<PathLockManager>,+ paths: MultiWritePaths,+ worker_errors: [AtomicU64; 4],}-/// Unified metadata store for `.redirect.json` and `.sync_log.json`.-///-/// All reads and writes go through `primary_backend`, inheriting its encryption-/// configuration. Directory-level locks ensure serialized access to both metadata-/// files within the same directory.-pub struct MetaStateStore {- /// Primary backend (may be encrypted)- primary_backend: Arc<dyn FileSystem>,- /// Per-directory locks for serialized read-modify-write- dir_locks: Mutex<HashMap<String, Arc<TrackedLock>>>,- /// Dedicated global state lock for next_seq persistence.- global_state_lock: Mutex<()>,- /// Context resolver for background tasks- ctx_resolver: Arc<dyn FsContextResolver>,-}--impl MetaStateStore {- /// Create a new MetaStateStore.+impl MetadataStore {+ /// Create a V2 metadata store for one logical mount prefix.pub fn new(- primary_backend: Arc<dyn FileSystem>,- ctx_resolver: Arc<dyn FsContextResolver>,- ) -> Self {- Self::with_cache_config(primary_backend, ctx_resolver, 0, Duration::ZERO)+ primary: Arc<dyn FileSystem>,+ pathlock_manager: Arc<PathLockManager>,+ mount_prefix: &str,+ ) -> Result<Self> {+ Ok(Self {+ primary,+ pathlock_manager,+ paths: MultiWritePaths::new(mount_prefix)?,+ worker_errors: std::array::from_fn(|_| AtomicU64::new(0)),+ })}- /// Create a new MetaStateStore; cache parameters are retained as no-op compatibility knobs.- pub fn with_cache_config(- primary_backend: Arc<dyn FileSystem>,- ctx_resolver: Arc<dyn FsContextResolver>,- _meta_cache_capacity: usize,- _meta_cache_ttl: Duration,- ) -> Self {- Self {- primary_backend,- dir_locks: Mutex::new(HashMap::new()),- global_state_lock: Mutex::new(()),- ctx_resolver,- }+ /// Record one final failure for the selected worker.+ pub(crate) fn record_worker_error(&self, worker: MultiWriteWorker) {+ self.worker_errors[worker as usize].fetch_add(1, Ordering::Relaxed);}- /// Get or create a per-directory lock.- async fn get_dir_lock(&self, dir: &str) -> Arc<TrackedLock> {- let mut locks = self.dir_locks.lock().await;- if let Some(lock) = locks.get(dir).cloned() {- return lock;- }- let lock = Arc::new(TrackedLock::new(()));- locks.insert(dir.to_string(), lock.clone());- lock+ /// Return the final failure count for the selected worker.+ pub(crate) fn worker_error_count(&self, worker: MultiWriteWorker) -> u64 {+ self.worker_errors[worker as usize].load(Ordering::Relaxed)}- /// Return the dedicated `_system` context used by global metadata.- fn system_ctx() -> FsContext {- Arc::new(FsContextInner::new("_system".to_string()))+ /// Return the logical-to-backend path mapper owned by this store.+ pub fn paths(&self) -> &MultiWritePaths {+ &self.paths}- /// Build the effective context for metadata access.- fn effective_meta_ctx(dir: &str, ctx: &FsContext) -> FsContext {- if dir == "/" {- Arc::new(Self::system_ctx().with_auto_pathlock_disabled())- } else {- Arc::new(ctx.with_auto_pathlock_disabled())+ /// List initialized scope directories currently visible on the primary filesystem.+ pub(crate) async fn initialized_accounts(&self) -> Result<Vec<String>> {+ let mut accounts = Vec::new();+ for entry in self.primary.read_internal_dir("/").await? {+ if !entry.is_dir {+ continue;+ }+ let manifest = self.paths.account_manifest(&entry.name)?.1;+ match self.primary.stat(&manifest).await {+ Ok(_) => accounts.push(entry.name),+ Err(Error::NotFound(_)) => {}+ Err(error) => return Err(error),+ }+ }+ accounts.sort();+ Ok(accounts)+ }++ /// Return whether the primary account root still exists.+ pub(crate) async fn account_exists(&self, account_id: &str) -> Result<bool> {+ let root = self+ .paths+ .raw_backend_path(&format!("{MULTIWRITE_MOUNT_PREFIX}/{account_id}"))?;+ match self.primary.stat(&root).await {+ Ok(_) => Ok(true),+ Err(Error::NotFound(_)) => Ok(false),+ Err(error) => Err(error),}}- /// Build the full path for a metadata file in a directory.- fn meta_path(dir: &str, filename: &str) -> String {- if dir == "/" {- format!("/{}", filename)- } else {- format!("{}/{}", dir, filename)- }+ /// Return the path lock manager used by V2 metadata operations.+ pub(crate) fn pathlock_manager(&self) -> &PathLockManager {+ &self.pathlock_manager}- /// Read one JSON metadata file, returning default for missing or empty files.- async fn read_meta<T>(&self, dir: &str, filename: &str, ctx: &FsContext) -> Result<T>- where- T: DeserializeOwned + Default,- {- let path = Self::meta_path(dir, filename);- let effective_ctx = Self::effective_meta_ctx(dir, ctx);- match FS_CTX- .scope(effective_ctx, async {- self.primary_backend.read(&path, 0, 0).await- })- .await+ /// Build the filesystem context used for one logical metadata path.+ fn metadata_context(&self, logical_path: &str) -> Result<FsContext> {+ let parts: Vec<&str> = logical_path.trim_start_matches('/').split('/').collect();+ if parts.len() >= 2+ && parts[0] == MULTIWRITE_MOUNT_PREFIX.trim_start_matches('/')+ && !parts[1].is_empty(){- Ok(data) => {- if data.is_empty() {- Ok(T::default())- } else {- serde_json::from_slice(&data).map_err(Error::from)- }- }- Err(Error::NotFound(_)) => Ok(T::default()),- Err(e) => Err(e),+ return Ok(Arc::new(+ FsContextInner::new(parts[1].to_string())+ .with_bypass_cache(true)+ .with_auto_pathlock_disabled(),+ ));}+ Err(Error::internal(format!(+ "cannot resolve metadata FsContext from path: {logical_path}"+ )))}- /// Read redirect metadata from a directory (returns default if not found).- async fn read_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result<RedirectMeta> {- self.read_meta(dir, REDIRECT_FILE, ctx).await- }-- /// Read sync log metadata from a directory (returns default if not found).- async fn read_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result<SyncLogMeta> {- self.read_meta(dir, SYNC_LOG_FILE, ctx).await- }-- /// Read both metadata files from the primary backend.- async fn read_dir_meta_pair(- &self,- dir: &str,- ctx: &FsContext,- ) -> Result<(RedirectMeta, SyncLogMeta)> {- let redirect = self.read_redirect_meta(dir, ctx).await?;- let sync_log = self.read_sync_log_meta(dir, ctx).await?;- Ok((redirect, sync_log))- }-- /// Write one JSON metadata file to a directory.- async fn write_meta<T>(- &self,- dir: &str,- filename: &str,- meta: &T,- ctx: &FsContext,- ) -> Result<()>- where- T: Serialize,- {- let path = Self::meta_path(dir, filename);- let data = serde_json::to_vec(meta)?;- let effective_ctx = Self::effective_meta_ctx(dir, ctx);+ /// Read one complete binary value from a canonical logical path.+ pub(crate) async fn read_bytes(&self, logical_path: &str) -> Result<Vec<u8>> {+ let backend_path = self.paths.raw_backend_path(logical_path)?;+ let context = self.metadata_context(logical_path)?;FS_CTX- .scope(effective_ctx, async {- self.primary_backend- .write(&path, &data, 0, WriteFlag::Create)- .await- .map(|_| ())- })+ .scope(context, self.primary.read(&backend_path, 0, 0)).await}- /// Write redirect metadata to a directory.- async fn write_redirect_meta(- &self,- dir: &str,- meta: &RedirectMeta,- ctx: &FsContext,- ) -> Result<()> {- self.write_meta(dir, REDIRECT_FILE, meta, ctx).await+ /// Read and validate the current V2 protocol status.+ pub(crate) async fn protocol_status(&self) -> Result<ProtocolStatus> {+ let state: ProtocolState = self.read_json(&self.paths.mount_protocol().0).await?;+ state.validate()?;+ Ok(state.status)}- /// Write sync log metadata to a directory.- async fn write_sync_log_meta(+ /// Replace protocol bytes under an Exact lock when the current value matches.+ pub(crate) async fn compare_protocol(&self,- dir: &str,- meta: &SyncLogMeta,- ctx: &FsContext,- ) -> Result<()> {- self.write_meta(dir, SYNC_LOG_FILE, meta, ctx).await+ expected: &[u8],+ replacement: &[u8],+ ) -> Result<bool> {+ let logical_path = self.paths.mount_protocol().0;+ let lease = self+ .pathlock_manager+ .acquire_exact(+ &logical_path,+ self.pathlock_manager+ .default_lock_timeout()+ .max(std::time::Duration::from_secs(5)),+ None,+ )+ .await?;+ let operation = async {+ match self.read_bytes(&logical_path).await {+ Ok(current) if current == expected => {+ self.write_bytes(&logical_path, replacement).await?;+ Ok(true)+ }+ Ok(_) | Err(Error::NotFound(_)) => Ok(false),+ Err(error) => Err(error),+ }+ }+ .await;+ let release = self+ .pathlock_manager+ .release(&lease)+ .await+ .map_err(Error::from);+ merge_operation_and_cleanup(operation, release)}- /// Serialized read-modify-write of both `.redirect.json` and `.sync_log.json` in a directory.- ///- /// Acquires the directory lock, reads both metadata files, applies `op`, and writes both back.- /// This prevents concurrent updates from losing entries.- pub async fn update_dir_meta<F>(&self, dir: &str, ctx: &FsContext, op: F) -> Result<()>+ /// Read and deserialize one JSON value from a canonical logical path.+ pub async fn read_json<T>(&self, logical_path: &str) -> Result<T>where- F: FnOnce(&mut RedirectMeta, &mut SyncLogMeta) -> Result<()>,+ T: DeserializeOwned,{- let lock = self.get_dir_lock(dir).await;- let _guard = lock.lock().await;-- let (mut redirect_meta, mut sync_log_meta) = self.read_dir_meta_pair(dir, ctx).await?;- let original_redirect = redirect_meta.clone();- let original_sync_log = sync_log_meta.clone();-- op(&mut redirect_meta, &mut sync_log_meta)?;+ let bytes = self.read_bytes(logical_path).await?;+ serde_json::from_slice(&bytes).map_err(Error::from)+ }- if redirect_meta != original_redirect {- self.write_redirect_meta(dir, &redirect_meta, ctx).await?;- }- if sync_log_meta != original_sync_log {- self.write_sync_log_meta(dir, &sync_log_meta, ctx).await?;- }- Ok(())+ /// Write complete binary bytes to a canonical logical path.+ pub(crate) async fn write_bytes(&self, logical_path: &str, bytes: &[u8]) -> Result<()> {+ let backend_path = self.paths.raw_backend_path(logical_path)?;+ let context = self.metadata_context(logical_path)?;+ FS_CTX+ .scope(context, async {+ self.primary+ .ensure_parent_dirs(&backend_path, 0o755)+ .await?;+ self.primary+ .write(&backend_path, bytes, 0, WriteFlag::Create)+ .await?;+ Ok(())+ })+ .await}- /// Serialized read-modify-write of two directories' metadata (for cross-directory rename).- ///- /// Acquires both directory locks in lexicographic order to prevent deadlock,- /// then reads and updates all four metadata files within the same critical section.- /// Caller must ensure source_dir != target_dir; use update_dir_meta for same-directory case.- pub async fn update_dual_dir_meta<F>(- &self,- source_dir: &str,- target_dir: &str,- ctx: &FsContext,- op: F,- ) -> Result<()>+ /// Validate and write one JSON value at a logical path.+ pub async fn publish_json<T, F>(&self, logical_path: &str, value: &T, validate: F) -> Result<()>where- F: FnOnce(- &mut RedirectMeta,- &mut SyncLogMeta,- &mut RedirectMeta,- &mut SyncLogMeta,- ) -> Result<()>,+ T: Serialize,+ F: FnOnce(&T) -> Result<()>,{- // Acquire locks in lexicographic order to avoid deadlock.- let (first_dir, second_dir) = if source_dir < target_dir {- (source_dir, target_dir)- } else {- (target_dir, source_dir)- };-- let lock1 = self.get_dir_lock(first_dir).await;- let lock2 = self.get_dir_lock(second_dir).await;- let _guard1 = lock1.lock().await;- let _guard2 = lock2.lock().await;-- let (mut src_redirect, mut src_sync_log) = self.read_dir_meta_pair(source_dir, ctx).await?;- let (mut tgt_redirect, mut tgt_sync_log) = self.read_dir_meta_pair(target_dir, ctx).await?;- let original_src_redirect = src_redirect.clone();- let original_src_sync_log = src_sync_log.clone();- let original_tgt_redirect = tgt_redirect.clone();- let original_tgt_sync_log = tgt_sync_log.clone();-- op(- &mut src_redirect,- &mut src_sync_log,- &mut tgt_redirect,- &mut tgt_sync_log,- )?;-- if src_redirect != original_src_redirect {- self.write_redirect_meta(source_dir, &src_redirect, ctx)- .await?;- }- if src_sync_log != original_src_sync_log {- self.write_sync_log_meta(source_dir, &src_sync_log, ctx)- .await?;- }- if tgt_redirect != original_tgt_redirect {- self.write_redirect_meta(target_dir, &tgt_redirect, ctx)- .await?;+ let bytes = serde_json::to_vec(value)?;+ validate(value)?;+ self.write_bytes(logical_path, &bytes).await+ }++ /// Create one immutable blob or accept an identical existing value.+ pub async fn write_immutable(&self, logical_path: &str, bytes: &[u8]) -> Result<()> {+ match self.read_bytes(logical_path).await {+ Ok(existing) if existing == bytes => Ok(()),+ Ok(_) => Err(Error::AlreadyExists(format!(+ "immutable metadata differs: {logical_path}"+ ))),+ Err(Error::NotFound(_)) => self.write_bytes(logical_path, bytes).await,+ Err(error) => Err(error),}- if tgt_sync_log != original_tgt_sync_log {- self.write_sync_log_meta(target_dir, &tgt_sync_log, ctx)- .await?;- }- Ok(())}- /// Read redirect metadata for a directory (public, used by read_dir to merge redirect entries).- pub async fn get_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result<RedirectMeta> {- self.read_redirect_meta(dir, ctx).await+ /// List one raw metadata directory without public hidden-name filtering.+ pub(crate) async fn list_directory(&self, logical_path: &str) -> Result<Vec<FileInfo>> {+ let backend_path = self.paths.raw_backend_path(logical_path)?;+ self.primary.read_internal_dir(&backend_path).await}- /// Read sync log metadata for a directory (public, used by retry_loop).- pub async fn get_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result<SyncLogMeta> {- self.read_sync_log_meta(dir, ctx).await+ /// Remove one raw metadata file while treating absence as success.+ pub(crate) async fn remove_file(&self, logical_path: &str) -> Result<()> {+ let backend_path = self.paths.raw_backend_path(logical_path)?;+ self.remove_if_present(&backend_path).await}- /// Get a reference to the context resolver.- pub fn ctx_resolver(&self) -> &Arc<dyn FsContextResolver> {- &self.ctx_resolver+ /// Remove one raw metadata tree while treating absence as success.+ pub(crate) async fn remove_all(&self, logical_path: &str) -> Result<()> {+ let backend_path = self.paths.raw_backend_path(logical_path)?;+ match self.primary.remove_all(&backend_path).await {+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),+ Err(error) => Err(error),+ }}- /// Get a reference to the primary backend.- pub fn primary_backend(&self) -> &Arc<dyn FileSystem> {- &self.primary_backend+ /// Recheck destructive GC safety while the caller holds the account Exact lock.+ pub(crate) async fn gc_scope_is_safe(&self, scope: &ScopeKey) -> Result<bool> {+ let path = self.paths.account_manifest(&scope.account_id)?.0;+ let manifest: PartitionsManifest = self.read_json(&path).await?;+ manifest.validate()?;+ scope.validate_against_manifest(&manifest)?;+ let stable = manifest+ .partitions+ .iter()+ .filter_map(|(&id, entry)| (entry.state == PartitionState::Stable).then_some(id))+ .collect::<Vec<_>>();+ Ok(stable.len() == manifest.partitions.len()+ && manifest.directory_events.iter().all(|event| {+ stable.iter().all(|id| {+ event.positions.iter().any(|position| {+ position.partition_id == *id && position.epoch == manifest.epoch+ })+ })+ }))+ }++ /// Initialize one account under an Exact manifest lock and return its manifest.+ pub async fn initialize_account(+ &self,+ account_id: &str,+ partition_count: u32,+ backup_names: &[String],+ ) -> Result<PartitionsManifest> {+ let logical_manifest = self.paths.account_manifest(account_id)?.0;+ let lease = self+ .pathlock_manager+ .acquire_exact(+ &logical_manifest,+ self.pathlock_manager+ .default_lock_timeout()+ .max(std::time::Duration::from_secs(5)),+ None,+ )+ .await?;+ let result = self+ .initialize_account_locked(account_id, partition_count, backup_names)+ .await;+ let release = self+ .pathlock_manager+ .release(&lease)+ .await+ .map_err(Error::from);+ merge_operation_and_cleanup(result, release)}- /// Allocate and persist the next global sequence number.- pub async fn next_seq(&self) -> Result<u64> {- let _guard = self.global_state_lock.lock().await;- let mut state = self.read_global_state().await?;- let seq = state.next_seq;- state.next_seq = state.next_seq.saturating_add(1);- self.write_global_state(&state).await?;- Ok(seq)+ /// Update one partitions manifest while holding its Exact lock.+ pub(crate) async fn update_partitions_manifest<T, F>(+ &self,+ account_id: &str,+ update: F,+ ) -> Result<T>+ where+ F: FnOnce(&mut PartitionsManifest) -> Result<T>,+ {+ let logical_manifest = self.paths.account_manifest(account_id)?.0;+ let lease = self+ .pathlock_manager+ .acquire_exact(+ &logical_manifest,+ self.pathlock_manager+ .default_lock_timeout()+ .max(std::time::Duration::from_secs(5)),+ None,+ )+ .await?;+ let operation = async {+ let mut manifest: PartitionsManifest = self.read_json(&logical_manifest).await?;+ manifest.validate()?;+ let original = manifest.clone();+ let output = update(&mut manifest)?;+ validate_partitions_manifest_update(&original, &manifest)?;+ manifest.validate()?;+ if manifest != original {+ self.publish_json(&logical_manifest, &manifest, PartitionsManifest::validate)+ .await?;+ }+ Ok(output)+ }+ .await;+ let release = self+ .pathlock_manager+ .release(&lease)+ .await+ .map_err(Error::from);+ merge_operation_and_cleanup(operation, release)}- /// Read the persisted global state file.- async fn read_global_state(&self) -> Result<GlobalMultiWriteState> {- let ctx = Self::system_ctx();- match FS_CTX- .scope(ctx.clone(), async {- self.primary_backend.read(GLOBAL_STATE_FILE, 0, 0).await- })+ /// Initialize account metadata while the caller holds the manifest lock.+ async fn initialize_account_locked(+ &self,+ account_id: &str,+ partition_count: u32,+ backup_names: &[String],+ ) -> Result<PartitionsManifest> {+ let logical_manifest = self.paths.account_manifest(account_id)?.0;+ match self+ .read_json::<PartitionsManifest>(&logical_manifest).await{- Ok(data) => {- if data.is_empty() {- Ok(GlobalMultiWriteState::default())- } else {- let state: GlobalMultiWriteState = serde_json::from_slice(&data)?;- if state.version != GLOBAL_STATE_VERSION {- return Err(Error::config(format!(- "unsupported multi-write global state version {}",- state.version- )));- }- Ok(state)- }+ Ok(manifest) => {+ manifest.validate()?;+ return Ok(manifest);}- Err(Error::NotFound(_)) => Ok(GlobalMultiWriteState::default()),- Err(e) => Err(e),+ Err(Error::NotFound(_)) => {}+ Err(error) => return Err(error),}- }- /// Persist the global sequence state through the primary backend.- async fn write_global_state(&self, state: &GlobalMultiWriteState) -> Result<()> {- let ctx = Self::system_ctx();- let data = serde_json::to_vec(state)?;- FS_CTX- .scope(ctx.clone(), async {- self.primary_backend- .ensure_parent_dirs(GLOBAL_STATE_FILE, 0o755)- .await?;- self.primary_backend- .write(GLOBAL_STATE_FILE, &data, 0, WriteFlag::Create)- .await- .map(|_| ())+ let routes = build_initial_routes(partition_count)?;+ let partitions = (0..partition_count)+ .map(|partition_id| {+ (+ partition_id,+ PartitionEntry {+ state: PartitionState::Stable,+ },+ )})- .await- }-}--/// Per-path serialization queue for async write ordering.-///-/// Ensures that multiple writes to the same path are executed in FIFO order-/// on backup backends, preventing out-of-order application.-pub struct PathSerializer {- queues: Mutex<HashMap<String, Arc<TrackedLock>>>,-}--impl PathSerializer {- /// Create a new PathSerializer.- pub fn new() -> Self {- Self::with_limits(0, Duration::ZERO)- }-- /// Create a PathSerializer; limit arguments are ignored for compatibility.- pub fn with_limits(_capacity: usize, _ttl: Duration) -> Self {- Self {- queues: Mutex::new(HashMap::new()),+ .collect::<BTreeMap<_, _>>();+ let manifest = PartitionsManifest {+ version: 1,+ epoch: 1,+ vbuckets: VBUCKETS,+ partitions,+ routes,+ directory_events: Vec::new(),+ };+ manifest.validate()?;++ let backend_states = backup_names+ .iter()+ .map(|name| (name.clone(), BackendState { synced_seq: 0 }))+ .collect::<BTreeMap<_, _>>();+ for partition_id in 0..partition_count {+ let partition = PartitionManifest {+ version: 1,+ partition_id,+ epoch: 1,+ };+ self.write_validated_json_immutable(+ &self.paths.partition_manifest(account_id, partition_id)?.0,+ &partition,+ PartitionManifest::validate,+ )+ .await?;++ let segments = SegmentManifest {+ version: 1,+ next_seq: 1,+ segments: Vec::new(),+ backend_states: backend_states.clone(),+ };+ self.write_validated_json_immutable(+ &self.paths.segment_manifest(account_id, partition_id)?.0,+ &segments,+ SegmentManifest::validate,+ )+ .await?;++ let checkpoints = CheckpointsManifest {+ version: 1,+ latest_checkpoint: None,+ };+ self.write_validated_json_immutable(+ &self.paths.checkpoints_manifest(account_id, partition_id)?.0,+ &checkpoints,+ CheckpointsManifest::validate,+ )+ .await?;}+ self.write_validated_json_immutable(+ &logical_manifest,+ &manifest,+ PartitionsManifest::validate,+ )+ .await?;+ Ok(manifest)}- /// Run one async operation under the per-path serialization lock.- pub async fn with_path_lock<F, Fut, T>(&self, path: &str, op: F) -> T+ /// Validate, serialize, and create one immutable JSON metadata file.+ async fn write_validated_json_immutable<T, F>(+ &self,+ logical_path: &str,+ value: &T,+ validate: F,+ ) -> Result<()>where- F: FnOnce() -> Fut,- Fut: std::future::Future<Output = T>,+ T: Serialize,+ F: FnOnce(&T) -> Result<()>,{- let mut queues = self.queues.lock().await;- let lock = queues- .entry(path.to_string())- .or_insert_with(|| Arc::new(TrackedLock::new(())))- .clone();- drop(queues);-- let _guard = lock.lock().await;- op().await+ let bytes = serde_json::to_vec(value)?;+ validate(value)?;+ self.write_immutable(logical_path, &bytes).await}- /// Return the number of tracked queue entries.- #[cfg(test)]- pub async fn len(&self) -> usize {- self.queues.lock().await.len()- }-}--impl Default for PathSerializer {- fn default() -> Self {- Self::new()+ /// Remove a temporary file while treating an absent file as clean.+ async fn remove_if_present(&self, backend_path: &str) -> Result<()> {+ match self.primary.remove(backend_path).await {+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),+ Err(error) => Err(error),+ }}}-/// Extract the directory path from a file path.-pub(crate) fn parent_dir(path: &str) -> String {- match path.rfind('/') {- Some(0) => "/".to_string(),- Some(pos) => path[..pos].to_string(),- None => "/".to_string(),+/// Reject updates that change the fixed partition routing identity.+fn validate_partitions_manifest_update(+ original: &PartitionsManifest,+ updated: &PartitionsManifest,+) -> Result<()> {+ if original.epoch != updated.epoch+ || original.partitions != updated.partitions+ || original.routes != updated.routes+ {+ return Err(Error::invalid_operation(+ "partition routing cannot change after account initialization",+ ));}+ Ok(())}-/// Extract the file name from a path.-pub(crate) fn file_name(path: &str) -> &str {- match path.rfind('/') {- Some(pos) => &path[pos + 1..],- None => path,+/// Preserve an operation error while requiring cleanup to succeed.+fn merge_operation_and_cleanup<T>(operation: Result<T>, cleanup: Result<()>) -> Result<T> {+ match (operation, cleanup) {+ (Ok(value), Ok(())) => Ok(value),+ (Ok(_), Err(error)) => Err(error),+ (Err(error), Ok(())) => Err(error),+ (Err(error), Err(cleanup_error)) => Err(Error::internal(format!(+ "{error}; cleanup failed: {cleanup_error}"+ ))),}}@@ -497,21 +532,6 @@ pub fn current_required_ctx() -> Result<FsContext> {#[cfg(test)]mod tests {use super::*;- use crate::plugins::memfs::MemFileSystem;- use std::sync::Arc;-- #[test]- fn test_parent_dir() {- assert_eq!(parent_dir("/a/b/c.txt"), "/a/b");- assert_eq!(parent_dir("/a"), "/");- assert_eq!(parent_dir("/"), "/");- }-- #[test]- fn test_file_name() {- assert_eq!(file_name("/a/b/c.txt"), "c.txt");- assert_eq!(file_name("/a"), "a");- }#[test]fn test_default_resolver() {@@ -527,48 +547,4 @@ mod tests {let resolver = DefaultFsContextResolver;assert!(resolver.resolve("/invalid/path").is_err());}--- #[tokio::test]- async fn test_invalid_sync_log_json_returns_error() {- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let store = MetaStateStore::new(primary.clone(), Arc::new(DefaultFsContextResolver));- let ctx = Arc::new(FsContextInner::new("acct".to_string()));-- FS_CTX- .scope(ctx.clone(), async {- primary- .ensure_parent_dirs("/local/acct/docs/.sync_log.json", 0o755)- .await?;- primary- .write(- "/local/acct/docs/.sync_log.json",- b"{not valid json",- 0,- WriteFlag::Create,- )- .await?;- Ok::<(), Error>(())- })- .await- .unwrap();-- let result = store.get_sync_log_meta("/local/acct/docs", &ctx).await;- assert!(result.is_err(), "corrupted metadata must fail fast");- }-- #[tokio::test]- async fn test_next_seq_uses_dedicated_global_lock() {- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());- let store = MetaStateStore::new(primary, Arc::new(DefaultFsContextResolver));-- assert_eq!(store.next_seq().await.unwrap(), 1);- assert_eq!(store.next_seq().await.unwrap(), 2);-- let dir_locks = store.dir_locks.lock().await;- assert!(- !dir_locks.contains_key(GLOBAL_STATE_FILE),- "global sequence locking should not occupy the directory lock pool"- );- }}diff --git a/crates/ragfs/src/multibackend/migration/mod.rs b/crates/ragfs/src/multibackend/migration/mod.rsnew file mode 100644index 000000000..2db469c39--- /dev/null+++ b/crates/ragfs/src/multibackend/migration/mod.rs@@ -0,0 +1,231 @@+//! V2 protocol initialization and resumable full-data import.++use std::sync::Arc;++use uuid::Uuid;++use crate::core::context::{FsContextInner, FS_CTX};+use crate::core::errors::{Error, Result};+use crate::core::filesystem::FileSystem;+use crate::core::internal_names::is_multiwrite_internal_path;+use crate::multibackend::constants::{MULTIWRITE_MOUNT_PREFIX, SYSTEM_DIR};+use crate::multibackend::meta::MetadataStore;+use crate::multibackend::model::{+ PendingEvent, PendingEventKind, ProtocolState, ProtocolStatus, ProtocolVersion,+ SegmentEventType,+};+use crate::multibackend::provider::{bootstrap_account, MultiWriteProvider};+use crate::multibackend::runtime::read_active_accounts;++/// Reads or initializes the persisted V2 multi-write protocol state.+pub struct ProtocolDetector {+ store: Arc<MetadataStore>,+}++impl ProtocolDetector {+ /// Create a detector backed by the supplied raw-primary metadata store.+ pub fn new(store: Arc<MetadataStore>) -> Self {+ Self { store }+ }++ /// Read the protocol state, creating V2 migrating state when absent.+ pub async fn detect(&self) -> Result<ProtocolState> {+ let (protocol_path, _) = self.store.paths().mount_protocol();+ match self.store.read_bytes(&protocol_path).await {+ Ok(bytes) => decode_protocol(&bytes),+ Err(Error::NotFound(_)) => {+ let state = ProtocolState {+ protocol_version: ProtocolVersion::V2,+ status: ProtocolStatus::Migrating,+ };+ let bytes = serde_json::to_vec(&state)?;+ match self.store.write_immutable(&protocol_path, &bytes).await {+ Ok(()) => Ok(state),+ Err(Error::AlreadyExists(_)) => {+ decode_protocol(&self.store.read_bytes(&protocol_path).await?)+ }+ Err(error) => Err(error),+ }+ }+ Err(error) => Err(error),+ }+ }+}++/// Imports current primary files into V2 segment metadata.+pub struct FullDataImporter {+ primary: Arc<dyn FileSystem>,+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+ initial_partitions: u32,+ backup_names: Vec<String>,+}++impl FullDataImporter {+ /// Create an importer for one primary, metadata store, and persistence provider.+ pub fn new(+ primary: Arc<dyn FileSystem>,+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+ initial_partitions: u32,+ backup_names: Vec<String>,+ ) -> Self {+ Self {+ primary,+ store,+ provider,+ initial_partitions,+ backup_names,+ }+ }++ /// Import every current account file and atomically mark V2 stable.+ pub async fn run(&self) -> Result<()> {+ let (protocol_path, _) = self.store.paths().mount_protocol();+ let migrating_bytes = self.store.read_bytes(&protocol_path).await?;+ if decode_protocol(&migrating_bytes)?.status == ProtocolStatus::Stable {+ return Ok(());+ }++ // ponytail: fixed delay assumes account bootstrap finishes within 10s; use readiness signaling if that changes.+ tokio::time::sleep(std::time::Duration::from_secs(10)).await;+ let mut accounts = read_active_accounts(self.primary.as_ref())+ .await?+ .into_iter()+ .collect::<Vec<_>>();+ accounts.sort();+ self.import_scope(SYSTEM_DIR).await?;+ for account in &accounts {+ if !read_active_accounts(self.primary.as_ref())+ .await?+ .contains(account)+ {+ continue;+ }+ self.import_scope(account).await?;+ }+ self.cleanup_v1_metadata().await?;++ let stable_bytes = serde_json::to_vec(&ProtocolState {+ protocol_version: ProtocolVersion::V2,+ status: ProtocolStatus::Stable,+ })?;+ if self+ .store+ .compare_protocol(&migrating_bytes, &stable_bytes)+ .await?+ {+ return Ok(());+ }+ if decode_protocol(&self.store.read_bytes(&protocol_path).await?)?.status+ == ProtocolStatus::Stable+ {+ Ok(())+ } else {+ Err(Error::invalid_operation(+ "multi-write protocol changed during full-data import",+ ))+ }+ }++ /// Initialize one V2 scope and import its current file states.+ async fn import_scope(&self, account: &str) -> Result<()> {+ let partitions = self+ .store+ .initialize_account(account, self.initial_partitions, &self.backup_names)+ .await?;+ bootstrap_account(+ self.provider.as_ref(),+ self.store.as_ref(),+ account,+ &partitions,+ )+ .await?;+ let context = Arc::new(FsContextInner::new(account).with_auto_pathlock_disabled());+ FS_CTX.scope(context, self.import_account(account)).await+ }++ /// Scan one account through the business filesystem.+ async fn import_account(&self, account: &str) -> Result<()> {+ let root = format!("/{account}");+ let mut pending = vec![root];+ while let Some(directory) = pending.pop() {+ let mut entries = self.primary.read_internal_dir(&directory).await?;+ entries.sort_by(|left, right| left.name.cmp(&right.name));+ for entry in entries {+ let path = child_path(&directory, &entry.name);+ let logical_path = format!("{MULTIWRITE_MOUNT_PREFIX}{path}");+ if is_multiwrite_internal_path(&logical_path) {+ continue;+ }+ if entry.is_dir {+ pending.push(path);+ continue;+ }+ self.provider+ .flush(+ None,+ vec![PendingEvent {+ operation_id: Uuid::new_v4(),+ account_id: account.to_string(),+ kind: PendingEventKind::Data(SegmentEventType::Write),+ path: logical_path,+ destination_path: None,+ route_hint: None,+ }],+ )+ .await?;+ }+ }+ Ok(())+ }++ /// Remove V1 metadata through the configured primary filesystem.+ /// Returns an error when scanning or primary removal fails.+ async fn cleanup_v1_metadata(&self) -> Result<()> {+ match self.primary.stat("/_system/.multiwrite.global.json").await {+ Ok(_) => {}+ Err(Error::NotFound(_)) => return Ok(()),+ Err(error) => return Err(error),+ }++ let page = self+ .primary+ .glob_directory("/", "**/.sync_log.json", true, None, None, None)+ .await?;+ for entry in page.entries {+ if entry.is_dir {+ continue;+ }+ match self.primary.remove(&entry.path).await {+ Ok(()) | Err(Error::NotFound(_)) => {}+ Err(error) => return Err(error),+ }+ }++ match self+ .primary+ .remove("/_system/.multiwrite.global.json")+ .await+ {+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),+ Err(error) => Err(error),+ }+ }+}++/// Deserialize and validate exact V2 protocol bytes.+fn decode_protocol(bytes: &[u8]) -> Result<ProtocolState> {+ let state: ProtocolState = serde_json::from_slice(bytes)?;+ state.validate()?;+ Ok(state)+}++/// Join one direct child name to a canonical mount-relative directory.+fn child_path(directory: &str, name: &str) -> String {+ if directory == "/" {+ format!("/{name}")+ } else {+ format!("{directory}/{name}")+ }+}diff --git a/crates/ragfs/src/multibackend/mod.rs b/crates/ragfs/src/multibackend/mod.rsindex 44e8e4ab2..569a2225a 100644--- a/crates/ragfs/src/multibackend/mod.rs+++ b/crates/ragfs/src/multibackend/mod.rs@@ -1,14 +1,32 @@-//! Multi-backend configuration, factory, metadata, and admin support.+//! Multi-backend configuration, factory, and metadata support.-/// Multi-backend administrative APIs.-pub mod admin;+/// Prefix reconciliation primitives for backup catch-up.+pub mod catch_up;+/// Checkpoint construction, validation, publication, and reading.+pub mod checkpoint;+/// Stable binary codecs for V2 segments and checkpoints.+pub mod codec;/// Multi-backend config validation and normalization helpers.pub mod config;+/// Shared constants for V2 multi-write metadata.+pub(crate) mod constants;/// Multi-backend runtime assembly from validated config.pub mod factory;+/// Metadata garbage collection.+pub mod gc;/// Metadata state management shared by multi-backend runtime paths.pub mod meta;+/// V2 protocol initialization and full-data import.+pub mod migration;+/// V2 multi-write persistence and in-memory models.+pub mod model;+/// V2 multi-write persistence provider interfaces and implementations.+pub mod provider;+/// Logical path mapping and account partition routing.+pub mod router;+/// Foreground event queue and metadata flush worker.+pub mod runtime;/// Shared build-time types for multi-backend assembly.pub mod types;-pub use meta::{FsContextResolver, MetaStateStore};+pub use meta::FsContextResolver;diff --git a/crates/ragfs/src/multibackend/model.rs b/crates/ragfs/src/multibackend/model.rsnew file mode 100644index 000000000..85b782d0f--- /dev/null+++ b/crates/ragfs/src/multibackend/model.rs@@ -0,0 +1,647 @@+//! V2 multi-write persistence and in-memory data models.+#![allow(missing_docs)]+use crate::core::errors::{Error, Result};+use crate::multibackend::constants::{+ CHECKPOINT_DIR_PREFIX, HEAD_SEGMENT_FILE_PREFIX, MAX_CHECKPOINT_NODES, MAX_SEGMENT_RECORDS,+ SEALED_SEGMENT_FILE_PREFIX, SEGMENT_FILE_EXTENSION,+};+use crate::multibackend::router::validate_routes;+use serde::{Deserialize, Serialize};+use std::collections::{BTreeMap, HashSet};+use uuid::Uuid;+macro_rules! ensure { ($condition:expr, $($arg:tt)*) => { if !$condition { return Err(Error::Serialization(format!($($arg)*))); } }; }+pub use crate::multibackend::constants::VBUCKETS;+const FORMAT_VERSION: u32 = 1;+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+#[serde(deny_unknown_fields)]+pub struct ProtocolState {+ pub protocol_version: ProtocolVersion,+ pub status: ProtocolStatus,+}+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]+pub enum ProtocolVersion {+ V2,+}+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]+#[serde(rename_all = "snake_case")]+pub enum ProtocolStatus {+ Migrating,+ Stable,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+#[serde(deny_unknown_fields)]+pub struct PartitionsManifest {+ pub version: u32,+ pub epoch: u64,+ pub vbuckets: u32,+ pub partitions: BTreeMap<u32, PartitionEntry>,+ pub routes: Vec<RouteEntry>,+ pub directory_events: Vec<DirectoryEvent>,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct PartitionEntry {+ pub state: PartitionState,+}+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]+pub enum PartitionState {+ Stable,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct RouteEntry {+ pub start: u32,+ pub end: u32,+ pub partition: u32,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct PartitionManifest {+ pub version: u32,+ pub partition_id: u32,+ pub epoch: u64,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct DirectoryEvent {+ pub op_id: Uuid,+ pub operation: DirectoryOperation,+ pub source_path: String,+ #[serde(skip_serializing_if = "Option::is_none")]+ pub destination_path: Option<String>,+ pub positions: Vec<MarkerPosition>,+}+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]+pub enum DirectoryOperation {+ RemoveTree,+ MoveTree,+ DeleteScope,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct MarkerPosition {+ pub partition_id: u32,+ pub epoch: u64,+ pub seq: u64,+}+#[derive(Debug, Clone, Hash, Serialize, Deserialize, PartialEq, Eq)]+pub struct ScopeKey {+ pub account_id: String,+ pub partition_id: u32,+ pub epoch: u64,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct ScopedSeq {+ pub scope: ScopeKey,+ pub seq: u64,+}+#[derive(Debug, Clone)]+pub struct PartitionContext {+ pub scope: ScopeKey,+ pub segment_dir: String,+}+#[derive(Debug, Clone)]+pub struct PendingEvent {+ pub operation_id: Uuid,+ pub account_id: String,+ pub kind: PendingEventKind,+ pub path: String,+ pub destination_path: Option<String>,+ pub route_hint: Option<PartitionContext>,+}+#[derive(Debug, Clone)]+pub enum PendingEventKind {+ Data(SegmentEventType),+ DeleteAccount,+}+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]+#[repr(u8)]+pub enum SegmentEventType {+ Write = 1,+ Remove = 2,+ RemoveTree = 3,+ MoveTree = 4,+ DeleteScope = 5,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct SegmentRecord {+ pub seq: u64,+ pub event_type: SegmentEventType,+ pub path: String,+ pub op_id: Option<Uuid>,+ pub destination_path: Option<String>,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct SegmentManifest {+ pub version: u32,+ pub next_seq: u64,+ pub segments: Vec<SegmentDescriptor>,+ pub backend_states: BTreeMap<String, BackendState>,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct SegmentDescriptor {+ pub path: String,+ pub segment_from_seq: u64,+ pub segment_to_seq: Option<u64>,+ pub record_count: u32,+ pub byte_len: u64,+ pub checksum: Option<String>,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct BackendState {+ pub synced_seq: u64,+}+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct FlushResult {+ pub scope: ScopeKey,+ pub first_seq: u64,+ pub last_seq: u64,+ pub sealed_segments: Vec<SegmentDescriptor>,+}+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct DecodedSegment {+ pub records: Vec<SegmentRecord>,+ pub valid_bytes: u64,+ pub truncated_tail: bool,+}+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct FileState {+ pub path: String,+ pub latest_seq: ScopedSeq,+ pub deleted: Option<bool>,+}+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct CheckpointNode {+ pub path_fragment: String,+ pub latest_seq: Option<u64>,+ pub deleted: Option<bool>,+ pub children: Vec<CheckpointNode>,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct CheckpointsManifest {+ pub version: u32,+ #[serde(deserialize_with = "Option::deserialize")]+ pub latest_checkpoint: Option<LatestCheckpoint>,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct LatestCheckpoint {+ pub path: String,+ pub checkpoint_to_seq: u64,+ pub updated_at_ns: u64,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct CheckpointManifest {+ pub version: u32,+ pub partition_id: u32,+ pub epoch: u64,+ pub checkpoint_from_seq: u64,+ pub checkpoint_to_seq: u64,+ pub created_at_ns: u64,+ pub root: ChunkDescriptor,+ pub checksum: String,+}+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct ChunkDescriptor {+ pub file: String,+ pub root_path: String,+ pub start_path: Option<String>,+ pub end_path: Option<String>,+ pub file_state_count: u32,+ pub checksum: String,+ pub chunks: Vec<ChunkDescriptor>,+}+impl ProtocolState {+ /// Validate the closed protocol version value.+ pub fn validate(&self) -> Result<()> {+ Ok(())+ }+}+impl PartitionsManifest {+ /// Validate routing and directory marker invariants.+ pub fn validate(&self) -> Result<()> {+ validate_version("partitions manifest", self.version)?;+ ensure!(self.epoch > 0, "partitions epoch must be positive");+ ensure!(self.vbuckets == VBUCKETS, "invalid vbucket count");+ ensure!(!self.partitions.is_empty(), "partitions must not be empty");+ for entry in self.partitions.values() {+ entry.validate()?;+ }+ validate_routes(&self.routes, self.partitions.keys().copied())?;+ let mut operation_ids = HashSet::new();+ for event in &self.directory_events {+ ensure!(+ operation_ids.insert(event.op_id),+ "duplicate directory op_id"+ );+ event.validate()?;+ for position in &event.positions {+ ensure!(+ position.epoch == self.epoch,+ "marker epoch does not match manifest"+ );+ ensure!(+ self.partitions+ .get(&position.partition_id)+ .is_some_and(|entry| entry.state == PartitionState::Stable),+ "marker partition is unavailable"+ );+ }+ }+ ensure!(+ self.partitions+ .values()+ .all(|entry| entry.state == PartitionState::Stable),+ "partition manifest contains an unsupported state"+ );+ Ok(())+ }+}+impl PartitionEntry {+ /// Validate the closed partition state value.+ pub fn validate(&self) -> Result<()> {+ Ok(())+ }+}+impl RouteEntry {+ /// Validate one inclusive vbucket route range.+ pub fn validate(&self) -> Result<()> {+ ensure!(+ self.start <= self.end && self.end < VBUCKETS,+ "invalid route range"+ );+ Ok(())+ }+}+impl PartitionManifest {+ /// Validate one partition manifest version and epoch.+ pub fn validate(&self) -> Result<()> {+ validate_version("partition manifest", self.version)?;+ ensure!(self.epoch > 0, "partition epoch must be positive");+ Ok(())+ }+}+impl DirectoryEvent {+ /// Validate one directory operation and its marker positions.+ pub fn validate(&self) -> Result<()> {+ validate_path(&self.source_path)?;+ match self.operation {+ DirectoryOperation::RemoveTree | DirectoryOperation::DeleteScope => ensure!(+ self.destination_path.is_none(),+ "remove tree has a destination"+ ),+ DirectoryOperation::MoveTree => {+ validate_path(self.destination_path.as_deref().ok_or_else(|| {+ Error::Serialization("move tree requires a destination".into())+ })?)?+ }+ }+ let mut partitions = HashSet::new();+ for position in &self.positions {+ position.validate()?;+ ensure!(+ partitions.insert(position.partition_id),+ "duplicate marker partition"+ );+ }+ Ok(())+ }+}+impl MarkerPosition {+ /// Validate one persisted directory marker position.+ pub fn validate(&self) -> Result<()> {+ ensure!(self.epoch > 0 && self.seq > 0, "invalid marker position");+ Ok(())+ }+}+impl ScopeKey {+ /// Validate an account-scoped partition identity.+ pub fn validate(&self) -> Result<()> {+ ensure!(+ !self.account_id.trim().is_empty() && self.epoch > 0,+ "invalid scope"+ );+ Ok(())+ }+ /// Validate this scope against its owning partitions manifest.+ pub fn validate_against_manifest(&self, manifest: &PartitionsManifest) -> Result<()> {+ self.validate()?;+ ensure!(+ manifest.partitions.contains_key(&self.partition_id),+ "scope partition is unknown"+ );+ ensure!(+ self.epoch == manifest.epoch,+ "scope epoch does not match manifest"+ );+ Ok(())+ }+}+impl ScopedSeq {+ /// Validate a scoped sequence value.+ pub fn validate(&self) -> Result<()> {+ self.scope.validate()?;+ ensure!(self.seq > 0, "scoped sequence must be positive");+ Ok(())+ }+}+impl SegmentRecord {+ /// Validate a persisted segment record.+ pub fn validate(&self) -> Result<()> {+ ensure!(self.seq > 0, "record sequence must be positive");+ validate_path(&self.path)?;+ match self.event_type {+ SegmentEventType::MoveTree => {+ ensure!(self.op_id.is_some(), "move marker requires an op_id");+ validate_path(self.destination_path.as_deref().ok_or_else(|| {+ Error::Serialization("move marker requires a destination".into())+ })?)?;+ }+ SegmentEventType::RemoveTree | SegmentEventType::DeleteScope => {+ ensure!(self.op_id.is_some(), "remove marker requires an op_id");+ ensure!(+ self.destination_path.is_none(),+ "remove marker has a destination"+ );+ }+ _ => ensure!(+ self.destination_path.is_none(),+ "data event has a destination"+ ),+ }+ Ok(())+ }+}+impl SegmentManifest {+ /// Validate segment continuity and backend progress.+ pub fn validate(&self) -> Result<()> {+ validate_version("segment manifest", self.version)?;+ ensure!(self.next_seq > 0, "next sequence must be positive");+ let (mut next, mut paths) = (None, HashSet::new());+ for (index, segment) in self.segments.iter().enumerate() {+ segment.validate()?;+ ensure!(paths.insert(&segment.path), "duplicate segment path");+ if let Some(expected) = next {+ ensure!(segment.segment_from_seq == expected, "segment sequence gap");+ }+ ensure!(+ segment.segment_to_seq.is_some() || index + 1 == self.segments.len(),+ "head segment must be last"+ );+ next = segment+ .segment_from_seq+ .checked_add(u64::from(segment.record_count));+ ensure!(next.is_some(), "segment sequence overflow");+ }+ if let Some(expected) = next {+ ensure!(+ self.next_seq == expected,+ "next sequence does not follow segments"+ );+ }+ for (backend, state) in &self.backend_states {+ ensure!(!backend.trim().is_empty(), "backend id must not be empty");+ state.validate(self.next_seq - 1)?;+ }+ Ok(())+ }+}+impl SegmentDescriptor {+ /// Validate one segment summary.+ pub fn validate(&self) -> Result<()> {+ ensure!(+ !self.path.is_empty() && self.segment_from_seq > 0,+ "invalid segment identity"+ );+ ensure!(self.byte_len >= 6, "segment byte length is too small");+ match self.segment_to_seq {+ Some(to) => {+ ensure!(to >= self.segment_from_seq, "segment range is invalid");+ ensure!(+ self.path+ == format!(+ "{SEALED_SEGMENT_FILE_PREFIX}{:020}-{to:020}{SEGMENT_FILE_EXTENSION}",+ self.segment_from_seq+ ),+ "sealed segment path does not match range"+ );+ ensure!(+ u64::from(self.record_count) == to - self.segment_from_seq + 1,+ "sealed segment count does not match range"+ );+ ensure!(+ self.record_count <= MAX_SEGMENT_RECORDS as u32,+ "sealed segment has too many records"+ );+ validate_checksum(self.checksum.as_deref().ok_or_else(|| {+ Error::Serialization("sealed segment requires a checksum".into())+ })?)?;+ }+ None => {+ ensure!(+ self.path+ == format!(+ "{HEAD_SEGMENT_FILE_PREFIX}{:020}{SEGMENT_FILE_EXTENSION}",+ self.segment_from_seq+ ),+ "head segment path does not match start sequence"+ );+ ensure!(self.checksum.is_none(), "head segment has a checksum");+ }+ }+ ensure!(+ self.segment_from_seq+ .checked_add(u64::from(self.record_count))+ .is_some(),+ "segment sequence overflow"+ );+ Ok(())+ }+}+impl BackendState {+ /// Validate backend progress against the committed sequence.+ pub fn validate(&self, committed_seq: u64) -> Result<()> {+ ensure!(+ self.synced_seq <= committed_seq,+ "backend progress is ahead"+ );+ Ok(())+ }+}+impl FileState {+ /// Validate a checkpoint file state against its owning scope.+ pub fn validate(&self, expected_scope: &ScopeKey) -> Result<()> {+ validate_path(&self.path)?;+ self.latest_seq.validate()?;+ ensure!(+ &self.latest_seq.scope == expected_scope,+ "file state crosses checkpoint scope"+ );+ validate_tombstone(self.deleted)+ }+}+impl CheckpointNode {+ /// Validate one checkpoint radix tree and its tombstones.+ pub fn validate(&self) -> Result<()> {+ ensure!(+ self.latest_seq.is_none_or(|seq| seq > 0),+ "invalid checkpoint node sequence"+ );+ validate_tombstone(self.deleted)?;+ ensure!(+ self.deleted.is_none() || self.latest_seq.is_some(),+ "tombstone requires a sequence"+ );+ let mut previous = None;+ for child in &self.children {+ child.validate()?;+ ensure!(+ previous.is_none_or(|fragment| fragment < child.path_fragment.as_str()),+ "checkpoint children are not strictly ordered"+ );+ previous = Some(child.path_fragment.as_str());+ }+ Ok(())+ }+}+impl CheckpointsManifest {+ /// Validate the checkpoint publication pointer.+ pub fn validate(&self) -> Result<()> {+ validate_version("checkpoints manifest", self.version)?;+ if let Some(latest) = &self.latest_checkpoint {+ latest.validate()?;+ }+ Ok(())+ }+}+impl LatestCheckpoint {+ /// Validate one published checkpoint reference.+ pub fn validate(&self) -> Result<()> {+ let name = self.path.strip_suffix('/').ok_or_else(|| {+ Error::Serialization("checkpoint pointer must reference a directory".into())+ })?;+ ensure!(+ is_checkpoint_directory_name(name),+ "invalid checkpoint reference"+ );+ Ok(())+ }+}++/// Return whether a single directory name identifies a published checkpoint.+pub(crate) fn is_checkpoint_directory_name(name: &str) -> bool {+ !name.contains('/') && name.starts_with(CHECKPOINT_DIR_PREFIX)+}++impl CheckpointManifest {+ /// Validate a checkpoint manifest and its chunk tree.+ pub fn validate(&self) -> Result<()> {+ validate_version("checkpoint manifest", self.version)?;+ ensure!(self.epoch > 0, "checkpoint epoch must be positive");+ ensure!(+ self.checkpoint_from_seq == 1 && self.checkpoint_to_seq == 0+ || self.checkpoint_from_seq > 0+ && self.checkpoint_from_seq <= self.checkpoint_to_seq,+ "invalid checkpoint sequence range"+ );+ validate_checksum(&self.checksum)?;+ self.root.validate()+ }+ /// Validate this checkpoint against its owning scope.+ pub fn validate_for_scope(&self, scope: &ScopeKey) -> Result<()> {+ self.validate()?;+ scope.validate()?;+ ensure!(+ self.partition_id == scope.partition_id,+ "checkpoint partition does not match scope"+ );+ ensure!(+ self.epoch == scope.epoch,+ "checkpoint epoch does not match scope"+ );+ Ok(())+ }+}+impl ChunkDescriptor {+ /// Validate a checkpoint chunk tree, including unique files and ranges.+ pub fn validate(&self) -> Result<()> {+ self.validate_inner(true, &mut HashSet::new())+ }+ /// Validate one chunk and recurse into its direct children.+ fn validate_inner(&self, root: bool, files: &mut HashSet<String>) -> Result<()> {+ ensure!(+ !self.file.is_empty() && files.insert(self.file.clone()),+ "invalid chunk file"+ );+ validate_path(&self.root_path)?;+ ensure!(+ self.file_state_count <= MAX_CHECKPOINT_NODES as u32,+ "chunk has too many file states"+ );+ validate_checksum(&self.checksum)?;+ let (range_start, range_end) =+ match (root, self.start_path.as_deref(), self.end_path.as_deref()) {+ (true, None, None) => (self.root_path.as_str(), None),+ (true, _, _) => {+ return Err(Error::Serialization(+ "root chunk must not have bounds".into(),+ ));+ }+ (false, Some(start), end) => {+ if let Some(end) = end {+ ensure!(start < end, "chunk range is reversed");+ }+ (start, end)+ }+ (false, None, _) => {+ return Err(Error::Serialization(+ "non-root chunk requires start bound".into(),+ ));+ }+ };+ let mut expected_start = Some(range_start);+ for child in &self.chunks {+ child.validate_inner(false, files)?;+ let child_start = child.start_path.as_deref().unwrap();+ ensure!(+ expected_start.is_some_and(|start| start == child_start),+ "chunk ranges are not contiguous"+ );+ expected_start = child.end_path.as_deref();+ }+ if !self.chunks.is_empty() {+ ensure!(+ expected_start == range_end,+ "chunk ranges are not contiguous"+ );+ }+ Ok(())+ }+}+/// Validate a supported persisted format version.+fn validate_version(name: &str, version: u32) -> Result<()> {+ ensure!(+ version == FORMAT_VERSION,+ "unsupported {name} version {version}"+ );+ Ok(())+}+/// Validate a complete logical path.+fn validate_path(path: &str) -> Result<()> {+ ensure!(path.starts_with('/'), "logical path must be absolute");+ Ok(())+}+/// Validate a lowercase SHA-256 hexadecimal checksum.+fn validate_checksum(checksum: &str) -> Result<()> {+ ensure!(+ checksum.len() == 64+ && checksum+ .bytes()+ .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),+ "invalid SHA-256 checksum"+ );+ Ok(())+}+/// Validate an optional deletion tombstone.+fn validate_tombstone(deleted: Option<bool>) -> Result<()> {+ ensure!(deleted != Some(false), "tombstone may only be true");+ Ok(())+}++#[cfg(test)]+#[path = "tests/test_model.rs"]+mod tests;diff --git a/crates/ragfs/src/multibackend/provider/cache.rs b/crates/ragfs/src/multibackend/provider/cache.rsnew file mode 100644index 000000000..88161ba58--- /dev/null+++ b/crates/ragfs/src/multibackend/provider/cache.rs@@ -0,0 +1,1555 @@+//! CacheRuntime-backed V2 multi-write persistence.++use std::collections::{BTreeMap, HashSet};+use std::sync::Arc;++use async_trait::async_trait;+use bytes::Bytes;+use serde::Deserialize;+use uuid::Uuid;++use crate::cache_runtime::{+ CacheError, CacheOperation, CacheRuntime, ScriptDefinition, ScriptRequest, ScriptValue,+};+use crate::core::errors::{Error, Result};+use crate::core::internal_names::is_multiwrite_internal_path;+use crate::multibackend::codec::{decode_segment, encode_segment, sha256_hex};+use crate::multibackend::constants::{+ HEAD_SEGMENT_FILE_PREFIX, MAX_SEGMENT_RECORDS, SEALED_SEGMENT_FILE_PREFIX,+ SEGMENT_FILE_EXTENSION,+};+use crate::multibackend::meta::MetadataStore;+use crate::multibackend::model::{+ BackendState, CheckpointsManifest, FlushResult, MarkerPosition, PartitionContext,+ PartitionsManifest, PendingEvent, PendingEventKind, ScopeKey, SegmentDescriptor,+ SegmentEventType, SegmentManifest, SegmentRecord,+};+use crate::multibackend::router::AccountRouter;++use super::{read_sealed_segment, select_record_range, MultiWriteProvider};++const FLUSH_SCRIPT_ID: &str = "multiwrite.flush.v1";+const SEAL_SCRIPT_ID: &str = "multiwrite.seal.v1";+const ADVANCE_SCRIPT_ID: &str = "multiwrite.advance.v1";++const FLUSH_SCRIPT: &str = r#"+if redis.call("EXISTS", KEYS[1]) == 0 then+ return { "not_found" }+end+if redis.call("HGET", KEYS[1], "account") ~= ARGV[1]+ or redis.call("HGET", KEYS[1], "partition") ~= ARGV[2]+ or redis.call("HGET", KEYS[1], "epoch") ~= ARGV[3] then+ return { "scope_mismatch" }+end+local requested = #ARGV - 6+if ARGV[4] == "1" and requested ~= 1 then+ return { "invalid", "marker flush requires exactly one record" }+end+local owner = redis.call("GET", KEYS[3])+if ARGV[6] == "1" then+ if owner ~= ARGV[5] then+ return { "fenced" }+ end+elseif owner and owner ~= ARGV[5] then+ return { "busy" }+else+ redis.call("SET", KEYS[3], ARGV[5], "PX", 30000)+end+redis.call("PEXPIRE", KEYS[3], 30000)+local marker_seq = tonumber(redis.call("HGET", KEYS[1], "marker_seq"))+if marker_seq ~= 0 then+ return { "seal_required" }+end+local head_len = redis.call("LLEN", KEYS[2])+local capacity = 8192 - head_len+if capacity <= 0 then+ return { "seal_required" }+end+local count = math.min(requested, capacity)+local next_seq = tonumber(redis.call("HGET", KEYS[1], "next_seq"))+local first_seq = next_seq+for index = 1, count do+ local record = cjson.decode(ARGV[index + 6])+ record.seq = next_seq+ redis.call("RPUSH", KEYS[2], cjson.encode(record))+ next_seq = next_seq + 1+end+if ARGV[4] == "1" then+ marker_seq = next_seq - 1+end+local next_seq_text = string.format("%.0f", next_seq)+local marker_seq_text = string.format("%.0f", marker_seq)+redis.call("HSET", KEYS[1], "next_seq", next_seq_text,+ "marker_seq", marker_seq_text)+local needs_seal = 0+if redis.call("LLEN", KEYS[2]) == 8192 or marker_seq ~= 0 then+ needs_seal = 1+end+if needs_seal == 0 then+ redis.call("DEL", KEYS[3])+else+ redis.call("PEXPIRE", KEYS[3], 30000)+end+return { "ok", string.format("%.0f", first_seq),+ string.format("%.0f", next_seq - 1), count, needs_seal }+"#;++const SEAL_SCRIPT: &str = r#"+local mode = ARGV[1]+if mode == "bootstrap" then+ if redis.call("EXISTS", KEYS[1]) ~= 0 then+ return { "exists" }+ end+ redis.call("DEL", KEYS[1])+ redis.call("HSET", KEYS[1],+ "account", ARGV[2],+ "partition", ARGV[3],+ "epoch", ARGV[4],+ "version", ARGV[5],+ "next_seq", ARGV[6],+ "marker_seq", "0",+ "segments", ARGV[7],+ "backend_states", ARGV[8])+ redis.call("DEL", KEYS[2])+ redis.call("DEL", KEYS[3])+ return { "ok" }+end+if redis.call("EXISTS", KEYS[1]) == 0 then+ return { "not_found" }+end+if redis.call("HGET", KEYS[1], "account") ~= ARGV[2]+ or redis.call("HGET", KEYS[1], "partition") ~= ARGV[3]+ or redis.call("HGET", KEYS[1], "epoch") ~= ARGV[4] then+ return { "scope_mismatch" }+end+if mode == "read" then+ return {+ "ok",+ redis.call("HGET", KEYS[1], "account"),+ redis.call("HGET", KEYS[1], "partition"),+ redis.call("HGET", KEYS[1], "epoch"),+ redis.call("HGET", KEYS[1], "version"),+ redis.call("HGET", KEYS[1], "next_seq"),+ redis.call("HGET", KEYS[1], "marker_seq"),+ redis.call("HGET", KEYS[1], "segments"),+ redis.call("HGET", KEYS[1], "backend_states")+ }+end+local segments = cjson.decode(redis.call("HGET", KEYS[1], "segments"))+if mode == "commit" then+ local first_seq = tonumber(ARGV[5])+ local last_seq = tonumber(ARGV[6])+ local count = tonumber(ARGV[7])+ local descriptor = cjson.decode(ARGV[8])+ local owner = ARGV[9]+ local keep_owner = ARGV[10]+ for _, current in ipairs(segments) do+ if current.path == descriptor.path then+ if current.segment_from_seq == descriptor.segment_from_seq+ and current.segment_to_seq == descriptor.segment_to_seq+ and current.record_count == descriptor.record_count+ and current.byte_len == descriptor.byte_len+ and current.checksum == descriptor.checksum then+ if owner ~= "" and redis.call("GET", KEYS[3]) == owner then+ if keep_owner == "1" then+ redis.call("PEXPIRE", KEYS[3], 30000)+ else+ redis.call("DEL", KEYS[3])+ end+ end+ return { "already" }+ end+ return { "invalid", "sealed descriptor path conflicts" }+ end+ end+ local current_owner = redis.call("GET", KEYS[3])+ if owner == "" then+ if current_owner then+ return { "busy" }+ end+ elseif current_owner ~= owner then+ return { "fenced" }+ end+ if #segments > 0+ and segments[#segments].segment_to_seq + 1 ~= first_seq then+ return { "invalid", "sealed descriptor is not contiguous" }+ end+ if count ~= last_seq - first_seq + 1 then+ return { "invalid", "sealed descriptor count is invalid" }+ end+ if redis.call("LLEN", KEYS[2]) < count then+ return { "changed" }+ end+ for index = 1, count do+ if redis.call("LINDEX", KEYS[2], index - 1) ~= ARGV[index + 10] then+ return { "changed" }+ end+ end+ segments[#segments + 1] = descriptor+ redis.call("HSET", KEYS[1], "segments", cjson.encode(segments))+ redis.call("LTRIM", KEYS[2], count, -1)+ local marker_seq = tonumber(redis.call("HGET", KEYS[1], "marker_seq"))+ if marker_seq ~= 0 and marker_seq <= last_seq then+ redis.call("HSET", KEYS[1], "marker_seq", "0")+ end+ if owner ~= "" then+ if keep_owner == "1" then+ redis.call("PEXPIRE", KEYS[3], 30000)+ else+ redis.call("DEL", KEYS[3])+ end+ end+ return { "ok" }+end+if mode == "prune" then+ local requested = cjson.decode(ARGV[5])+ local requested_set = {}+ for _, path in ipairs(requested) do+ if requested_set[path] then+ return { "invalid", "prune paths contain duplicates" }+ end+ requested_set[path] = true+ end+ local removed = {}+ local kept = {}+ for _, descriptor in ipairs(segments) do+ if requested_set[descriptor.path] then+ removed[#removed + 1] = descriptor+ requested_set[descriptor.path] = nil+ else+ kept[#kept + 1] = descriptor+ end+ end+ for path, _ in pairs(requested_set) do+ return { "invalid", "sealed segment is not referenced: " .. path }+ end+ for index = 2, #kept do+ if kept[index].segment_from_seq ~= kept[index - 1].segment_to_seq + 1 then+ return { "invalid", "prune would create a segment gap" }+ end+ end+ local head = redis.call("LINDEX", KEYS[2], 0)+ if head and #kept > 0 then+ local first_head = cjson.decode(head)+ if first_head.seq ~= kept[#kept].segment_to_seq + 1 then+ return { "invalid", "prune would create a head gap" }+ end+ elseif not head and #kept > 0 then+ local next_seq = tonumber(redis.call("HGET", KEYS[1], "next_seq"))+ if next_seq ~= kept[#kept].segment_to_seq + 1 then+ return { "invalid", "prune would detach next_seq" }+ end+ end+ local encoded_kept = "[]"+ if #kept > 0 then+ encoded_kept = cjson.encode(kept)+ end+ local encoded_removed = "[]"+ if #removed > 0 then+ encoded_removed = cjson.encode(removed)+ end+ redis.call("HSET", KEYS[1], "segments", encoded_kept)+ return { "ok", encoded_removed }+end+return { "invalid", "unknown seal operation" }+"#;++const ADVANCE_SCRIPT: &str = r#"+if redis.call("EXISTS", KEYS[1]) == 0 then+ return { "not_found" }+end+if redis.call("HGET", KEYS[1], "account") ~= ARGV[1]+ or redis.call("HGET", KEYS[1], "partition") ~= ARGV[2]+ or redis.call("HGET", KEYS[1], "epoch") ~= ARGV[3] then+ return { "scope_mismatch" }+end+local states = cjson.decode(redis.call("HGET", KEYS[1], "backend_states"))+local state = states[ARGV[4]]+if not state then+ return { "invalid", "unknown backend id: " .. ARGV[4] }+end+local synced_seq = tonumber(ARGV[5])+if synced_seq ~= 0 and synced_seq < state.synced_seq then+ return { "invalid", "backend progress cannot regress" }+end+local next_seq = tonumber(redis.call("HGET", KEYS[1], "next_seq"))+if synced_seq >= next_seq then+ return { "invalid", "backend progress must remain below next_seq" }+end+state.synced_seq = synced_seq+redis.call("HSET", KEYS[1], "backend_states", cjson.encode(states))+return { "ok" }+"#;++const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[+ ScriptDefinition {+ id: FLUSH_SCRIPT_ID,+ redis_lua: FLUSH_SCRIPT,+ },+ ScriptDefinition {+ id: SEAL_SCRIPT_ID,+ redis_lua: SEAL_SCRIPT,+ },+ ScriptDefinition {+ id: ADVANCE_SCRIPT_ID,+ redis_lua: ADVANCE_SCRIPT,+ },+];++/// Cache keys owned by one account partition.+pub(crate) struct CachePartitionKeys {+ pub(crate) state: String,+ pub(crate) head: String,+ pub(crate) flush_gate: String,+}++#[derive(Debug, Clone, PartialEq, Eq)]+struct CacheState {+ scope: ScopeKey,+ version: u32,+ next_seq: u64,+ marker_seq: u64,+ segments: Vec<SegmentDescriptor>,+ backend_states: BTreeMap<String, BackendState>,+}++struct FlushScriptResult {+ first_seq: u64,+ last_seq: u64,+ appended: usize,+ needs_seal: bool,+}++enum FlushScriptOutcome {+ Appended(FlushScriptResult),+ SealRequired,+ Busy,+}++/// Persists V2 mutable state through CacheRuntime and immutable segments on the primary filesystem.+pub struct CacheProvider {+ store: Arc<MetadataStore>,+ router: AccountRouter,+ runtime: Arc<CacheRuntime>,+ namespace: String,+}++impl CacheProvider {+ /// Creates a cache-backed metadata provider.+ pub fn new(+ store: Arc<MetadataStore>,+ runtime: Arc<CacheRuntime>,+ namespace: String,+ ) -> Result<Self> {+ Self::build(store, runtime, namespace)+ }++ /// Validates runtime capabilities and registers stable provider scripts.+ fn build(+ store: Arc<MetadataStore>,+ runtime: Arc<CacheRuntime>,+ namespace: String,+ ) -> Result<Self> {+ Self::validate_namespace(&namespace)?;+ runtime+ .require_operations(&[CacheOperation::Lrange, CacheOperation::ExecuteScript])+ .map_err(|error| {+ Error::config(format!(+ "Cache-backed multi-write provider is incompatible: {error}"+ ))+ })?;+ for definition in SCRIPT_DEFINITIONS {+ runtime.register_script(*definition).map_err(|error| {+ Error::config(format!(+ "Cache-backed multi-write script registration failed: {error}"+ ))+ })?;+ }+ Ok(Self {+ router: AccountRouter::new(store.clone()),+ store,+ runtime,+ namespace,+ })+ }++ /// Validate the Redis hash tag namespace used by multi-write cache keys.+ fn validate_namespace(namespace: &str) -> Result<()> {+ if namespace.trim().is_empty() || namespace.contains(['{', '}']) {+ return Err(Error::config(+ "cache-backed multi-write namespace must be non-empty and must not contain '{' or '}'",+ ));+ }+ Ok(())+ }++ /// Builds deterministic same-slot keys for one validated scope.+ pub(crate) fn keys_for_scope(&self, scope: &ScopeKey) -> Result<CachePartitionKeys> {+ scope.validate()?;+ if matches!(scope.account_id.as_str(), "." | "..")+ || scope.account_id.contains('/')+ || scope+ .account_id+ .bytes()+ .any(|byte| byte.is_ascii_control() || matches!(byte, b'{' | b'}'))+ {+ return Err(Error::invalid_path(+ "Cache-backed multi-write account id contains an invalid key character",+ ));+ }+ let prefix = format!(+ "{{{}}}:ov:multiwrite:{}:{}",+ self.namespace, scope.account_id, scope.partition_id+ );+ Ok(CachePartitionKeys {+ state: format!("{prefix}:state"),+ head: format!("{prefix}:head"),+ flush_gate: format!("{prefix}:flush-gate"),+ })+ }++ /// Validates normal flush input before routing or cache access.+ fn validate_flush_events(events: &[PendingEvent]) -> Result<()> {+ let first = events+ .first()+ .ok_or_else(|| Error::invalid_operation("flush requires at least one event"))?;+ for event in events {+ if event.account_id != first.account_id {+ return Err(Error::invalid_operation(+ "flush events must belong to one account",+ ));+ }+ if is_multiwrite_internal_path(&event.path) {+ return Err(Error::invalid_path(format!(+ "cannot flush internal path: {}",+ event.path+ )));+ }+ match event.kind {+ PendingEventKind::Data(SegmentEventType::Write | SegmentEventType::Remove) => {}+ _ => {+ return Err(Error::invalid_operation(+ "normal flush accepts only write and remove events",+ ))+ }+ }+ if event.destination_path.is_some() {+ return Err(Error::invalid_operation(+ "normal data event cannot have a destination",+ ));+ }+ }+ Ok(())+ }++ /// Routes every event and requires one current partition scope.+ async fn route_events(&self, events: &[PendingEvent]) -> Result<PartitionContext> {+ let mut selected = None;+ for event in events {+ let current = self.router.route(&event.account_id, &event.path).await?;+ if selected+ .as_ref()+ .is_some_and(|existing: &PartitionContext| existing.scope != current.scope)+ {+ return Err(Error::invalid_operation(+ "flush events route to different partition scopes",+ ));+ }+ selected.get_or_insert(current);+ }+ selected.ok_or_else(|| Error::invalid_operation("flush requires at least one event"))+ }++ /// Validates a scope against its current account partitions manifest.+ async fn validate_scope(&self, scope: &ScopeKey) -> Result<()> {+ self.keys_for_scope(scope)?;+ let path = self.store.paths().account_manifest(&scope.account_id)?.0;+ let manifest: PartitionsManifest = self.store.read_json(&path).await?;+ manifest.validate()?;+ scope.validate_against_manifest(&manifest)+ }++ /// Converts one pending event into a validated record template.+ fn record(event: &PendingEvent) -> Result<SegmentRecord> {+ let event_type = match event.kind {+ PendingEventKind::Data(kind) => kind,+ PendingEventKind::DeleteAccount => {+ return Err(Error::invalid_operation(+ "account deletion is not a segment record",+ ))+ }+ };+ let marker = matches!(+ event_type,+ SegmentEventType::RemoveTree+ | SegmentEventType::MoveTree+ | SegmentEventType::DeleteScope+ );+ let record = SegmentRecord {+ seq: 1,+ event_type,+ path: event.path.clone(),+ op_id: marker.then_some(event.operation_id),+ destination_path: event.destination_path.clone(),+ };+ record.validate()?;+ encode_segment(std::slice::from_ref(&record))?;+ Ok(record)+ }++ /// Validates one marker against its caller-selected scope.+ fn validate_marker(&self, scope: &ScopeKey, event: &PendingEvent) -> Result<()> {+ if event.account_id != scope.account_id {+ return Err(Error::invalid_operation(+ "marker account does not match supplied scope",+ ));+ }+ match event.kind {+ PendingEventKind::Data(+ SegmentEventType::RemoveTree+ | SegmentEventType::MoveTree+ | SegmentEventType::DeleteScope,+ ) => {}+ _ => {+ return Err(Error::invalid_operation(+ "append_marker accepts only directory markers",+ ))+ }+ }+ if is_multiwrite_internal_path(&event.path) {+ return Err(Error::invalid_path(format!(+ "cannot append marker for internal path: {}",+ event.path+ )));+ }+ if !matches!(+ event.kind,+ PendingEventKind::Data(SegmentEventType::DeleteScope)+ ) {+ self.store+ .paths()+ .backend_path(&scope.account_id, &event.path)?;+ }+ if let Some(destination) = &event.destination_path {+ if is_multiwrite_internal_path(destination) {+ return Err(Error::invalid_path(format!(+ "cannot append marker for internal destination: {destination}"+ )));+ }+ self.store+ .paths()+ .backend_path(&scope.account_id, destination)?;+ }+ Self::record(event).map(|_| ())+ }++ /// Executes one registered script and requires an array result.+ async fn execute_array(+ &self,+ script_id: &str,+ keys: &CachePartitionKeys,+ args: Vec<Bytes>,+ ) -> Result<Vec<ScriptValue>> {+ let result = self+ .runtime+ .execute_script(ScriptRequest {+ script_id: script_id.to_string(),+ keys: vec![+ keys.state.clone(),+ keys.head.clone(),+ keys.flush_gate.clone(),+ ],+ args,+ })+ .await+ .and_then(|result| result.decode())+ .map_err(|error| Self::cache_error(script_id, error))?;+ match result {+ ScriptValue::Array(values) => Ok(values),+ other => Err(Error::Serialization(format!(+ "Cache-backed multi-write {script_id} returned non-array result: {other:?}"+ ))),+ }+ }++ /// Executes one bounded atomic append attempt.+ async fn flush_once(+ &self,+ scope: &ScopeKey,+ records: &[SegmentRecord],+ marker: bool,+ owner: &str,+ continuing: bool,+ ) -> Result<FlushScriptOutcome> {+ let keys = self.keys_for_scope(scope)?;+ let mut args = Self::scope_args(scope);+ args.push(Bytes::from(if marker { "1" } else { "0" }));+ args.push(Bytes::from(owner.to_string()));+ args.push(Bytes::from(if continuing { "1" } else { "0" }));+ for record in records {+ args.push(Bytes::from(serde_json::to_vec(record)?));+ }+ let values = self.execute_array(FLUSH_SCRIPT_ID, &keys, args).await?;+ match Self::status(&values, "flush")? {+ "ok" => Ok(FlushScriptOutcome::Appended(FlushScriptResult {+ first_seq: Self::u64_value(&values, 1, "flush first_seq")?,+ last_seq: Self::u64_value(&values, 2, "flush last_seq")?,+ appended: Self::usize_value(&values, 3, "flush appended count")?,+ needs_seal: Self::u64_value(&values, 4, "flush seal flag")? == 1,+ })),+ "seal_required" => Ok(FlushScriptOutcome::SealRequired),+ "busy" => Ok(FlushScriptOutcome::Busy),+ "fenced" => Err(Error::would_block(+ "cache-backed flush owner was fenced by another writer",+ )),+ "not_found" => Err(Error::not_found(keys.state)),+ "scope_mismatch" => Err(Error::invalid_operation(+ "Cache-backed multi-write state scope does not match",+ )),+ "invalid" => Err(Self::script_invalid(&values, "flush")),+ other => Err(Self::unknown_status("flush", other)),+ }+ }++ /// Revalidates routing and executes one append while holding the account manifest lock.+ async fn flush_routed_once(+ &self,+ events: &[PendingEvent],+ expected_scope: Option<&ScopeKey>,+ records: &[SegmentRecord],+ owner: &str,+ continuing: bool,+ ) -> Result<(PartitionContext, FlushScriptOutcome)> {+ let account_id = &events+ .first()+ .ok_or_else(|| Error::invalid_operation("flush requires at least one event"))?+ .account_id;+ let manifest_path = self.store.paths().account_manifest(account_id)?.0;+ let lock_timeout = self+ .store+ .pathlock_manager()+ .default_lock_timeout()+ .max(std::time::Duration::from_secs(30));+ let lease = self+ .store+ .pathlock_manager()+ .acquire_exact(&manifest_path, lock_timeout, None)+ .await?;+ let operation = async {+ let route = self.route_events(events).await?;+ if expected_scope.is_some_and(|scope| scope != &route.scope) {+ return Err(Error::would_block(+ "multi-write route changed during cache-backed flush",+ ));+ }+ let outcome = self+ .flush_once(&route.scope, records, false, owner, continuing)+ .await?;+ Ok((route, outcome))+ }+ .await;+ let release = self+ .store+ .pathlock_manager()+ .release(&lease)+ .await+ .map_err(Error::from);+ match (operation, release) {+ (Err(error), _) => Err(error),+ (Ok(_), Err(error)) => Err(error),+ (Ok(result), Ok(())) => Ok(result),+ }+ }++ /// Revalidates a supplied scope and appends one marker under its manifest lock.+ async fn flush_marker_once(+ &self,+ scope: &ScopeKey,+ record: &SegmentRecord,+ owner: &str,+ continuing: bool,+ ) -> Result<FlushScriptOutcome> {+ let manifest_path = self.store.paths().account_manifest(&scope.account_id)?.0;+ let lock_timeout = self+ .store+ .pathlock_manager()+ .default_lock_timeout()+ .max(std::time::Duration::from_secs(30));+ let lease = self+ .store+ .pathlock_manager()+ .acquire_exact(&manifest_path, lock_timeout, None)+ .await?;+ let operation = async {+ self.validate_scope(scope).await?;+ self.flush_once(scope, std::slice::from_ref(record), true, owner, continuing)+ .await+ }+ .await;+ let release = self+ .store+ .pathlock_manager()+ .release(&lease)+ .await+ .map_err(Error::from);+ match (operation, release) {+ (Err(error), _) => Err(error),+ (Ok(_), Err(error)) => Err(error),+ (Ok(result), Ok(())) => Ok(result),+ }+ }++ /// Reads one stable cache state and head-list snapshot.+ async fn snapshot(+ &self,+ scope: &ScopeKey,+ ) -> Result<(CacheState, Vec<SegmentRecord>, Vec<Bytes>)> {+ self.validate_scope(scope).await?;+ let keys = self.keys_for_scope(scope)?;+ loop {+ let before = self.read_state_once(scope, &keys).await?;+ let raw_records = self+ .runtime+ .lrange(&keys.head, 0, -1)+ .await+ .map_err(|error| Self::cache_error("read head", error))?;+ let after = self.read_state_once(scope, &keys).await?;+ if before == after {+ let records = raw_records+ .iter()+ .map(|raw| {+ serde_json::from_slice::<SegmentRecord>(raw)+ .map_err(|error| Error::Serialization(error.to_string()))+ })+ .collect::<Result<Vec<_>>>()?;+ Self::validate_snapshot(&before, &records)?;+ return Ok((before, records, raw_records));+ }+ }+ }++ /// Reads one cache state value through the shared seal script.+ async fn read_state_once(+ &self,+ scope: &ScopeKey,+ keys: &CachePartitionKeys,+ ) -> Result<CacheState> {+ let mut args = vec![Bytes::from_static(b"read")];+ args.extend(Self::scope_args(scope));+ let values = self.execute_array(SEAL_SCRIPT_ID, keys, args).await?;+ match Self::status(&values, "read state")? {+ "ok" => {+ let state = CacheState {+ scope: ScopeKey {+ account_id: Self::string_value(&values, 1, "state account")?.to_string(),+ partition_id: Self::u32_value(&values, 2, "state partition")?,+ epoch: Self::u64_value(&values, 3, "state epoch")?,+ },+ version: Self::u32_value(&values, 4, "state version")?,+ next_seq: Self::u64_value(&values, 5, "state next_seq")?,+ marker_seq: Self::u64_value(&values, 6, "state marker_seq")?,+ segments: Self::json_value(&values, 7, "state segments")?,+ backend_states: Self::json_value(&values, 8, "state backend states")?,+ };+ Self::validate_state(scope, &state)?;+ Ok(state)+ }+ "not_found" => Err(Error::not_found(keys.state.clone())),+ "scope_mismatch" => Err(Error::invalid_operation(+ "Cache-backed multi-write state scope does not match",+ )),+ other => Err(Self::unknown_status("read state", other)),+ }+ }++ /// Validates cache state fields that do not depend on the mutable head.+ fn validate_state(scope: &ScopeKey, state: &CacheState) -> Result<()> {+ if &state.scope != scope {+ return Err(Error::Serialization(+ "Cache-backed multi-write state has the wrong scope".to_string(),+ ));+ }+ if state.version != 1 || state.next_seq == 0 {+ return Err(Error::Serialization(+ "Cache-backed multi-write state has invalid version or next_seq".to_string(),+ ));+ }+ let mut previous_to = None;+ let mut paths = HashSet::new();+ for descriptor in &state.segments {+ descriptor.validate()?;+ if descriptor.segment_to_seq.is_none() || !paths.insert(&descriptor.path) {+ return Err(Error::Serialization(+ "cache state contains an invalid sealed descriptor".to_string(),+ ));+ }+ if previous_to.is_some_and(|to| descriptor.segment_from_seq != to + 1) {+ return Err(Error::Serialization(+ "cache sealed descriptors are not contiguous".to_string(),+ ));+ }+ previous_to = descriptor.segment_to_seq;+ }+ for (backend, backend_state) in &state.backend_states {+ if backend.trim().is_empty() {+ return Err(Error::Serialization(+ "cache backend id must not be empty".to_string(),+ ));+ }+ backend_state.validate(state.next_seq - 1)?;+ }+ if state.marker_seq >= state.next_seq {+ return Err(Error::Serialization(+ "cache marker is ahead of next_seq".to_string(),+ ));+ }+ Ok(())+ }++ /// Validates head ordering and its continuity with cache manifest state.+ fn validate_snapshot(state: &CacheState, records: &[SegmentRecord]) -> Result<()> {+ if records.len() > MAX_SEGMENT_RECORDS {+ return Err(Error::Serialization(+ "cache head contains too many records".to_string(),+ ));+ }+ for (index, record) in records.iter().enumerate() {+ record.validate()?;+ if index > 0 && record.seq != records[index - 1].seq + 1 {+ return Err(Error::Serialization(+ "cache head contains a sequence gap".to_string(),+ ));+ }+ }+ if let Some(first) = records.first() {+ if state+ .segments+ .last()+ .and_then(|descriptor| descriptor.segment_to_seq)+ .is_some_and(|last| first.seq != last + 1)+ {+ return Err(Error::Serialization(+ "cache head does not follow sealed segments".to_string(),+ ));+ }+ if records.last().unwrap().seq.checked_add(1) != Some(state.next_seq) {+ return Err(Error::Serialization(+ "cache head does not end at next_seq".to_string(),+ ));+ }+ } else if state+ .segments+ .last()+ .and_then(|descriptor| descriptor.segment_to_seq)+ .is_some_and(|last| last.checked_add(1) != Some(state.next_seq))+ {+ return Err(Error::Serialization(+ "cache next_seq does not follow sealed segments".to_string(),+ ));+ }+ if state.marker_seq != 0+ && records.last().map(|record| record.seq) != Some(state.marker_seq)+ {+ return Err(Error::Serialization(+ "cache marker is not the final head record".to_string(),+ ));+ }+ Ok(())+ }++ /// Converts a stable snapshot into the public validated manifest.+ fn manifest(state: CacheState, records: &[SegmentRecord]) -> Result<SegmentManifest> {+ let mut manifest = SegmentManifest {+ version: state.version,+ next_seq: state.next_seq,+ segments: state.segments,+ backend_states: state.backend_states,+ };+ if let Some(first) = records.first() {+ let bytes = encode_segment(records)?;+ manifest.segments.push(SegmentDescriptor {+ path: Self::head_name(first.seq),+ segment_from_seq: first.seq,+ segment_to_seq: None,+ record_count: records.len() as u32,+ byte_len: bytes.len() as u64,+ checksum: None,+ });+ }+ manifest.validate()?;+ Ok(manifest)+ }++ /// Persists one immutable segment snapshot on the shared raw primary.+ async fn persist_sealed(+ &self,+ scope: &ScopeKey,+ records: &[SegmentRecord],+ ) -> Result<SegmentDescriptor> {+ let first = records+ .first()+ .ok_or_else(|| Error::invalid_operation("cannot seal an empty cache head"))?+ .seq;+ let last = records.last().unwrap().seq;+ let bytes = encode_segment(records)?;+ let descriptor = SegmentDescriptor {+ path: Self::sealed_name(first, last),+ segment_from_seq: first,+ segment_to_seq: Some(last),+ record_count: records.len() as u32,+ byte_len: bytes.len() as u64,+ checksum: Some(sha256_hex(&bytes)),+ };+ let directory = self+ .store+ .paths()+ .segments_dir(&scope.account_id, scope.partition_id)?+ .0;+ self.store+ .write_immutable(&format!("{directory}/{}", descriptor.path), &bytes)+ .await?;+ Ok(descriptor)+ }++ /// Commits one immutable descriptor and trims its exact cache head prefix.+ pub(crate) async fn commit_seal(+ &self,+ scope: &ScopeKey,+ raw_records: &[Bytes],+ descriptor: &SegmentDescriptor,+ owner: Option<&str>,+ keep_owner: bool,+ ) -> Result<()> {+ let keys = self.keys_for_scope(scope)?;+ let mut args = vec![Bytes::from_static(b"commit")];+ args.extend(Self::scope_args(scope));+ args.push(Bytes::from(descriptor.segment_from_seq.to_string()));+ args.push(Bytes::from(descriptor.segment_to_seq.unwrap().to_string()));+ args.push(Bytes::from(descriptor.record_count.to_string()));+ args.push(Bytes::from(serde_json::to_vec(descriptor)?));+ args.push(Bytes::from(owner.unwrap_or_default().to_string()));+ args.push(Bytes::from(if keep_owner { "1" } else { "0" }));+ args.extend(raw_records.iter().cloned());+ let values = self.execute_array(SEAL_SCRIPT_ID, &keys, args).await?;+ match Self::status(&values, "seal")? {+ "ok" | "already" => Ok(()),+ "not_found" => Err(Error::not_found(keys.state)),+ "busy" => Err(Error::would_block("cache head is owned by an active flush")),+ "fenced" => Err(Error::would_block(+ "cache seal owner was fenced by another writer",+ )),+ "changed" => Err(Error::would_block(+ "cache head changed while committing a sealed prefix",+ )),+ "scope_mismatch" => Err(Error::invalid_operation(+ "Cache-backed multi-write state scope does not match",+ )),+ "invalid" => Err(Self::script_invalid(&values, "seal")),+ other => Err(Self::unknown_status("seal", other)),+ }+ }++ /// Seals one snapshot while enforcing an optional flush-owner fence.+ async fn seal_head_with_owner(+ &self,+ scope: &ScopeKey,+ owner: Option<&str>,+ keep_owner: bool,+ ) -> Result<Option<SegmentDescriptor>> {+ let (_, records, raw_records) = self.snapshot(scope).await?;+ if records.is_empty() {+ return Ok(None);+ }+ let directory = self+ .store+ .paths()+ .segments_dir(&scope.account_id, scope.partition_id)?+ .0;+ let path = format!(+ "{directory}/{}",+ Self::sealed_name(records.first().unwrap().seq, records.last().unwrap().seq)+ );+ let lease = self+ .store+ .pathlock_manager()+ .acquire_exact(+ &path,+ self.store+ .pathlock_manager()+ .default_lock_timeout()+ .max(std::time::Duration::from_secs(30)),+ None,+ )+ .await?;+ let operation = async {+ let descriptor = self.persist_sealed(scope, &records).await?;+ self.commit_seal(scope, &raw_records, &descriptor, owner, keep_owner)+ .await?;+ Ok(Some(descriptor))+ }+ .await;+ let release = self+ .store+ .pathlock_manager()+ .release(&lease)+ .await+ .map_err(Error::from);+ match operation {+ Ok(result) => release.map(|()| result),+ Err(error) => Err(error),+ }+ }++ /// Returns whether one marker is the end of a referenced sealed segment.+ async fn marker_is_sealed(&self, scope: &ScopeKey, seq: u64) -> Result<bool> {+ Ok(self+ .read_manifest(scope)+ .await?+ .segments+ .iter()+ .any(|descriptor| descriptor.segment_to_seq == Some(seq)))+ }++ /// Returns the deterministic mutable head filename.+ fn head_name(from_seq: u64) -> String {+ format!("{HEAD_SEGMENT_FILE_PREFIX}{from_seq:020}{SEGMENT_FILE_EXTENSION}")+ }++ /// Returns the deterministic immutable segment filename.+ fn sealed_name(from_seq: u64, to_seq: u64) -> String {+ format!("{SEALED_SEGMENT_FILE_PREFIX}{from_seq:020}-{to_seq:020}{SEGMENT_FILE_EXTENSION}")+ }++ /// Rebuild missing mutable state from the latest checkpoint and retained segments.+ async fn recovery_manifest(+ &self,+ scope: &ScopeKey,+ mut manifest: SegmentManifest,+ ) -> Result<SegmentManifest> {+ manifest.validate()?;+ for state in manifest.backend_states.values_mut() {+ state.synced_seq = 0;+ }+ manifest.segments.clear();+ manifest.next_seq = match self+ .store+ .read_json::<CheckpointsManifest>(+ &self+ .store+ .paths()+ .checkpoints_manifest(&scope.account_id, scope.partition_id)?+ .0,+ )+ .await+ {+ Ok(pointer) => {+ pointer.validate()?;+ pointer.latest_checkpoint.map_or(1, |latest| {+ latest.checkpoint_to_seq.checked_add(1).unwrap_or(u64::MAX)+ })+ }+ Err(Error::NotFound(_)) => 1,+ Err(error) => return Err(error),+ };+ let directory = self+ .store+ .paths()+ .segments_dir(&scope.account_id, scope.partition_id)?+ .0;+ let mut candidates = match self.store.list_directory(&directory).await {+ Ok(entries) => entries+ .into_iter()+ .filter_map(|entry| {+ (!entry.is_dir)+ .then(|| Self::sealed_range(&entry.name).map(|range| (range, entry.name)))+ .flatten()+ })+ .collect::<Vec<_>>(),+ Err(Error::NotFound(_)) => Vec::new(),+ Err(error) => return Err(error),+ };+ candidates.sort_by_key(|(range, _)| *range);+ for ((from, to), path) in candidates {+ if to < manifest.next_seq {+ continue;+ }+ if from != manifest.next_seq {+ break;+ }+ let bytes = self+ .store+ .read_bytes(&format!("{directory}/{path}"))+ .await?;+ let decoded = decode_segment(&bytes)?;+ if decoded.truncated_tail+ || decoded.records.first().map(|record| record.seq) != Some(from)+ || decoded.records.last().map(|record| record.seq) != Some(to)+ {+ return Err(Error::Serialization(+ "retained cache segment does not match its filename".to_string(),+ ));+ }+ manifest.segments.push(SegmentDescriptor {+ path,+ segment_from_seq: from,+ segment_to_seq: Some(to),+ record_count: decoded.records.len() as u32,+ byte_len: bytes.len() as u64,+ checksum: Some(sha256_hex(&bytes)),+ });+ manifest.next_seq = to+ .checked_add(1)+ .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?;+ }+ manifest.validate()?;+ Ok(manifest)+ }++ /// Parse one deterministic sealed segment filename.+ fn sealed_range(name: &str) -> Option<(u64, u64)> {+ let range = name+ .strip_prefix(SEALED_SEGMENT_FILE_PREFIX)?+ .strip_suffix(SEGMENT_FILE_EXTENSION)?;+ let (from, to) = range.split_once('-')?;+ Some((from.parse().ok()?, to.parse().ok()?))+ }++ /// Encodes account partition identity as script arguments.+ fn scope_args(scope: &ScopeKey) -> Vec<Bytes> {+ vec![+ Bytes::from(scope.account_id.clone()),+ Bytes::from(scope.partition_id.to_string()),+ Bytes::from(scope.epoch.to_string()),+ ]+ }++ /// Returns the leading UTF-8 status from one script result.+ fn status<'a>(values: &'a [ScriptValue], operation: &str) -> Result<&'a str> {+ Self::string_value(values, 0, operation)+ }++ /// Reads one UTF-8 byte result at the requested index.+ fn string_value<'a>(values: &'a [ScriptValue], index: usize, field: &str) -> Result<&'a str> {+ match values.get(index) {+ Some(ScriptValue::Bytes(value)) => std::str::from_utf8(value).map_err(|error| {+ Error::Serialization(format!(+ "Cache-backed multi-write {field} is not UTF-8: {error}"+ ))+ }),+ other => Err(Error::Serialization(format!(+ "Cache-backed multi-write {field} has invalid type: {other:?}"+ ))),+ }+ }++ /// Reads one non-negative integer script result as u64.+ fn u64_value(values: &[ScriptValue], index: usize, field: &str) -> Result<u64> {+ match values.get(index) {+ Some(ScriptValue::Integer(value)) => u64::try_from(*value).map_err(|_| {+ Error::Serialization(format!("Cache-backed multi-write {field} is negative"))+ }),+ Some(ScriptValue::Bytes(value)) => std::str::from_utf8(value)+ .ok()+ .and_then(|value| value.parse().ok())+ .ok_or_else(|| {+ Error::Serialization(format!(+ "Cache-backed multi-write {field} is not an integer"+ ))+ }),+ other => Err(Error::Serialization(format!(+ "Cache-backed multi-write {field} has invalid type: {other:?}"+ ))),+ }+ }++ /// Reads one script integer as u32.+ fn u32_value(values: &[ScriptValue], index: usize, field: &str) -> Result<u32> {+ u32::try_from(Self::u64_value(values, index, field)?).map_err(|_| {+ Error::Serialization(format!("Cache-backed multi-write {field} exceeds u32"))+ })+ }++ /// Reads one script integer as usize.+ fn usize_value(values: &[ScriptValue], index: usize, field: &str) -> Result<usize> {+ usize::try_from(Self::u64_value(values, index, field)?).map_err(|_| {+ Error::Serialization(format!("Cache-backed multi-write {field} exceeds usize"))+ })+ }++ /// Deserializes one JSON byte result.+ fn json_value<T>(values: &[ScriptValue], index: usize, field: &str) -> Result<T>+ where+ T: for<'de> Deserialize<'de>,+ {+ match values.get(index) {+ Some(ScriptValue::Bytes(value)) => serde_json::from_slice(value)+ .map_err(|error| Error::Serialization(format!("{field}: {error}"))),+ other => Err(Error::Serialization(format!(+ "Cache-backed multi-write {field} has invalid type: {other:?}"+ ))),+ }+ }++ /// Builds a validated operation error from a script response.+ fn script_invalid(values: &[ScriptValue], operation: &str) -> Error {+ match Self::string_value(values, 1, operation) {+ Ok(message) => Error::invalid_operation(message),+ Err(error) => error,+ }+ }++ /// Builds an error for an unknown script status.+ fn unknown_status(operation: &str, status: &str) -> Error {+ Error::Serialization(format!(+ "Cache-backed multi-write {operation} returned unknown status: {status}"+ ))+ }++ /// Maps cache runtime failures into existing RAGFS error categories.+ fn cache_error(operation: &str, error: CacheError) -> Error {+ match error {+ CacheError::Timeout(message) => {+ Error::Timeout(format!("Cache-backed multi-write {operation}: {message}"))+ }+ CacheError::Unavailable(message) => {+ Error::Network(format!("Cache-backed multi-write {operation}: {message}"))+ }+ CacheError::Closed => Error::Network(format!(+ "Cache-backed multi-write {operation}: runtime is closed"+ )),+ CacheError::InvalidData(message) => {+ Error::Serialization(format!("Cache-backed multi-write {operation}: {message}"))+ }+ other => Error::internal(format!("Cache-backed multi-write {operation}: {other}")),+ }+ }+}++#[async_trait]+impl MultiWriteProvider for CacheProvider {+ /// Atomically installs reconstructed state only when cache state is absent.+ async fn bootstrap_manifest(&self, scope: &ScopeKey, manifest: SegmentManifest) -> Result<()> {+ self.validate_scope(scope).await?;+ match self.read_manifest(scope).await {+ Ok(_) => return Ok(()),+ Err(Error::NotFound(_)) => {}+ Err(error) => return Err(error),+ }+ let manifest = self.recovery_manifest(scope, manifest).await?;+ let keys = self.keys_for_scope(scope)?;+ let mut args = vec![Bytes::from_static(b"bootstrap")];+ args.extend(Self::scope_args(scope));+ args.push(Bytes::from(manifest.version.to_string()));+ args.push(Bytes::from(manifest.next_seq.to_string()));+ args.push(Bytes::from(serde_json::to_vec(&manifest.segments)?));+ args.push(Bytes::from(serde_json::to_vec(&manifest.backend_states)?));+ let values = self.execute_array(SEAL_SCRIPT_ID, &keys, args).await?;+ match Self::status(&values, "bootstrap")? {+ "ok" | "exists" => self.read_manifest(scope).await.map(|_| ()),+ "invalid" => Err(Self::script_invalid(&values, "bootstrap")),+ other => Err(Self::unknown_status("bootstrap", other)),+ }+ }++ /// Flushes normal events through bounded atomic cache appends.+ async fn flush(+ &self,+ _route_hint: Option<&PartitionContext>,+ events: Vec<PendingEvent>,+ ) -> Result<FlushResult> {+ Self::validate_flush_events(&events)?;+ let records = events+ .iter()+ .map(Self::record)+ .collect::<Result<Vec<_>>>()?;+ let mut offset = 0;+ let mut route = None;+ let mut continuing = false;+ let mut first_seq = None;+ let mut last_seq = None;+ let mut sealed_segments = Vec::new();+ let owner = Uuid::new_v4().to_string();+ while offset < records.len() {+ let (current_route, outcome) = self+ .flush_routed_once(+ &events,+ route.as_ref().map(|route: &PartitionContext| &route.scope),+ &records[offset..],+ &owner,+ continuing,+ )+ .await?;+ match outcome {+ FlushScriptOutcome::SealRequired => {+ route.get_or_insert(current_route);+ continuing = true;+ if let Some(descriptor) = self+ .seal_head_with_owner(&route.as_ref().unwrap().scope, Some(&owner), true)+ .await?+ {+ if !sealed_segments+ .iter()+ .any(|sealed: &SegmentDescriptor| sealed.path == descriptor.path)+ {+ sealed_segments.push(descriptor);+ }+ }+ }+ FlushScriptOutcome::Busy => {+ tokio::time::sleep(std::time::Duration::from_millis(1)).await;+ }+ FlushScriptOutcome::Appended(result) => {+ if result.appended == 0 {+ return Err(Error::Serialization(+ "cache-backed flush appended no records".to_string(),+ ));+ }+ route.get_or_insert(current_route);+ first_seq.get_or_insert(result.first_seq);+ last_seq = Some(result.last_seq);+ offset += result.appended;+ if result.needs_seal {+ let keep_owner = offset < records.len();+ if let Some(descriptor) = self+ .seal_head_with_owner(+ &route.as_ref().unwrap().scope,+ Some(&owner),+ keep_owner,+ )+ .await?+ {+ if !sealed_segments+ .iter()+ .any(|sealed| sealed.path == descriptor.path)+ {+ sealed_segments.push(descriptor);+ }+ }+ continuing = keep_owner;+ } else if offset < records.len() {+ return Err(Error::Serialization(+ "cache-backed flush released its owner before completing the batch"+ .to_string(),+ ));+ }+ }+ }+ }+ Ok(FlushResult {+ scope: route.unwrap().scope,+ first_seq: first_seq.unwrap(),+ last_seq: last_seq.unwrap(),+ sealed_segments,+ })+ }++ /// Reads and validates committed cache head records in one half-open range.+ async fn read_committed_head(+ &self,+ scope: &ScopeKey,+ from_seq_inclusive: u64,+ to_seq_exclusive: u64,+ ) -> Result<Vec<SegmentRecord>> {+ let (state, records, _) = self.snapshot(scope).await?;+ if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > state.next_seq {+ return Err(Error::invalid_operation(+ "committed head range is outside the manifest",+ ));+ }+ let head_from = records+ .first()+ .map(|record| record.seq)+ .unwrap_or(state.next_seq);+ if from_seq_inclusive < head_from {+ return Err(Error::invalid_operation(+ "committed head range starts before the current cache head",+ ));+ }+ let expected_len = usize::try_from(to_seq_exclusive - from_seq_inclusive)+ .map_err(|_| Error::invalid_operation("committed head range is too large"))?;+ let selected = records+ .into_iter()+ .filter(|record| record.seq >= from_seq_inclusive && record.seq < to_seq_exclusive)+ .collect::<Vec<_>>();+ if selected.len() != expected_len+ || selected.iter().enumerate().any(|(index, record)| {+ record.seq+ != from_seq_inclusive+ .checked_add(index as u64)+ .unwrap_or(u64::MAX)+ })+ {+ return Err(Error::Serialization(+ "cache committed head range is incomplete".to_string(),+ ));+ }+ Ok(selected)+ }++ /// Reads one complete committed range from sealed storage and the cache head.+ async fn read_committed_range(+ &self,+ scope: &ScopeKey,+ from_seq_inclusive: u64,+ to_seq_exclusive: u64,+ ) -> Result<Vec<SegmentRecord>> {+ let (state, head, _) = self.snapshot(scope).await?;+ if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > state.next_seq {+ return Err(Error::invalid_operation(+ "committed range is outside the manifest",+ ));+ }+ let mut records = Vec::new();+ for descriptor in &state.segments {+ if descriptor.segment_from_seq >= to_seq_exclusive+ || descriptor+ .segment_to_seq+ .is_some_and(|to| to < from_seq_inclusive)+ {+ continue;+ }+ records.extend(read_sealed_segment(&self.store, scope, descriptor).await?);+ }+ records.extend(head);+ select_record_range(records, from_seq_inclusive, to_seq_exclusive)+ }++ /// Reads one immutable sealed segment named by a validated descriptor.+ async fn read_sealed_segment(+ &self,+ scope: &ScopeKey,+ descriptor: &SegmentDescriptor,+ ) -> Result<Vec<SegmentRecord>> {+ read_sealed_segment(&self.store, scope, descriptor).await+ }++ /// Returns validated sealed state plus a derived mutable-head descriptor.+ async fn read_manifest(&self, scope: &ScopeKey) -> Result<SegmentManifest> {+ let (state, records, _) = self.snapshot(scope).await?;+ Self::manifest(state, &records)+ }++ /// Appends one caller-scoped marker and seals it before returning.+ async fn append_marker(+ &self,+ scope: &ScopeKey,+ event: &PendingEvent,+ ) -> Result<MarkerPosition> {+ self.validate_scope(scope).await?;+ self.validate_marker(scope, event)?;+ let record = Self::record(event)?;+ let owner = Uuid::new_v4().to_string();+ let mut continuing = false;+ loop {+ match self+ .flush_marker_once(scope, &record, &owner, continuing)+ .await?+ {+ FlushScriptOutcome::SealRequired => {+ continuing = true;+ self.seal_head_with_owner(scope, Some(&owner), true).await?;+ }+ FlushScriptOutcome::Busy => {+ tokio::time::sleep(std::time::Duration::from_millis(1)).await;+ }+ FlushScriptOutcome::Appended(result) => {+ if result.appended != 1 || !result.needs_seal {+ return Err(Error::Serialization(+ "cache marker append returned an invalid result".to_string(),+ ));+ }+ self.seal_head_with_owner(scope, Some(&owner), false)+ .await?;+ if !self.marker_is_sealed(scope, result.first_seq).await? {+ return Err(Error::Serialization(+ "cache marker is not referenced by a sealed segment".to_string(),+ ));+ }+ return Ok(MarkerPosition {+ partition_id: scope.partition_id,+ epoch: scope.epoch,+ seq: result.first_seq,+ });+ }+ }+ }+ }++ /// Advances one backend's monotonic progress atomically through CacheRuntime.+ async fn advance_backend_state(+ &self,+ scope: &ScopeKey,+ backend_id: &str,+ synced_seq: u64,+ ) -> Result<()> {+ self.validate_scope(scope).await?;+ if backend_id.trim().is_empty() {+ return Err(Error::invalid_operation("backend id must not be empty"));+ }+ let keys = self.keys_for_scope(scope)?;+ let mut args = Self::scope_args(scope);+ args.push(Bytes::from(backend_id.to_string()));+ args.push(Bytes::from(synced_seq.to_string()));+ let values = self.execute_array(ADVANCE_SCRIPT_ID, &keys, args).await?;+ match Self::status(&values, "advance backend")? {+ "ok" => Ok(()),+ "not_found" => Err(Error::not_found(keys.state)),+ "scope_mismatch" => Err(Error::invalid_operation(+ "Cache-backed multi-write state scope does not match",+ )),+ "invalid" => Err(Self::script_invalid(&values, "advance backend")),+ other => Err(Self::unknown_status("advance backend", other)),+ }+ }++ /// Atomically validates and removes exactly the requested sealed descriptors.+ async fn prune_sealed(+ &self,+ scope: &ScopeKey,+ removable_paths: &[String],+ ) -> Result<Vec<SegmentDescriptor>> {+ let path = self.store.paths().account_manifest(&scope.account_id)?.0;+ let lease = self+ .store+ .pathlock_manager()+ .acquire_exact(+ &path,+ self.store+ .pathlock_manager()+ .default_lock_timeout()+ .max(std::time::Duration::from_secs(30)),+ None,+ )+ .await?;+ let operation = async {+ if !self.store.gc_scope_is_safe(scope).await? {+ return Ok(Vec::new());+ }+ let keys = self.keys_for_scope(scope)?;+ let mut args = vec![Bytes::from_static(b"prune")];+ args.extend(Self::scope_args(scope));+ args.push(Bytes::from(serde_json::to_vec(removable_paths)?));+ let values = self.execute_array(SEAL_SCRIPT_ID, &keys, args).await?;+ match Self::status(&values, "prune")? {+ "ok" => Self::json_value(&values, 1, "pruned descriptors"),+ "not_found" => Err(Error::not_found(keys.state)),+ "scope_mismatch" => Err(Error::invalid_operation(+ "Cache-backed multi-write state scope does not match",+ )),+ "invalid" => Err(Self::script_invalid(&values, "prune")),+ other => Err(Self::unknown_status("prune", other)),+ }+ }+ .await;+ let release = self+ .store+ .pathlock_manager()+ .release(&lease)+ .await+ .map_err(Error::from);+ match operation {+ Ok(result) => release.map(|()| result),+ Err(error) => Err(error),+ }+ }+}diff --git a/crates/ragfs/src/multibackend/provider/filesystem.rs b/crates/ragfs/src/multibackend/provider/filesystem.rsnew file mode 100644index 000000000..89840e632--- /dev/null+++ b/crates/ragfs/src/multibackend/provider/filesystem.rs@@ -0,0 +1,767 @@+//! Filesystem-backed V2 multi-write persistence.++use std::collections::{BTreeSet, HashSet};+use std::sync::Arc;++use async_trait::async_trait;++use crate::core::errors::{Error, Result};+use crate::core::internal_names::is_multiwrite_internal_path;+use crate::lock::{OwnedPathLockLease, PathLockKind, PathLockRequest};+use crate::multibackend::codec::{decode_segment, encode_segment, sha256_hex};+use crate::multibackend::constants::{+ HEAD_SEGMENT_FILE_PREFIX, MAX_SEGMENT_RECORDS, SEALED_SEGMENT_FILE_PREFIX,+ SEGMENT_FILE_EXTENSION,+};+use crate::multibackend::meta::MetadataStore;+use crate::multibackend::model::{+ FlushResult, MarkerPosition, PartitionContext, PartitionsManifest, PendingEvent,+ PendingEventKind, ScopeKey, SegmentDescriptor, SegmentEventType, SegmentManifest,+ SegmentRecord,+};+use crate::multibackend::router::AccountRouter;++use super::{read_sealed_segment, select_record_range, MultiWriteProvider};++/// Persists V2 manifests and segment blobs on the primary filesystem.+pub struct FilesystemProvider {+ store: Arc<MetadataStore>,+ router: AccountRouter,+}++impl FilesystemProvider {+ /// Creates a filesystem-backed metadata provider.+ pub fn new(store: Arc<MetadataStore>) -> Self {+ Self {+ router: AccountRouter::new(store.clone()),+ store,+ }+ }++ /// Validates normal flush input before any route or storage access.+ fn validate_flush_events(events: &[PendingEvent]) -> Result<()> {+ let first = events+ .first()+ .ok_or_else(|| Error::invalid_operation("flush requires at least one event"))?;+ for event in events {+ if event.account_id != first.account_id {+ return Err(Error::invalid_operation(+ "flush events must belong to one account",+ ));+ }+ if is_multiwrite_internal_path(&event.path) {+ return Err(Error::invalid_path(format!(+ "cannot flush internal path: {}",+ event.path+ )));+ }+ match event.kind {+ PendingEventKind::Data(SegmentEventType::Write | SegmentEventType::Remove) => {}+ _ => {+ return Err(Error::invalid_operation(+ "normal flush accepts only write and remove events",+ ))+ }+ }+ if event.destination_path.is_some() {+ return Err(Error::invalid_operation(+ "normal data event cannot have a destination",+ ));+ }+ }+ Ok(())+ }++ /// Routes every event and requires one current partition scope.+ async fn route_events(&self, events: &[PendingEvent]) -> Result<PartitionContext> {+ let mut selected = None;+ for event in events {+ let current = self.router.route(&event.account_id, &event.path).await?;+ if selected+ .as_ref()+ .is_some_and(|existing: &PartitionContext| existing.scope != current.scope)+ {+ return Err(Error::invalid_operation(+ "flush events route to different partition scopes",+ ));+ }+ selected.get_or_insert(current);+ }+ selected.ok_or_else(|| Error::invalid_operation("flush requires at least one event"))+ }++ /// Reads and validates the partitions manifest owning one scope.+ async fn validate_scope(&self, scope: &ScopeKey) -> Result<()> {+ let path = self.store.paths().account_manifest(&scope.account_id)?.0;+ let manifest: PartitionsManifest = self.store.read_json(&path).await?;+ manifest.validate()?;+ scope.validate_against_manifest(&manifest)+ }++ /// Reads and validates one segment manifest without acquiring locks.+ async fn load_manifest(&self, scope: &ScopeKey) -> Result<SegmentManifest> {+ self.validate_scope(scope).await?;+ let path = self+ .store+ .paths()+ .segment_manifest(&scope.account_id, scope.partition_id)?+ .0;+ let manifest: SegmentManifest = self.store.read_json(&path).await?;+ manifest.validate()?;+ Ok(manifest)+ }++ /// Returns the current mutable head descriptor.+ fn head(manifest: &SegmentManifest) -> Option<&SegmentDescriptor> {+ manifest+ .segments+ .last()+ .filter(|segment| segment.segment_to_seq.is_none())+ }++ /// Returns the deterministic mutable head filename.+ fn head_name(from_seq: u64) -> String {+ format!("{HEAD_SEGMENT_FILE_PREFIX}{from_seq:020}{SEGMENT_FILE_EXTENSION}")+ }++ /// Returns the deterministic immutable sealed filename.+ fn sealed_name(from_seq: u64, to_seq: u64) -> String {+ format!("{SEALED_SEGMENT_FILE_PREFIX}{from_seq:020}-{to_seq:020}{SEGMENT_FILE_EXTENSION}")+ }++ /// Returns one logical segment blob path.+ fn segment_path(&self, scope: &ScopeKey, name: &str) -> Result<String> {+ let directory = self+ .store+ .paths()+ .segments_dir(&scope.account_id, scope.partition_id)?+ .0;+ Ok(format!("{directory}/{name}"))+ }++ /// Builds the complete Exact-lock set for one append candidate.+ fn append_lock_requests(+ &self,+ scope: &ScopeKey,+ manifest: &SegmentManifest,+ event_count: usize,+ force_final_seal: bool,+ ) -> Result<Vec<PathLockRequest>> {+ let mut paths = BTreeSet::from([self+ .store+ .paths()+ .segment_manifest(&scope.account_id, scope.partition_id)?+ .0]);+ paths.insert(self.store.paths().account_manifest(&scope.account_id)?.0);+ let mut remaining = event_count;+ let mut next_seq = manifest.next_seq;+ if next_seq.checked_add(event_count as u64).is_none() {+ return Err(Error::invalid_operation("multi-write sequence overflow"));+ }+ let mut count = 0;+ let mut segment_from_seq = next_seq;+ if let Some(head) = Self::head(manifest) {+ paths.insert(self.segment_path(scope, &head.path)?);+ count = head.record_count as usize;+ segment_from_seq = head.segment_from_seq;+ }++ if count == MAX_SEGMENT_RECORDS {+ paths.insert(+ self.segment_path(scope, &Self::sealed_name(segment_from_seq, next_seq - 1))?,+ );+ count = 0;+ segment_from_seq = next_seq;+ }+ while remaining > 0 {+ let appended = remaining.min(MAX_SEGMENT_RECORDS - count);+ remaining -= appended;+ next_seq = next_seq+ .checked_add(appended as u64)+ .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?;+ count += appended;+ if count == MAX_SEGMENT_RECORDS || (force_final_seal && remaining == 0) {+ paths.insert(+ self.segment_path(scope, &Self::sealed_name(segment_from_seq, next_seq - 1))?,+ );+ count = 0;+ segment_from_seq = next_seq;+ }+ }+ if count > 0 {+ let name = if force_final_seal {+ Self::sealed_name(segment_from_seq, next_seq - 1)+ } else {+ Self::head_name(segment_from_seq)+ };+ paths.insert(self.segment_path(scope, &name)?);+ }+ Ok(paths+ .into_iter()+ .map(|path| PathLockRequest {+ path,+ kind: PathLockKind::Exact,+ })+ .collect())+ }++ /// Builds the Exact-lock set for a manifest-only state mutation.+ fn manifest_lock_requests(&self, scope: &ScopeKey) -> Result<Vec<PathLockRequest>> {+ let mut paths = vec![PathLockRequest {+ path: self+ .store+ .paths()+ .segment_manifest(&scope.account_id, scope.partition_id)?+ .0,+ kind: PathLockKind::Exact,+ }];+ paths.push(PathLockRequest {+ path: self.store.paths().account_manifest(&scope.account_id)?.0,+ kind: PathLockKind::Exact,+ });+ Ok(paths)+ }++ /// Acquires one all-or-nothing Exact-lock batch.+ async fn acquire(&self, requests: &[PathLockRequest]) -> Result<OwnedPathLockLease> {+ let manager = self.store.pathlock_manager();+ Ok(manager+ .acquire_batch(requests, std::time::Duration::from_secs(5), None)+ .await?)+ }++ /// Releases a lock lease while preserving any operation failure.+ async fn release<T>(&self, lease: &OwnedPathLockLease, operation: Result<T>) -> Result<T> {+ let release = self+ .store+ .pathlock_manager()+ .release(lease)+ .await+ .map_err(Error::from);+ merge_operation_and_release(operation, release)+ }++ /// Reads and validates only the committed prefix of the current head.+ async fn committed_head(+ &self,+ scope: &ScopeKey,+ manifest: &SegmentManifest,+ ) -> Result<Vec<SegmentRecord>> {+ let Some(head) = Self::head(manifest) else {+ return Ok(Vec::new());+ };+ if head.path != Self::head_name(head.segment_from_seq) {+ return Err(Error::Serialization(+ "head segment path is not deterministic".to_string(),+ ));+ }+ let bytes = self+ .store+ .read_bytes(&self.segment_path(scope, &head.path)?)+ .await?;+ let committed_len = usize::try_from(head.byte_len)+ .map_err(|_| Error::Serialization("head byte length is too large".to_string()))?;+ if bytes.len() < committed_len {+ return Err(Error::Serialization(+ "head segment is shorter than its committed byte length".to_string(),+ ));+ }+ let decoded = decode_segment(&bytes[..committed_len])?;+ if decoded.truncated_tail+ || decoded.valid_bytes != head.byte_len+ || decoded.records.len() != head.record_count as usize+ {+ return Err(Error::Serialization(+ "head segment committed prefix does not match its descriptor".to_string(),+ ));+ }+ for (index, record) in decoded.records.iter().enumerate() {+ let expected = head+ .segment_from_seq+ .checked_add(index as u64)+ .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?;+ if record.seq != expected {+ return Err(Error::Serialization(+ "head segment contains a sequence gap".to_string(),+ ));+ }+ }+ Ok(decoded.records)+ }++ /// Converts one pending event into its persisted segment record.+ fn record(event: &PendingEvent, seq: u64) -> Result<SegmentRecord> {+ let event_type = match event.kind {+ PendingEventKind::Data(kind) => kind,+ PendingEventKind::DeleteAccount => {+ return Err(Error::invalid_operation(+ "account deletion is not a segment record",+ ))+ }+ };+ let marker = matches!(+ event_type,+ SegmentEventType::RemoveTree+ | SegmentEventType::MoveTree+ | SegmentEventType::DeleteScope+ );+ let record = SegmentRecord {+ seq,+ event_type,+ path: event.path.clone(),+ op_id: marker.then_some(event.operation_id),+ destination_path: event.destination_path.clone(),+ };+ record.validate()?;+ Ok(record)+ }++ /// Persists one immutable sealed segment and returns its descriptor.+ async fn persist_sealed(+ &self,+ scope: &ScopeKey,+ manifest: &SegmentManifest,+ records: &[SegmentRecord],+ ) -> Result<SegmentDescriptor> {+ let first = records+ .first()+ .ok_or_else(|| Error::invalid_operation("cannot seal an empty segment"))?+ .seq;+ let last = records.last().unwrap().seq;+ let bytes = encode_segment(records)?;+ let name = Self::sealed_name(first, last);+ let path = self.segment_path(scope, &name)?;+ if !manifest.segments.iter().any(|segment| segment.path == name) {+ self.store.remove_file(&path).await?;+ }+ self.store.write_immutable(&path, &bytes).await?;+ Ok(SegmentDescriptor {+ path: name,+ segment_from_seq: first,+ segment_to_seq: Some(last),+ record_count: records.len() as u32,+ byte_len: bytes.len() as u64,+ checksum: Some(sha256_hex(&bytes)),+ })+ }++ /// Publishes one mutable head and returns its descriptor.+ async fn persist_head(+ &self,+ scope: &ScopeKey,+ records: &[SegmentRecord],+ ) -> Result<SegmentDescriptor> {+ let first = records+ .first()+ .ok_or_else(|| Error::invalid_operation("cannot publish an empty head"))?+ .seq;+ let bytes = encode_segment(records)?;+ let name = Self::head_name(first);+ self.store+ .write_bytes(&self.segment_path(scope, &name)?, &bytes)+ .await?;+ Ok(SegmentDescriptor {+ path: name,+ segment_from_seq: first,+ segment_to_seq: None,+ record_count: records.len() as u32,+ byte_len: bytes.len() as u64,+ checksum: None,+ })+ }++ /// Appends records to a locked manifest and publishes blobs before the manifest.+ async fn commit_events(+ &self,+ scope: &ScopeKey,+ mut manifest: SegmentManifest,+ events: &[PendingEvent],+ force_final_seal: bool,+ ) -> Result<(SegmentManifest, Vec<SegmentDescriptor>)> {+ let mut records = self.committed_head(scope, &manifest).await?;+ let mut sealed = Vec::new();+ if records.len() == MAX_SEGMENT_RECORDS {+ manifest.segments.pop();+ let descriptor = self.persist_sealed(scope, &manifest, &records).await?;+ manifest.segments.push(descriptor.clone());+ sealed.push(descriptor);+ records.clear();+ }++ for (index, event) in events.iter().enumerate() {+ let seq = manifest.next_seq;+ manifest.next_seq = manifest+ .next_seq+ .checked_add(1)+ .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?;+ records.push(Self::record(event, seq)?);+ let seal_now = records.len() == MAX_SEGMENT_RECORDS+ || (force_final_seal && index + 1 == events.len());+ if seal_now {+ if Self::head(&manifest).is_some() {+ manifest.segments.pop();+ }+ let descriptor = self.persist_sealed(scope, &manifest, &records).await?;+ manifest.segments.push(descriptor.clone());+ sealed.push(descriptor);+ records.clear();+ }+ }++ if !records.is_empty() {+ let descriptor = self.persist_head(scope, &records).await?;+ if Self::head(&manifest).is_some() {+ manifest.segments.pop();+ }+ manifest.segments.push(descriptor);+ }+ manifest.validate()?;+ let path = self+ .store+ .paths()+ .segment_manifest(&scope.account_id, scope.partition_id)?+ .0;+ self.store+ .publish_json(&path, &manifest, SegmentManifest::validate)+ .await?;+ Ok((manifest, sealed))+ }++ /// Returns whether the locked append snapshot still matches its candidate.+ fn append_snapshot_matches(+ candidate_scope: &ScopeKey,+ current_scope: &ScopeKey,+ candidate_manifest: &SegmentManifest,+ current_manifest: &SegmentManifest,+ candidate_requests: &[PathLockRequest],+ current_requests: &[PathLockRequest],+ ) -> bool {+ candidate_scope == current_scope+ && candidate_manifest == current_manifest+ && candidate_requests == current_requests+ }++ /// Validates one marker against its caller-selected scope.+ fn validate_marker(&self, scope: &ScopeKey, event: &PendingEvent) -> Result<()> {+ if event.account_id != scope.account_id {+ return Err(Error::invalid_operation(+ "marker account does not match supplied scope",+ ));+ }+ match event.kind {+ PendingEventKind::Data(+ SegmentEventType::RemoveTree+ | SegmentEventType::MoveTree+ | SegmentEventType::DeleteScope,+ ) => {}+ _ => {+ return Err(Error::invalid_operation(+ "append_marker accepts only directory markers",+ ))+ }+ }+ if is_multiwrite_internal_path(&event.path) {+ return Err(Error::invalid_path(format!(+ "cannot append marker for internal path: {}",+ event.path+ )));+ }+ if !matches!(+ event.kind,+ PendingEventKind::Data(SegmentEventType::DeleteScope)+ ) {+ self.store+ .paths()+ .backend_path(&scope.account_id, &event.path)?;+ }+ if let Some(destination) = &event.destination_path {+ if is_multiwrite_internal_path(destination) {+ return Err(Error::invalid_path(format!(+ "cannot append marker for internal destination: {destination}"+ )));+ }+ self.store+ .paths()+ .backend_path(&scope.account_id, destination)?;+ }+ Self::record(event, 1).map(|_| ())+ }+}++#[async_trait]+impl MultiWriteProvider for FilesystemProvider {+ /// Confirms that filesystem state exists for the account partition.+ async fn bootstrap_manifest(&self, scope: &ScopeKey, manifest: SegmentManifest) -> Result<()> {+ manifest.validate()?;+ self.load_manifest(scope).await.map(|_| ())+ }++ /// Flushes normal events after validating their current partition route.+ async fn flush(+ &self,+ _route_hint: Option<&PartitionContext>,+ events: Vec<PendingEvent>,+ ) -> Result<FlushResult> {+ Self::validate_flush_events(&events)?;+ loop {+ let candidate = self.route_events(&events).await?;+ let candidate_manifest = self.load_manifest(&candidate.scope).await?;+ let requests = self.append_lock_requests(+ &candidate.scope,+ &candidate_manifest,+ events.len(),+ false,+ )?;+ let lease = self.acquire(&requests).await?;+ let operation = async {+ let current = self.route_events(&events).await?;+ let manifest = self.load_manifest(¤t.scope).await?;+ let current_requests =+ self.append_lock_requests(¤t.scope, &manifest, events.len(), false)?;+ if !Self::append_snapshot_matches(+ &candidate.scope,+ ¤t.scope,+ &candidate_manifest,+ &manifest,+ &requests,+ ¤t_requests,+ ) {+ return Ok(None);+ }+ let first_seq = manifest.next_seq;+ let (_, sealed_segments) = self+ .commit_events(¤t.scope, manifest, &events, false)+ .await?;+ let last_seq = first_seq+ .checked_add(events.len() as u64 - 1)+ .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?;+ Ok(Some(FlushResult {+ scope: current.scope,+ first_seq,+ last_seq,+ sealed_segments,+ }))+ }+ .await;+ if let Some(result) = self.release(&lease, operation).await? {+ return Ok(result);+ }+ }+ }++ /// Reads committed head records in the requested half-open sequence range.+ async fn read_committed_head(+ &self,+ scope: &ScopeKey,+ from_seq_inclusive: u64,+ to_seq_exclusive: u64,+ ) -> Result<Vec<SegmentRecord>> {+ let manifest = self.load_manifest(scope).await?;+ if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > manifest.next_seq {+ return Err(Error::invalid_operation(+ "committed head range is outside the manifest",+ ));+ }+ let records = self.committed_head(scope, &manifest).await?;+ Ok(records+ .into_iter()+ .filter(|record| record.seq >= from_seq_inclusive && record.seq < to_seq_exclusive)+ .collect())+ }++ /// Reads one complete committed range from sealed segments and the current head.+ async fn read_committed_range(+ &self,+ scope: &ScopeKey,+ from_seq_inclusive: u64,+ to_seq_exclusive: u64,+ ) -> Result<Vec<SegmentRecord>> {+ let manifest = self.load_manifest(scope).await?;+ if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > manifest.next_seq {+ return Err(Error::invalid_operation(+ "committed range is outside the manifest",+ ));+ }+ let mut records = Vec::new();+ for descriptor in &manifest.segments {+ if descriptor.segment_from_seq >= to_seq_exclusive+ || descriptor+ .segment_to_seq+ .is_some_and(|to| to < from_seq_inclusive)+ {+ continue;+ }+ if descriptor.segment_to_seq.is_some() {+ records.extend(read_sealed_segment(&self.store, scope, descriptor).await?);+ } else {+ records.extend(self.committed_head(scope, &manifest).await?);+ }+ }+ select_record_range(records, from_seq_inclusive, to_seq_exclusive)+ }++ /// Reads one immutable sealed segment named by a validated descriptor.+ async fn read_sealed_segment(+ &self,+ scope: &ScopeKey,+ descriptor: &SegmentDescriptor,+ ) -> Result<Vec<SegmentRecord>> {+ read_sealed_segment(&self.store, scope, descriptor).await+ }++ /// Returns the current segment state for one scope.+ async fn read_manifest(&self, scope: &ScopeKey) -> Result<SegmentManifest> {+ self.load_manifest(scope).await+ }++ /// Appends and seals one directory marker in the caller-selected scope.+ async fn append_marker(+ &self,+ scope: &ScopeKey,+ event: &PendingEvent,+ ) -> Result<MarkerPosition> {+ self.validate_marker(scope, event)?;+ loop {+ let candidate_manifest = self.load_manifest(scope).await?;+ let requests = self.append_lock_requests(scope, &candidate_manifest, 1, true)?;+ let lease = self.acquire(&requests).await?;+ let operation = async {+ let manifest = self.load_manifest(scope).await?;+ let current_requests = self.append_lock_requests(scope, &manifest, 1, true)?;+ if !Self::append_snapshot_matches(+ scope,+ scope,+ &candidate_manifest,+ &manifest,+ &requests,+ ¤t_requests,+ ) {+ return Ok(None);+ }+ let seq = manifest.next_seq;+ self.commit_events(scope, manifest, std::slice::from_ref(event), true)+ .await?;+ Ok(Some(MarkerPosition {+ partition_id: scope.partition_id,+ epoch: scope.epoch,+ seq,+ }))+ }+ .await;+ if let Some(position) = self.release(&lease, operation).await? {+ return Ok(position);+ }+ }+ }++ /// Advances one backup's progress while its catch-up lease is held.+ async fn advance_backend_state(+ &self,+ scope: &ScopeKey,+ backend_id: &str,+ synced_seq: u64,+ ) -> Result<()> {+ let requests = self.manifest_lock_requests(scope)?;+ let lease = self.acquire(&requests).await?;+ let operation = async {+ let mut manifest = self.load_manifest(scope).await?;+ let state = manifest.backend_states.get_mut(backend_id).ok_or_else(|| {+ Error::invalid_operation(format!("unknown backend id: {backend_id}"))+ })?;+ if synced_seq != 0 && synced_seq < state.synced_seq {+ return Err(Error::invalid_operation("backend progress cannot regress"));+ }+ if synced_seq >= manifest.next_seq {+ return Err(Error::invalid_operation(+ "backend progress must remain below next_seq",+ ));+ }+ state.synced_seq = synced_seq;+ let path = self+ .store+ .paths()+ .segment_manifest(&scope.account_id, scope.partition_id)?+ .0;+ self.store+ .publish_json(&path, &manifest, SegmentManifest::validate)+ .await+ }+ .await;+ self.release(&lease, operation).await+ }++ /// Atomically removes selected sealed descriptors and returns the removed set.+ async fn prune_sealed(+ &self,+ scope: &ScopeKey,+ removable_paths: &[String],+ ) -> Result<Vec<SegmentDescriptor>> {+ let requested = removable_paths.iter().collect::<HashSet<_>>();+ if requested.len() != removable_paths.len() {+ return Err(Error::invalid_operation(+ "prune paths must not contain duplicates",+ ));+ }+ let requests = self.manifest_lock_requests(scope)?;+ let lease = self.acquire(&requests).await?;+ let operation = async {+ if !self.store.gc_scope_is_safe(scope).await? {+ return Ok(Vec::new());+ }+ let mut manifest = self.load_manifest(scope).await?;+ for path in removable_paths {+ let descriptor = manifest+ .segments+ .iter()+ .find(|segment| segment.path == *path)+ .ok_or_else(|| {+ Error::invalid_operation(format!(+ "sealed segment is not referenced: {path}"+ ))+ })?;+ if descriptor.segment_to_seq.is_none() {+ return Err(Error::invalid_operation(format!(+ "cannot prune mutable head: {path}"+ )));+ }+ }+ let mut removed = Vec::new();+ manifest.segments.retain(|segment| {+ if requested.contains(&segment.path) {+ removed.push(segment.clone());+ false+ } else {+ true+ }+ });+ manifest.validate()?;+ let path = self+ .store+ .paths()+ .segment_manifest(&scope.account_id, scope.partition_id)?+ .0;+ self.store+ .publish_json(&path, &manifest, SegmentManifest::validate)+ .await?;+ Ok(removed)+ }+ .await;+ self.release(&lease, operation).await+ }+}++/// Preserves an operation error when lock release also fails.+fn merge_operation_and_release<T>(operation: Result<T>, release: Result<()>) -> Result<T> {+ match (operation, release) {+ (Ok(value), Ok(())) => Ok(value),+ (Ok(_), Err(error)) => Err(error),+ (Err(error), Ok(())) => Err(error),+ (Err(error), Err(release_error)) => Err(Error::internal(format!(+ "{error}; lock release failed: {release_error}"+ ))),+ }+}++#[cfg(test)]+#[path = "tests/test_filesystem.rs"]+mod tests;diff --git a/crates/ragfs/src/multibackend/provider/mod.rs b/crates/ragfs/src/multibackend/provider/mod.rsnew file mode 100644index 000000000..1f9911728--- /dev/null+++ b/crates/ragfs/src/multibackend/provider/mod.rs@@ -0,0 +1,179 @@+//! Persistence providers for V2 multi-write state.++use async_trait::async_trait;++use crate::core::errors::{Error, Result};+use crate::multibackend::codec::{decode_segment, sha256_hex};+use crate::multibackend::meta::MetadataStore;+use crate::multibackend::model::{+ FlushResult, MarkerPosition, PartitionContext, PartitionState, PartitionsManifest,+ PendingEvent, ScopeKey, SegmentDescriptor, SegmentManifest, SegmentRecord,+};++mod filesystem;+#[cfg(feature = "cache")]+mod cache;++pub use filesystem::FilesystemProvider;+#[cfg(feature = "cache")]+pub use cache::CacheProvider;++/// Persists and reads partitioned multi-write segment state.+#[async_trait]+pub trait MultiWriteProvider: Send + Sync {+ /// Creates missing provider state without replacing an existing manifest.+ async fn bootstrap_manifest(+ &self,+ scope: &ScopeKey,+ manifest: SegmentManifest,+ ) -> Result<()>;++ /// Flushes normal events after validating their current partition route.+ async fn flush(+ &self,+ route_hint: Option<&PartitionContext>,+ events: Vec<PendingEvent>,+ ) -> Result<FlushResult>;++ /// Reads committed head records in the requested half-open sequence range.+ async fn read_committed_head(+ &self,+ scope: &ScopeKey,+ from_seq_inclusive: u64,+ to_seq_exclusive: u64,+ ) -> Result<Vec<SegmentRecord>>;++ /// Reads committed records across sealed segments and the current head.+ async fn read_committed_range(+ &self,+ scope: &ScopeKey,+ from_seq_inclusive: u64,+ to_seq_exclusive: u64,+ ) -> Result<Vec<SegmentRecord>>;++ /// Reads one immutable sealed segment named by a validated descriptor.+ async fn read_sealed_segment(+ &self,+ scope: &ScopeKey,+ descriptor: &SegmentDescriptor,+ ) -> Result<Vec<SegmentRecord>>;++ /// Returns the current segment state for one scope.+ async fn read_manifest(&self, scope: &ScopeKey) -> Result<SegmentManifest>;++ /// Appends and seals one directory marker in the caller-selected scope.+ async fn append_marker(+ &self,+ scope: &ScopeKey,+ event: &PendingEvent,+ ) -> Result<MarkerPosition>;++ /// Advances one backup's monotonic progress while its catch-up lease is held.+ async fn advance_backend_state(+ &self,+ scope: &ScopeKey,+ backend_id: &str,+ synced_seq: u64,+ ) -> Result<()>;++ /// Atomically removes selected sealed descriptors and returns the removed set.+ async fn prune_sealed(+ &self,+ scope: &ScopeKey,+ removable_paths: &[String],+ ) -> Result<Vec<SegmentDescriptor>>;++}++/// Bootstrap every stable provider partition from its raw-primary manifest.+pub(crate) async fn bootstrap_account(+ provider: &dyn MultiWriteProvider,+ store: &MetadataStore,+ account_id: &str,+ partitions: &PartitionsManifest,+) -> Result<()> {+ for (&partition_id, partition) in &partitions.partitions {+ if partition.state != PartitionState::Stable {+ continue;+ }+ let scope = ScopeKey {+ account_id: account_id.to_string(),+ partition_id,+ epoch: partitions.epoch,+ };+ let path = store.paths().segment_manifest(account_id, partition_id)?.0;+ provider.bootstrap_manifest(&scope, store.read_json(&path).await?).await?;+ }+ Ok(())+}++/// Read and validate one immutable segment through the shared metadata store.+async fn read_sealed_segment(+ store: &MetadataStore,+ scope: &ScopeKey,+ descriptor: &SegmentDescriptor,+) -> Result<Vec<SegmentRecord>> {+ let directory = store+ .paths()+ .segments_dir(&scope.account_id, scope.partition_id)?+ .0;+ let bytes = store+ .read_bytes(&format!("{directory}/{}", descriptor.path))+ .await?;+ if bytes.len() as u64 != descriptor.byte_len+ || sha256_hex(&bytes) != descriptor.checksum.as_deref().unwrap_or_default()+ {+ return Err(Error::Serialization(+ "sealed segment bytes do not match descriptor".to_string(),+ ));+ }+ let decoded = decode_segment(&bytes)?;+ if decoded.truncated_tail+ || decoded.valid_bytes != descriptor.byte_len+ || decoded.records.len() != descriptor.record_count as usize+ || decoded.records.iter().enumerate().any(|(index, record)| {+ record.seq+ != descriptor+ .segment_from_seq+ .checked_add(index as u64)+ .unwrap_or(u64::MAX)+ })+ {+ return Err(Error::Serialization(+ "sealed segment records do not match descriptor".to_string(),+ ));+ }+ Ok(decoded.records)+}++/// Select and validate one complete half-open sequence range.+fn select_record_range(+ records: impl IntoIterator<Item = SegmentRecord>,+ from_seq_inclusive: u64,+ to_seq_exclusive: u64,+) -> Result<Vec<SegmentRecord>> {+ let range_len = to_seq_exclusive+ .checked_sub(from_seq_inclusive)+ .ok_or_else(|| Error::invalid_operation("committed range is reversed"))?;+ let expected_len = usize::try_from(range_len)+ .map_err(|_| Error::invalid_operation("committed range is too large"))?;+ let selected = records+ .into_iter()+ .filter(|record| {+ record.seq >= from_seq_inclusive && record.seq < to_seq_exclusive+ })+ .collect::<Vec<_>>();+ if selected.len() != expected_len+ || selected.iter().enumerate().any(|(index, record)| {+ record.seq+ != from_seq_inclusive+ .checked_add(index as u64)+ .unwrap_or(u64::MAX)+ })+ {+ return Err(Error::Serialization(+ "committed record range is incomplete".to_string(),+ ));+ }+ Ok(selected)+}diff --git a/crates/ragfs/src/multibackend/provider/tests/test_filesystem.rs b/crates/ragfs/src/multibackend/provider/tests/test_filesystem.rsnew file mode 100644index 000000000..636eb4381--- /dev/null+++ b/crates/ragfs/src/multibackend/provider/tests/test_filesystem.rs@@ -0,0 +1,72 @@+use super::*;+use crate::core::FileSystem;+use crate::lock::{MemoryPathLockProvider, PathLockConfig, PathLockManager};+use crate::plugins::memfs::MemFileSystem;++/// Verifies flush publishes a contiguous range that committed reads can recover.+#[tokio::test]+async fn flush_publishes_a_complete_committed_range() {+ let raw_primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let pathlock_manager = Arc::new(PathLockManager::new(+ raw_primary.clone(),+ Arc::new(MemoryPathLockProvider::new()),+ PathLockConfig::default(),+ ));+ let store = Arc::new(MetadataStore::new(raw_primary, pathlock_manager, "/local").unwrap());+ let partitions = store.initialize_account("account", 1, &[]).await.unwrap();+ let provider = FilesystemProvider::new(store);+ let scope = ScopeKey {+ account_id: "account".to_string(),+ partition_id: 0,+ epoch: partitions.epoch,+ };++ let result = provider+ .flush(+ None,+ vec![+ PendingEvent {+ operation_id: uuid::Uuid::from_u128(1),+ account_id: "account".to_string(),+ kind: PendingEventKind::Data(SegmentEventType::Write),+ path: "/local/account/first.txt".to_string(),+ destination_path: None,+ route_hint: None,+ },+ PendingEvent {+ operation_id: uuid::Uuid::from_u128(2),+ account_id: "account".to_string(),+ kind: PendingEventKind::Data(SegmentEventType::Remove),+ path: "/local/account/second.txt".to_string(),+ destination_path: None,+ route_hint: None,+ },+ ],+ )+ .await+ .unwrap();++ assert_eq!(result.scope, scope);+ assert_eq!((result.first_seq, result.last_seq), (1, 2));+ let records = provider.read_committed_range(&scope, 1, 3).await.unwrap();+ assert_eq!(+ records,+ vec![+ SegmentRecord {+ seq: 1,+ event_type: SegmentEventType::Write,+ path: "/local/account/first.txt".to_string(),+ op_id: None,+ destination_path: None,+ },+ SegmentRecord {+ seq: 2,+ event_type: SegmentEventType::Remove,+ path: "/local/account/second.txt".to_string(),+ op_id: None,+ destination_path: None,+ },+ ]+ );+ assert!(provider.read_committed_range(&scope, 2, 4).await.is_err());+}diff --git a/crates/ragfs/src/multibackend/router.rs b/crates/ragfs/src/multibackend/router.rsnew file mode 100644index 000000000..209df71f2--- /dev/null+++ b/crates/ragfs/src/multibackend/router.rs@@ -0,0 +1,274 @@+//! Logical path mapping and account partition routing.++use std::collections::HashSet;+use std::sync::Arc;++use xxhash_rust::xxh3::xxh3_64;++use crate::core::errors::{Error, Result};+use crate::multibackend::constants::{+ CHECKPOINTS_DIR, MANIFEST_FILE, MULTIWRITE_MOUNT_PREFIX, MULTIWRITE_PROTOCOL_FILE,+ PARTITIONS_DIR, SEGMENTS_DIR, SYSTEM_DIR, VBUCKETS,+};+use crate::multibackend::meta::MetadataStore;+use crate::multibackend::model::{PartitionContext, PartitionsManifest, RouteEntry, ScopeKey};++/// Maps canonical logical metadata paths to raw backend paths.+#[derive(Debug, Clone)]+pub struct MultiWritePaths {+ mount_prefix: String,+}++impl MultiWritePaths {+ /// Create a path mapper for one canonical absolute mount prefix.+ pub fn new(mount_prefix: &str) -> Result<Self> {+ validate_canonical_path(mount_prefix)?;+ if mount_prefix != MULTIWRITE_MOUNT_PREFIX {+ return Err(Error::invalid_path(format!(+ "unsupported multi-write mount prefix: {mount_prefix}"+ )));+ }+ Ok(Self {+ mount_prefix: mount_prefix.to_string(),+ })+ }++ /// Return logical and backend paths for the mount protocol manifest.+ pub fn mount_protocol(&self) -> (String, String) {+ self.pair(&format!(+ "{}/{SYSTEM_DIR}/{MULTIWRITE_PROTOCOL_FILE}",+ self.mount_prefix+ ))+ }++ /// Return logical and backend paths for an account partitions directory.+ pub fn account_partitions(&self, account_id: &str) -> Result<(String, String)> {+ validate_account(account_id)?;+ Ok(self.pair(&format!(+ "{}/{account_id}/{SYSTEM_DIR}/{PARTITIONS_DIR}",+ self.mount_prefix+ )))+ }++ /// Return logical and backend paths for an account partitions manifest.+ pub fn account_manifest(&self, account_id: &str) -> Result<(String, String)> {+ let (logical, _) = self.account_partitions(account_id)?;+ Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}")))+ }++ /// Return logical and backend paths for one partition manifest.+ pub fn partition_manifest(+ &self,+ account_id: &str,+ partition_id: u32,+ ) -> Result<(String, String)> {+ let (logical, _) = self.account_partitions(account_id)?;+ Ok(self.pair(&format!("{logical}/{partition_id}/{MANIFEST_FILE}")))+ }++ /// Return logical and backend paths for one partition segments directory.+ pub fn segments_dir(&self, account_id: &str, partition_id: u32) -> Result<(String, String)> {+ let (logical, _) = self.account_partitions(account_id)?;+ Ok(self.pair(&format!("{logical}/{partition_id}/{SEGMENTS_DIR}")))+ }++ /// Return logical and backend paths for one segment manifest.+ pub fn segment_manifest(+ &self,+ account_id: &str,+ partition_id: u32,+ ) -> Result<(String, String)> {+ let (logical, _) = self.segments_dir(account_id, partition_id)?;+ Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}")))+ }++ /// Return logical and backend paths for one partition checkpoints directory.+ pub fn checkpoints_dir(&self, account_id: &str, partition_id: u32) -> Result<(String, String)> {+ let (logical, _) = self.account_partitions(account_id)?;+ Ok(self.pair(&format!("{logical}/{partition_id}/{CHECKPOINTS_DIR}")))+ }++ /// Return logical and backend paths for one checkpoints manifest.+ pub fn checkpoints_manifest(+ &self,+ account_id: &str,+ partition_id: u32,+ ) -> Result<(String, String)> {+ let (logical, _) = self.checkpoints_dir(account_id, partition_id)?;+ Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}")))+ }++ /// Convert a canonical logical account path to its raw backend path.+ pub fn backend_path(&self, account_id: &str, logical_path: &str) -> Result<String> {+ validate_account(account_id)?;+ let account_root = format!("{}/{account_id}", self.mount_prefix);+ if logical_path != account_root+ && !logical_path+ .strip_prefix(&account_root)+ .is_some_and(|suffix| suffix.starts_with('/'))+ {+ return Err(Error::invalid_path(format!(+ "logical path does not belong to account '{account_id}': {logical_path}"+ )));+ }+ self.raw_backend_path(logical_path)+ }++ /// Convert any canonical logical mount path to its raw backend path.+ pub fn raw_backend_path(&self, logical_path: &str) -> Result<String> {+ validate_canonical_path(logical_path)?;+ let suffix = logical_path+ .strip_prefix(&self.mount_prefix)+ .filter(|suffix| suffix.starts_with('/'))+ .ok_or_else(|| {+ Error::invalid_path(format!(+ "logical path is outside mount '{}': {logical_path}",+ self.mount_prefix+ ))+ })?;+ Ok(suffix.to_string())+ }++ /// Hash a canonical full logical account path into its fixed vbucket.+ pub fn vbucket(&self, account_id: &str, logical_path: &str) -> Result<u32> {+ self.backend_path(account_id, logical_path)?;+ Ok((xxh3_64(logical_path.as_bytes()) & u64::from(VBUCKETS - 1)) as u32)+ }++ /// Pair one validated logical path with its raw backend form.+ fn pair(&self, logical_path: &str) -> (String, String) {+ (+ logical_path.to_string(),+ logical_path[self.mount_prefix.len()..].to_string(),+ )+ }+}++/// Routes account paths through the current persisted partitions manifest.+pub struct AccountRouter {+ store: Arc<MetadataStore>,+}++impl AccountRouter {+ /// Create an account router backed by the supplied metadata store.+ pub fn new(store: Arc<MetadataStore>) -> Self {+ Self { store }+ }++ /// Route one canonical logical account path to its partition context.+ pub async fn route(&self, account_id: &str, logical_path: &str) -> Result<PartitionContext> {+ let bucket = self.store.paths().vbucket(account_id, logical_path)?;+ let manifest_path = self.store.paths().account_manifest(account_id)?.0;+ let manifest: PartitionsManifest = self.store.read_json(&manifest_path).await?;+ manifest.validate()?;+ let index = manifest.routes.partition_point(|route| route.end < bucket);+ let route = manifest+ .routes+ .get(index)+ .ok_or_else(|| Error::Serialization("vbucket route is missing".to_string()))?;+ let segment_dir = self+ .store+ .paths()+ .segments_dir(account_id, route.partition)?+ .1;+ Ok(PartitionContext {+ scope: ScopeKey {+ account_id: account_id.to_string(),+ partition_id: route.partition,+ epoch: manifest.epoch,+ },+ segment_dir,+ })+ }+}++/// Validate compressed complete routes against the known partition identifiers.+pub fn validate_routes(+ routes: &[RouteEntry],+ partition_ids: impl IntoIterator<Item = u32>,+) -> Result<()> {+ let partition_ids = partition_ids.into_iter().collect::<HashSet<_>>();+ let mut next = 0;+ let mut previous_owner = None;+ for route in routes {+ route.validate()?;+ if route.start != next {+ return Err(Error::Serialization(+ "routes are not contiguous".to_string(),+ ));+ }+ if !partition_ids.contains(&route.partition) {+ return Err(Error::Serialization("route owner is unknown".to_string()));+ }+ if previous_owner == Some(route.partition) {+ return Err(Error::Serialization(+ "adjacent routes have the same owner".to_string(),+ ));+ }+ next = route.end + 1;+ previous_owner = Some(route.partition);+ }+ if next != VBUCKETS {+ return Err(Error::Serialization(+ "routes do not cover all vbuckets".to_string(),+ ));+ }+ Ok(())+}++/// Build deterministic balanced routes for the requested initial partition count.+pub fn build_initial_routes(partition_count: u32) -> Result<Vec<RouteEntry>> {+ if !(1..=VBUCKETS).contains(&partition_count) {+ return Err(Error::Serialization(format!(+ "partition count must be between 1 and {VBUCKETS}"+ )));+ }+ let owners = (0..VBUCKETS)+ .map(|bucket| { ((u64::from(bucket) * u64::from(partition_count)) / u64::from(VBUCKETS)) as u32})+ .collect::<Vec<_>>();+ let routes = compress_owners(&owners);+ validate_routes(&routes, 0..partition_count)?;+ Ok(routes)+}++/// Compress a complete owner table into inclusive route ranges.+pub(crate) fn compress_owners(owners: &[u32]) -> Vec<RouteEntry> {+ let mut routes = Vec::new();+ let mut start = 0;+ for index in 1..=owners.len() {+ if index == owners.len() || owners[index] != owners[start] {+ routes.push(RouteEntry {+ start: start as u32,+ end: index as u32 - 1,+ partition: owners[start],+ });+ start = index;+ }+ }+ routes+}++/// Validate an account identifier as one canonical path component.+fn validate_account(account_id: &str) -> Result<()> {+ if account_id.is_empty() || account_id == "." || account_id == ".." || account_id.contains('/')+ {+ return Err(Error::invalid_path(format!(+ "invalid account identifier: {account_id}"+ )));+ }+ Ok(())+}++/// Validate an absolute path without empty, dot, or trailing components.+fn validate_canonical_path(path: &str) -> Result<()> {+ if !path.starts_with('/')+ || path.len() <= 1+ || path.ends_with('/')+ || path.split('/').skip(1).any(|part| {+ part.is_empty() || part == "." || part == ".."+ })+ {+ return Err(Error::invalid_path(format!("non-canonical path: {path}")));+ }+ Ok(())+}diff --git a/crates/ragfs/src/multibackend/runtime.rs b/crates/ragfs/src/multibackend/runtime.rsnew file mode 100644index 000000000..c9f2e16b8--- /dev/null+++ b/crates/ragfs/src/multibackend/runtime.rs@@ -0,0 +1,755 @@+//! Foreground event queue and metadata flush worker for V2 multi-write.++use std::collections::{HashMap, HashSet};+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+use std::sync::{Arc, Mutex as StdMutex};+use std::time::Duration;++use tokio::sync::{mpsc, watch};+use tokio::task::JoinHandle;+use tracing::{debug, error};++use crate::core::context::{FsContextInner, FS_CTX};+use crate::core::errors::{Error, Result};+use crate::core::filesystem::FileSystem;+use crate::multibackend::catch_up::{CatchUpTarget, CatchUpWorker};+use crate::multibackend::checkpoint::{CheckpointReader, CheckpointWorker};+use crate::multibackend::codec::{decode_segment, sha256_hex};+use crate::multibackend::constants::SYSTEM_DIR;+use crate::multibackend::meta::{MetadataStore, MultiWriteWorker};+use crate::multibackend::model::{+ DirectoryEvent, DirectoryOperation, MarkerPosition, PartitionContext, PartitionState,+ PartitionsManifest, PendingEvent, PendingEventKind, ScopeKey, SegmentEventType,+};+use crate::multibackend::provider::{bootstrap_account, MultiWriteProvider};+use crate::multibackend::router::AccountRouter;++/// Read active account ids from the encrypted or plain primary registry.+pub(crate) async fn read_active_accounts(primary: &dyn FileSystem) -> Result<HashSet<String>> {+ let context = Arc::new(+ FsContextInner::new(SYSTEM_DIR)+ .with_bypass_cache(true)+ .with_auto_pathlock_disabled(),+ );+ let bytes = match FS_CTX+ .scope(context, primary.read("/_system/accounts.json", 0, 0))+ .await+ {+ Ok(bytes) => bytes,+ Err(Error::NotFound(_)) => return Ok(HashSet::new()),+ Err(error) => return Err(error),+ };+ let value: serde_json::Value = serde_json::from_slice(&bytes)?;+ let accounts = value+ .get("accounts")+ .and_then(serde_json::Value::as_object)+ .ok_or_else(|| Error::Serialization("accounts.json lacks an accounts object".into()))?;+ Ok(accounts+ .iter()+ .filter(|(_, account)| {+ account+ .get("deletion")+ .is_none_or(serde_json::Value::is_null)+ })+ .map(|(account_id, _)| account_id.clone())+ .collect())+}++/// Owns the non-blocking foreground sender and its background flush worker.+pub struct MultiWriteRuntime {+ sender: StdMutex<Option<mpsc::UnboundedSender<PendingEvent>>>,+ activation: watch::Sender<Option<bool>>,+ import_worker: StdMutex<Option<JoinHandle<Result<()>>>>,+ store: Arc<MetadataStore>,+ catch_up_cancelled: watch::Sender<bool>,+ catch_up_wake: Arc<tokio::sync::Notify>,+ running: Arc<AtomicBool>,+ pending_events: Arc<AtomicUsize>,+}++impl MultiWriteRuntime {+ /// Assemble runtime tasks without allowing background work to start.+ pub(crate) async fn prepare(+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+ initial_partitions: u32,+ backup_names: Vec<String>,+ primary: Option<Arc<dyn FileSystem>>,+ targets: Vec<CatchUpTarget>,+ checkpoint_interval: Duration,+ ) -> Self {+ let (sender, receiver) = mpsc::unbounded_channel();+ let running = Arc::new(AtomicBool::new(true));+ let worker_running = running.clone();+ let pending_events = Arc::new(AtomicUsize::new(0));+ let worker_pending_events = pending_events.clone();+ let (activation, _) = watch::channel(None);+ let (catch_up_cancelled, _) = watch::channel(false);+ let catch_up_wake = Arc::new(tokio::sync::Notify::new());+ let flush_primary = primary.clone();+ let checkpoint_state = primary.as_ref().map(|primary| {+ Arc::new(CheckpointWorker::new(+ primary.clone(),+ store.clone(),+ provider.clone(),+ ))+ });+ let checkpoint_reader = Arc::new(CheckpointReader::new(store.clone()));+ if let Some(worker) = checkpoint_state {+ let cancellation = catch_up_cancelled.subscribe();+ let mut activated = activation.subscribe();+ drop(tokio::spawn(async move {+ if wait_for_start(&mut activated).await {+ worker.run(checkpoint_interval, cancellation).await;+ }+ }));+ }+ let catch_up_workers = primary.map_or_else(Vec::new, |primary| {+ targets+ .into_iter()+ .map(|target| {+ Arc::new(+ CatchUpWorker::new(+ target.backend_id,+ primary.clone(),+ target.backend,+ store.clone(),+ provider.clone(),+ )+ .with_checkpoint_consumer(checkpoint_reader.clone())+ .with_cancellation(catch_up_cancelled.subscribe()),+ )+ })+ .collect::<Vec<_>>()+ });+ let worker = FlushWorker::new(+ store.clone(),+ provider,+ initial_partitions,+ backup_names,+ flush_primary,+ );+ let mut activated = activation.subscribe();+ drop(tokio::spawn(async move {+ if wait_for_start(&mut activated).await {+ worker.recover_directory_events().await;+ if *activated.borrow() == Some(true) {+ worker.run(receiver, worker_pending_events).await;+ }+ }+ worker_running.store(false, Ordering::SeqCst);+ }));+ for worker in catch_up_workers {+ let mut cancelled = catch_up_cancelled.subscribe();+ let wake = catch_up_wake.clone();+ let mut activated = activation.subscribe();+ drop(tokio::spawn(async move {+ if !wait_for_start(&mut activated).await {+ return;+ }+ while !*cancelled.borrow() {+ if let Err(error) = worker.run_once().await {+ debug!(error = %error, "multi-write catch-up round failed");+ }+ tokio::select! {+ _ = tokio::time::sleep(Duration::from_secs(300)) => {}+ _ = wake.notified() => {}+ changed = cancelled.changed() => {+ if changed.is_err() || *cancelled.borrow() {+ break;+ }+ }+ }+ }+ }));+ }+ Self {+ sender: StdMutex::new(Some(sender)),+ activation,+ import_worker: StdMutex::new(None),+ store,+ catch_up_cancelled,+ catch_up_wake,+ running,+ pending_events,+ }+ }++ /// Attach the full-data import task to this runtime's lifecycle.+ pub(crate) fn set_import_worker(+ &self,+ worker: impl std::future::Future<Output = Result<()>> + Send + 'static,+ ) {+ let mut activated = self.activation.subscribe();+ let worker = tokio::spawn(async move {+ if wait_for_start(&mut activated).await {+ worker.await+ } else {+ Ok(())+ }+ });+ *self+ .import_worker+ .lock()+ .unwrap_or_else(|error| error.into_inner()) = Some(worker);+ }++ /// Release prepared runtime work after the mount becomes visible.+ pub(crate) fn activate(&self) {+ self.activation.send_replace(Some(true));+ }++ /// Submit one event without waiting for metadata work or taking business locks.+ pub fn submit(&self, event: PendingEvent) {+ let failed = {+ let sender = self+ .sender+ .lock()+ .unwrap_or_else(|error| error.into_inner());+ match sender.as_ref() {+ Some(sender) => {+ self.pending_events.fetch_add(1, Ordering::Relaxed);+ sender.send(event).err().map(|error| {+ self.pending_events.fetch_sub(1, Ordering::Relaxed);+ error.0+ })+ }+ None => Some(event),+ }+ };+ if let Some(event) = failed {+ self.store.record_worker_error(MultiWriteWorker::Flush);+ error!(+ operation_id = %event.operation_id,+ account = %event.account_id,+ event_kind = ?event.kind,+ "multi-write event queue is closed"+ );+ }+ self.catch_up_wake.notify_waiters();+ }++ /// Return events still waiting in the foreground channel.+ pub(crate) fn pending_event_count(&self) -> usize {+ self.pending_events.load(Ordering::Relaxed)+ }++ /// Return whether the flush worker has not exited yet.+ pub fn is_running(&self) -> bool {+ self.running.load(Ordering::SeqCst)+ }+}++impl Drop for MultiWriteRuntime {+ /// Signal background tasks when the owning wrapper is dropped.+ fn drop(&mut self) {+ self.sender+ .lock()+ .unwrap_or_else(|error| error.into_inner())+ .take();+ self.activation.send_replace(Some(false));+ if let Some(worker) = self+ .import_worker+ .lock()+ .unwrap_or_else(|error| error.into_inner())+ .take()+ {+ worker.abort();+ }+ self.catch_up_cancelled.send_replace(true);+ self.catch_up_wake.notify_waiters();+ }+}++/// Wait until a prepared runtime gate is started or stopped.+async fn wait_for_start(gate: &mut watch::Receiver<Option<bool>>) -> bool {+ gate.wait_for(|state| state.is_some())+ .await+ .is_ok_and(|state| *state == Some(true))+}++#[cfg(test)]+#[path = "tests/test_runtime.rs"]+mod tests;++/// Drains queued events and persists normal file events by account and scope.+pub struct FlushWorker {+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+ primary: Option<Arc<dyn FileSystem>>,+ router: AccountRouter,+ initial_partitions: u32,+ backup_names: Vec<String>,+}++impl FlushWorker {+ /// Create a worker using one metadata store and persistence provider.+ pub fn new(+ store: Arc<MetadataStore>,+ provider: Arc<dyn MultiWriteProvider>,+ initial_partitions: u32,+ backup_names: Vec<String>,+ primary: Option<Arc<dyn FileSystem>>,+ ) -> Self {+ Self {+ router: AccountRouter::new(store.clone()),+ store,+ provider,+ primary,+ initial_partitions,+ backup_names,+ }+ }++ /// Drain the receiver until every sender is closed and queued event is consumed.+ pub async fn run(+ mut self,+ mut receiver: mpsc::UnboundedReceiver<PendingEvent>,+ pending_events: Arc<AtomicUsize>,+ ) {+ while let Some(first) = receiver.recv().await {+ let mut batch = vec![first];+ while let Ok(event) = receiver.try_recv() {+ batch.push(event);+ }+ let attempted = batch.len();+ self.flush_batch(batch).await;+ pending_events.fetch_sub(attempted, Ordering::Relaxed);+ }+ }++ /// Flush one batch while recording and dropping failed events.+ async fn flush_batch(&mut self, batch: Vec<PendingEvent>) {+ let mut failed = false;+ let mut accounts = HashMap::<String, Vec<PendingEvent>>::new();+ for event in batch {+ if matches!(event.kind, PendingEventKind::DeleteAccount) {+ accounts+ .entry(SYSTEM_DIR.to_string())+ .or_default()+ .push(Self::delete_scope_event(event));+ continue;+ }+ accounts+ .entry(event.account_id.clone())+ .or_default()+ .push(event);+ }++ for (account, events) in accounts {+ match self.account_exists(&account).await {+ Ok(false) => continue,+ Err(error) => {+ failed = true;+ error!(account = %account, error = %error,+ "multi-write primary account check failed");+ continue;+ }+ Ok(true) => {}+ }+ match self+ .store+ .initialize_account(&account, self.initial_partitions, &self.backup_names)+ .await+ {+ Ok(partitions) => {+ if let Err(error) = bootstrap_account(+ self.provider.as_ref(),+ self.store.as_ref(),+ &account,+ &partitions,+ )+ .await+ {+ failed = true;+ error!(account = %account, error = %error,+ "multi-write provider bootstrap failed");+ continue;+ }+ }+ Err(error) => {+ failed = true;+ error!(+ account = %account,+ error = %error,+ "multi-write account initialization failed"+ );+ continue;+ }+ }++ let mut groups = HashMap::<ScopeKey, (PartitionContext, Vec<PendingEvent>)>::new();+ for mut event in events {+ match event.kind {+ PendingEventKind::Data(SegmentEventType::Write | SegmentEventType::Remove) => {+ match self.router.route(&account, &event.path).await {+ Ok(route) => {+ event.route_hint = Some(route.clone());+ groups+ .entry(route.scope.clone())+ .or_insert_with(|| (route, Vec::new()))+ .1+ .push(event);+ }+ Err(error) => {+ failed = true;+ error!(+ operation_id = %event.operation_id,+ account = %account,+ event_kind = ?event.kind,+ error = %error,+ "multi-write event routing failed"+ );+ }+ }+ }+ PendingEventKind::Data(+ SegmentEventType::RemoveTree+ | SegmentEventType::MoveTree+ | SegmentEventType::DeleteScope,+ ) => {+ if let Err(error) = self.flush_directory_event(&account, &event).await {+ failed = true;+ error!(+ operation_id = %event.operation_id,+ account = %account,+ event_kind = ?event.kind,+ error = %error,+ "multi-write directory event persistence failed"+ );+ }+ }+ PendingEventKind::DeleteAccount => unreachable!(),+ }+ }++ for (scope, (route, events)) in groups {+ if let Err(error) = self.provider.flush(Some(&route), events).await {+ failed = true;+ error!(+ account = %account,+ partition = scope.partition_id,+ epoch = scope.epoch,+ error = %error,+ "multi-write metadata flush failed"+ );+ }+ }+ }+ if failed {+ self.store.record_worker_error(MultiWriteWorker::Flush);+ }+ }++ /// Convert one account-root deletion into a durable system-scope marker.+ fn delete_scope_event(mut event: PendingEvent) -> PendingEvent {+ event.account_id = SYSTEM_DIR.to_string();+ event.kind = PendingEventKind::Data(SegmentEventType::DeleteScope);+ event.route_hint = None;+ event+ }++ /// Check account membership through the user registry when the primary view is available.+ async fn account_exists(&self, account_id: &str) -> Result<bool> {+ if account_id == SYSTEM_DIR {+ return Ok(true);+ }+ let Some(primary) = &self.primary else {+ return self.store.account_exists(account_id).await;+ };+ Ok(read_active_accounts(primary.as_ref())+ .await?+ .contains(account_id))+ }++ /// Register and persist one directory marker to every captured stable partition.+ async fn flush_directory_event(&self, account: &str, event: &PendingEvent) -> Result<()> {+ let scopes = self.register_directory_event(account, event).await?;+ let mut failed = false;+ for scope in scopes {+ let result = match self.find_or_append_marker(&scope, event).await {+ Ok(position) => {+ self.backfill_position(account, event.operation_id, position)+ .await+ }+ Err(error) => Err(error),+ };+ if let Err(error) = result {+ failed = true;+ error!(+ operation_id = %event.operation_id, account = %account,+ partition = scope.partition_id, epoch = scope.epoch, error = %error,+ "multi-write directory marker scope remains incomplete"+ );+ }+ }+ if failed {+ Err(Error::internal("directory marker scopes remain incomplete"))+ } else {+ Ok(())+ }+ }++ /// Register one operation idempotently and return its missing stable scopes.+ async fn register_directory_event(+ &self,+ account: &str,+ event: &PendingEvent,+ ) -> Result<Vec<ScopeKey>> {+ let persisted = Self::directory_event(event)?;+ self.store+ .update_partitions_manifest(account, move |manifest| {+ let index = manifest+ .directory_events+ .iter()+ .position(|candidate| candidate.op_id == persisted.op_id);+ if let Some(index) = index {+ let existing = &manifest.directory_events[index];+ if existing.operation != persisted.operation+ || existing.source_path != persisted.source_path+ || existing.destination_path != persisted.destination_path+ {+ return Err(Error::invalid_operation(+ "directory op_id was reused with different content",+ ));+ }+ } else {+ manifest.directory_events.push(persisted);+ }+ let event = manifest+ .directory_events+ .iter()+ .find(|candidate| candidate.op_id == event.operation_id)+ .unwrap();+ Ok(Self::missing_scopes(account, manifest, &event.positions))+ })+ .await+ }++ /// Return Stable scopes that do not yet have a marker position.+ fn missing_scopes(+ account: &str,+ manifest: &PartitionsManifest,+ positions: &[MarkerPosition],+ ) -> Vec<ScopeKey> {+ let completed = positions+ .iter()+ .map(|position| position.partition_id)+ .collect::<HashSet<_>>();+ manifest+ .partitions+ .iter()+ .filter(|(id, partition)| {+ partition.state == PartitionState::Stable && !completed.contains(id)+ })+ .map(|(&partition_id, _)| ScopeKey {+ account_id: account.to_string(),+ partition_id,+ epoch: manifest.epoch,+ })+ .collect()+ }++ /// Convert one queued directory marker into its persisted notification.+ fn directory_event(event: &PendingEvent) -> Result<DirectoryEvent> {+ let operation = match event.kind {+ PendingEventKind::Data(SegmentEventType::RemoveTree) => DirectoryOperation::RemoveTree,+ PendingEventKind::Data(SegmentEventType::MoveTree) => DirectoryOperation::MoveTree,+ PendingEventKind::Data(SegmentEventType::DeleteScope) => {+ DirectoryOperation::DeleteScope+ }+ _ => {+ return Err(Error::invalid_operation(+ "directory notification requires a tree event",+ ))+ }+ };+ Ok(DirectoryEvent {+ op_id: event.operation_id,+ operation,+ source_path: event.path.clone(),+ destination_path: event.destination_path.clone(),+ positions: Vec::new(),+ })+ }++ /// Idempotently persist one marker position under the manifest Exact lock.+ async fn backfill_position(+ &self,+ account: &str,+ operation_id: uuid::Uuid,+ position: MarkerPosition,+ ) -> Result<()> {+ self.store+ .update_partitions_manifest(account, move |manifest| {+ let event = manifest+ .directory_events+ .iter_mut()+ .find(|event| event.op_id == operation_id)+ .ok_or_else(|| Error::not_found(operation_id.to_string()))?;+ if let Some(existing) = event+ .positions+ .iter()+ .find(|existing| existing.partition_id == position.partition_id)+ {+ if existing != &position {+ return Err(Error::invalid_operation(+ "conflicting directory marker positions",+ ));+ }+ return Ok(());+ }+ event.positions.push(position);+ event+ .positions+ .sort_by_key(|position| position.partition_id);+ Ok(())+ })+ .await+ }++ /// Recover every incomplete notification discoverable before queue processing starts.+ async fn recover_directory_events(&self) {+ let accounts = match self.store.initialized_accounts().await {+ Ok(accounts) => accounts,+ Err(error) => {+ self.store.record_worker_error(MultiWriteWorker::Flush);+ error!(error = %error, "multi-write directory recovery discovery failed");+ return;+ }+ };+ for account in accounts {+ match self.recover_account(&account).await {+ Ok(()) | Err(Error::NotFound(_)) => {}+ Err(error) => {+ self.store.record_worker_error(MultiWriteWorker::Flush);+ error!(account = %account, error = %error,+ "multi-write directory recovery failed");+ }+ }+ }+ }++ /// Recover all currently missing marker positions for one account.+ async fn recover_account(&self, account: &str) -> Result<()> {+ let manifest_path = self.store.paths().account_manifest(account)?.0;+ let manifest: PartitionsManifest = self.store.read_json(&manifest_path).await?;+ manifest.validate()?;+ let mut failed = false;+ for persisted in &manifest.directory_events {+ let event = PendingEvent {+ operation_id: persisted.op_id,+ account_id: account.to_string(),+ kind: PendingEventKind::Data(match persisted.operation {+ DirectoryOperation::RemoveTree => SegmentEventType::RemoveTree,+ DirectoryOperation::MoveTree => SegmentEventType::MoveTree,+ DirectoryOperation::DeleteScope => SegmentEventType::DeleteScope,+ }),+ path: persisted.source_path.clone(),+ destination_path: persisted.destination_path.clone(),+ route_hint: None,+ };+ for scope in Self::missing_scopes(account, &manifest, &persisted.positions) {+ let result = self.find_or_append_marker(&scope, &event).await;+ match result {+ Ok(position) => {+ if let Err(error) = self+ .backfill_position(account, event.operation_id, position)+ .await+ {+ failed = true;+ error!(operation_id = %event.operation_id, account = %account,+ partition = scope.partition_id, error = %error,+ "multi-write recovered marker position update failed");+ }+ }+ Err(error) => {+ failed = true;+ error!(operation_id = %event.operation_id, account = %account,+ partition = scope.partition_id, error = %error,+ "multi-write directory marker recovery scope failed");+ }+ }+ }+ }+ if failed {+ Err(Error::internal(+ "directory marker recovery remains incomplete",+ ))+ } else {+ Ok(())+ }+ }++ /// Reuse a retained sealed marker or append it when none exists.+ async fn find_or_append_marker(+ &self,+ scope: &ScopeKey,+ event: &PendingEvent,+ ) -> Result<MarkerPosition> {+ match self.find_marker(scope, event).await? {+ Some(position) => Ok(position),+ None => self.provider.append_marker(scope, event).await,+ }+ }++ /// Find one operation marker in currently retained sealed segments.+ async fn find_marker(+ &self,+ scope: &ScopeKey,+ event: &PendingEvent,+ ) -> Result<Option<MarkerPosition>> {+ let manifest = self.provider.read_manifest(scope).await?;+ let directory = self+ .store+ .paths()+ .segments_dir(&scope.account_id, scope.partition_id)?+ .0;+ for descriptor in manifest+ .segments+ .iter()+ .filter(|segment| segment.segment_to_seq.is_some())+ {+ let bytes = self+ .store+ .read_bytes(&format!("{directory}/{}", descriptor.path))+ .await?;+ if bytes.len() as u64 != descriptor.byte_len+ || descriptor.checksum.as_deref() != Some(&sha256_hex(&bytes))+ {+ return Err(Error::Serialization(+ "sealed marker segment mismatch".to_string(),+ ));+ }+ let decoded = decode_segment(&bytes)?;+ if decoded.truncated_tail || decoded.valid_bytes != descriptor.byte_len {+ return Err(Error::Serialization(+ "sealed marker segment is truncated".to_string(),+ ));+ }+ if let Some(record) = decoded+ .records+ .iter()+ .find(|record| record.op_id == Some(event.operation_id))+ {+ let PendingEventKind::Data(kind) = event.kind else {+ unreachable!()+ };+ if record.event_type != kind+ || record.path != event.path+ || record.destination_path != event.destination_path+ {+ return Err(Error::Serialization(+ "directory op_id marker mismatch".to_string(),+ ));+ }+ return Ok(Some(MarkerPosition {+ partition_id: scope.partition_id,+ epoch: scope.epoch,+ seq: record.seq,+ }));+ }+ }+ Ok(None)+ }+}diff --git a/crates/ragfs/src/multibackend/tests/test_catch_up.rs b/crates/ragfs/src/multibackend/tests/test_catch_up.rsnew file mode 100644index 000000000..2ee8a5a33--- /dev/null+++ b/crates/ragfs/src/multibackend/tests/test_catch_up.rs@@ -0,0 +1,89 @@+use super::*;++use std::collections::HashMap;+use std::sync::Arc;++use crate::core::context::{FsContextInner, FS_CTX};+use crate::core::encryption_wrapper::EncryptionWrappedFS;+use crate::core::{MountableFS, PluginConfig};+use crate::crypto;+use crate::lock::{MemoryPathLockProvider, PathLockConfig, PathLockManager};+use crate::plugins::memfs::MemFileSystem;+use crate::plugins::MemFSPlugin;++/// Build one encrypted MemFS backup and expose its raw mounted storage.+async fn encrypted_backup() -> (Arc<MountableFS>, EncryptionWrappedFS) {+ let raw = Arc::new(MountableFS::new());+ raw.register_plugin(MemFSPlugin).await;+ raw.mount(PluginConfig::single_backend(+ "memfs",+ "/mem",+ HashMap::new(),+ ))+ .await+ .unwrap();+ let manager = Arc::new(PathLockManager::new(+ raw.clone() as Arc<dyn FileSystem>,+ Arc::new(MemoryPathLockProvider::new()),+ PathLockConfig::default(),+ ));+ raw.set_pathlock_manager(manager.clone()).await;+ let encrypted = EncryptionWrappedFS::new(+ raw.clone(),+ [7_u8; 32],+ crypto::PROVIDER_LOCAL,+ manager,+ "/mem".to_string(),+ );+ (raw, encrypted)+}++/// Verify encrypted writes preserve their historical physical byte count.+#[tokio::test]+async fn encrypted_write_returns_physical_length() {+ let (raw, encrypted) = encrypted_backup().await;+ let data = b"plain";+ let written = FS_CTX+ .scope(+ Arc::new(FsContextInner::new("acct")),+ encrypted.write("/mem/file.txt", data, 0, WriteFlag::Create),+ )+ .await+ .unwrap();+ let stored = raw.read("/mem/file.txt", 0, 0).await.unwrap();++ assert_eq!(written, stored.len() as u64);+ assert!(written > data.len() as u64);+}++/// Verify catch-up accepts an encrypted backend's physical byte count.+#[tokio::test]+async fn reconcile_file_accepts_encrypted_write_length() {+ let primary = MemFileSystem::new();+ primary+ .ensure_parent_dirs("/mem/file.txt", 0o755)+ .await+ .unwrap();+ primary+ .write("/mem/file.txt", b"plain", 0, WriteFlag::Create)+ .await+ .unwrap();+ let (_, encrypted) = encrypted_backup().await;++ FS_CTX+ .scope(+ Arc::new(FsContextInner::new("acct")),+ reconcile_file(&primary, &encrypted, "/mem/file.txt"),+ )+ .await+ .unwrap();++ let copied = FS_CTX+ .scope(+ Arc::new(FsContextInner::new("acct")),+ encrypted.read("/mem/file.txt", 0, 0),+ )+ .await+ .unwrap();+ assert_eq!(copied, b"plain");+}diff --git a/crates/ragfs/src/multibackend/tests/test_codec.rs b/crates/ragfs/src/multibackend/tests/test_codec.rsnew file mode 100644index 000000000..2dc542fea--- /dev/null+++ b/crates/ragfs/src/multibackend/tests/test_codec.rs@@ -0,0 +1,35 @@+use super::*;+use uuid::Uuid;++/// Verifies segment round trips and recovery from an incomplete final frame.+#[test]+fn segment_round_trip_recovers_complete_prefix_from_truncated_tail() {+ let records = vec![+ SegmentRecord {+ seq: 1,+ event_type: SegmentEventType::Write,+ path: "/local/account/file.txt".to_string(),+ op_id: None,+ destination_path: None,+ },+ SegmentRecord {+ seq: 2,+ event_type: SegmentEventType::MoveTree,+ path: "/local/account/source".to_string(),+ op_id: Some(Uuid::from_u128(7)),+ destination_path: Some("/local/account/destination".to_string()),+ },+ ];++ let encoded = encode_segment(&records).unwrap();+ let decoded = decode_segment(&encoded).unwrap();+ assert_eq!(decoded.records, records);+ assert_eq!(decoded.valid_bytes, encoded.len() as u64);+ assert!(!decoded.truncated_tail);++ let first_record_bytes = encode_segment(&records[..1]).unwrap();+ let truncated = decode_segment(&encoded[..encoded.len() - 1]).unwrap();+ assert_eq!(truncated.records, records[..1]);+ assert_eq!(truncated.valid_bytes, first_record_bytes.len() as u64);+ assert!(truncated.truncated_tail);+}diff --git a/crates/ragfs/src/multibackend/tests/test_model.rs b/crates/ragfs/src/multibackend/tests/test_model.rsnew file mode 100644index 000000000..1f6678110--- /dev/null+++ b/crates/ragfs/src/multibackend/tests/test_model.rs@@ -0,0 +1,45 @@+use super::*;+use std::collections::BTreeMap;+use uuid::Uuid;++/// Verifies complete routing and directory marker ownership invariants.+#[test]+fn partitions_manifest_rejects_route_gaps_and_stale_markers() {+ let manifest = PartitionsManifest {+ version: 1,+ epoch: 3,+ vbuckets: VBUCKETS,+ partitions: BTreeMap::from([(+ 0,+ PartitionEntry {+ state: PartitionState::Stable,+ },+ )]),+ routes: vec![RouteEntry {+ start: 0,+ end: VBUCKETS - 1,+ partition: 0,+ }],+ directory_events: vec![DirectoryEvent {+ op_id: Uuid::from_u128(1),+ operation: DirectoryOperation::RemoveTree,+ source_path: "/local/account/removed".to_string(),+ destination_path: None,+ positions: vec![MarkerPosition {+ partition_id: 0,+ epoch: 3,+ seq: 1,+ }],+ }],+ };++ assert!(manifest.validate().is_ok());++ let mut route_gap = manifest.clone();+ route_gap.routes[0].start = 1;+ assert!(route_gap.validate().is_err());++ let mut stale_marker = manifest;+ stale_marker.directory_events[0].positions[0].epoch = 2;+ assert!(stale_marker.validate().is_err());+}diff --git a/crates/ragfs/src/multibackend/tests/test_runtime.rs b/crates/ragfs/src/multibackend/tests/test_runtime.rsnew file mode 100644index 000000000..e9523af26--- /dev/null+++ b/crates/ragfs/src/multibackend/tests/test_runtime.rs@@ -0,0 +1,97 @@+use super::*;+use crate::core::{FileSystem, WriteFlag};+use crate::lock::{MemoryPathLockProvider, PathLockConfig, PathLockManager};+use crate::multibackend::provider::FilesystemProvider;+use crate::plugins::memfs::MemFileSystem;++/// Verifies deleted registry entries are excluded while null deletions stay active.+#[tokio::test]+async fn active_accounts_exclude_completed_deletions() {+ let primary = MemFileSystem::new();+ let registry = serde_json::json!({+ "accounts": {+ "active": {},+ "pending": {"deletion": null},+ "deleted": {"deletion": {"deleted_at_ns": 1}}+ }+ });+ primary+ .ensure_parent_dirs("/_system/accounts.json", 0o755)+ .await+ .unwrap();+ primary+ .write(+ "/_system/accounts.json",+ &serde_json::to_vec(®istry).unwrap(),+ 0,+ WriteFlag::Create,+ )+ .await+ .unwrap();++ let accounts = read_active_accounts(&primary).await.unwrap();+ assert_eq!(+ accounts,+ HashSet::from(["active".to_string(), "pending".to_string()])+ );+}++/// Verifies delete events are persisted in the system scope and remove backup data.+#[tokio::test]+async fn delete_account_event_removes_backup_scope() {+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ primary+ .ensure_parent_dirs("/_system/accounts.json", 0o755)+ .await+ .unwrap();+ primary+ .write(+ "/_system/accounts.json",+ br#"{"accounts":{"account":{}}}"#,+ 0,+ WriteFlag::Create,+ )+ .await+ .unwrap();+ backup.mkdir("/account", 0o755).await.unwrap();++ let manager = Arc::new(PathLockManager::new(+ primary.clone(),+ Arc::new(MemoryPathLockProvider::new()),+ PathLockConfig::default(),+ ));+ let store = Arc::new(MetadataStore::new(primary.clone(), manager, "/local").unwrap());+ let provider = Arc::new(FilesystemProvider::new(store.clone()));+ let catch_up = Arc::new(CatchUpWorker::new(+ "backup".to_string(),+ primary.clone(),+ backup.clone(),+ store.clone(),+ provider.clone(),+ ));+ let mut worker = FlushWorker::new(+ store,+ provider,+ 1,+ vec!["backup".to_string()],+ Some(primary),+ );++ worker+ .flush_batch(vec![PendingEvent {+ operation_id: uuid::Uuid::from_u128(1),+ account_id: "account".to_string(),+ kind: PendingEventKind::DeleteAccount,+ path: "/local/account".to_string(),+ destination_path: None,+ route_hint: None,+ }])+ .await;+ catch_up.run_once().await.unwrap();++ assert!(matches!(+ backup.stat("/account").await.unwrap_err(),+ Error::NotFound(_)+ ));+}diff --git a/crates/ragfs/src/multibackend/types.rs b/crates/ragfs/src/multibackend/types.rsindex 78bb71d66..09a4f53ec 100644--- a/crates/ragfs/src/multibackend/types.rs+++ b/crates/ragfs/src/multibackend/types.rs@@ -2,6 +2,8 @@use std::sync::Arc;+#[cfg(feature = "cache")]+use crate::cache_runtime::CacheRuntime;use crate::lock::PathLockManager;/// Encryption context used while assembling one multi-backend mount.@@ -15,6 +17,9 @@ pub struct MultiBackendBuildContext {pub pathlock_manager: Arc<PathLockManager>,/// Mount prefix used to restore manager-visible backend paths.pub backend_prefix: String,+ /// Optional shared cache runtime used by cache-backed metadata.+ #[cfg(feature = "cache")]+ pub cache_runtime: Option<Arc<CacheRuntime>>,}impl MultiBackendBuildContext {diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rsindex 67b50a7dc..463f7ae3b 100644--- a/crates/ragfs/src/plugins/localfs/mod.rs+++ b/crates/ragfs/src/plugins/localfs/mod.rs@@ -897,6 +897,8 @@ impl FileSystem for LocalFileSystem {let mut file = options.open(&local_path).map_err(|e| {if e.kind() == std::io::ErrorKind::NotFound {Error::NotFound(path.to_string())+ } else if e.kind() == std::io::ErrorKind::AlreadyExists {+ Error::AlreadyExists(path.to_string())} else {Error::plugin(format!("failed to open file: {}", e))}diff --git a/crates/ragfs/tests/cache_wrapper.rs b/crates/ragfs/tests/cache_wrapper.rsdeleted file mode 100644index 4c31ab288..000000000--- a/crates/ragfs/tests/cache_wrapper.rs+++ /dev/null@@ -1,1478 +0,0 @@-use async_trait::async_trait;-use ragfs::cache::{- CacheDecision, CacheNamespace, CachePolicy, CacheTraversalMode, CachedFileSystem,-};-use ragfs::cache_runtime::{CacheRuntime, MemoryMockProvider};-use ragfs::core::{- FsContextInner, GrepOptions, GrepResult, MultiWriteWrappedFS, TreeEntry, FS_CTX,-};-use ragfs::plugins::MemFileSystem;-use ragfs::{Error, FileInfo, FileSystem, Result, WriteFlag};-use std::sync::atomic::{AtomicU64, Ordering};-use std::sync::Arc;-use std::time::Duration;--#[derive(Clone)]-struct CountingFileSystem {- inner: Arc<MemFileSystem>,- reads: Arc<AtomicU64>,- read_dirs: Arc<AtomicU64>,- stats: Arc<AtomicU64>,- greps: Arc<AtomicU64>,- trees: Arc<AtomicU64>,- read_delay: Duration,- partial_remove_path: Option<String>,-}--impl CountingFileSystem {- fn new() -> Self {- Self {- inner: Arc::new(MemFileSystem::new()),- reads: Arc::new(AtomicU64::new(0)),- read_dirs: Arc::new(AtomicU64::new(0)),- stats: Arc::new(AtomicU64::new(0)),- greps: Arc::new(AtomicU64::new(0)),- trees: Arc::new(AtomicU64::new(0)),- read_delay: Duration::ZERO,- partial_remove_path: None,- }- }-- fn with_read_delay(mut self, delay: Duration) -> Self {- self.read_delay = delay;- self- }-- fn with_partial_remove_failure(mut self, path: &str) -> Self {- self.partial_remove_path = Some(path.to_string());- self- }-- fn read_count(&self) -> u64 {- self.reads.load(Ordering::Relaxed)- }-- fn read_dir_count(&self) -> u64 {- self.read_dirs.load(Ordering::Relaxed)- }-- fn stat_count(&self) -> u64 {- self.stats.load(Ordering::Relaxed)- }-- fn grep_count(&self) -> u64 {- self.greps.load(Ordering::Relaxed)- }-- fn tree_count(&self) -> u64 {- self.trees.load(Ordering::Relaxed)- }-}--#[async_trait]-impl FileSystem for CountingFileSystem {- async fn create(&self, path: &str) -> Result<()> {- self.inner.create(path).await- }-- async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {- self.inner.mkdir(path, mode).await- }-- async fn remove(&self, path: &str) -> Result<()> {- self.inner.remove(path).await- }-- async fn remove_all(&self, path: &str) -> Result<()> {- if let Some(partial_remove_path) = &self.partial_remove_path {- self.inner.remove(partial_remove_path).await?;- return Err(Error::internal(format!(- "remove_all partially removed {partial_remove_path} under {path}"- )));- }- self.inner.remove_all(path).await- }-- async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {- self.reads.fetch_add(1, Ordering::Relaxed);- if !self.read_delay.is_zero() {- tokio::time::sleep(self.read_delay).await;- }- self.inner.read(path, offset, size).await- }-- async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {- self.inner.write(path, data, offset, flags).await- }-- async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {- self.read_dirs.fetch_add(1, Ordering::Relaxed);- self.inner.read_dir(path).await- }-- async fn stat(&self, path: &str) -> Result<FileInfo> {- self.stats.fetch_add(1, Ordering::Relaxed);- self.inner.stat(path).await- }-- async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {- self.inner.rename(old_path, new_path).await- }-- async fn chmod(&self, path: &str, mode: u32) -> Result<()> {- self.inner.chmod(path, mode).await- }-- async fn truncate(&self, path: &str, size: u64) -> Result<()> {- self.inner.truncate(path, size).await- }-- async fn grep(- &self,- path: &str,- pattern: &str,- options: GrepOptions<'_>,- ) -> Result<GrepResult> {- self.greps.fetch_add(1, Ordering::Relaxed);- self.inner.grep(path, pattern, options).await- }-- async fn tree_directory(- &self,- path: &str,- show_hidden: bool,- node_limit: Option<usize>,- level_limit: Option<usize>,- ) -> Result<Vec<TreeEntry>> {- self.trees.fetch_add(1, Ordering::Relaxed);- self.inner- .tree_directory(path, show_hidden, node_limit, level_limit)- .await- }-}--fn cached_fs(backend: CountingFileSystem) -> (Arc<CachedFileSystem>, Arc<MemoryMockProvider>) {- cached_fs_with_policy(backend, CachePolicy::default())-}--fn cached_fs_with_policy(- backend: CountingFileSystem,- policy: CachePolicy,-) -> (Arc<CachedFileSystem>, Arc<MemoryMockProvider>) {- let provider = Arc::new(MemoryMockProvider::new());- let fs = Arc::new(CachedFileSystem::with_runtime(- Box::new(backend),- CacheRuntime::memory_with_provider(provider.clone()),- CacheNamespace::new("test"),- policy,- ));- (fs, provider)-}--fn cached_fs_with_tracking_provider(- backend: CountingFileSystem,- policy: CachePolicy,-) -> (Arc<CachedFileSystem>, Arc<MemoryMockProvider>) {- cached_fs_with_tracking_provider_instance(backend, policy, Arc::new(MemoryMockProvider::new()))-}--fn cached_fs_with_tracking_provider_instance(- backend: CountingFileSystem,- policy: CachePolicy,- provider: Arc<MemoryMockProvider>,-) -> (Arc<CachedFileSystem>, Arc<MemoryMockProvider>) {- let fs = Arc::new(CachedFileSystem::with_runtime(- Box::new(backend),- CacheRuntime::memory_with_provider(provider.clone()),- CacheNamespace::new("tracking"),- policy,- ));- (fs, provider)-}--#[test]-fn cache_policy_traversal_mode_defaults_to_backend() {- assert_eq!(- CachePolicy::default().traversal_mode(),- CacheTraversalMode::Backend- );-}--#[tokio::test]-async fn cached_filesystem_accepts_the_unified_runtime_without_changing_read_through() {- let backend = CountingFileSystem::new();- backend- .write("/runtime.md", b"runtime", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let fs = CachedFileSystem::with_runtime(- Box::new(backend),- CacheRuntime::memory(),- CacheNamespace::new("runtime"),- CachePolicy::default(),- );-- assert_eq!(fs.read("/runtime.md", 0, 0).await.unwrap(), b"runtime");- assert_eq!(fs.read("/runtime.md", 0, 0).await.unwrap(), b"runtime");- assert_eq!(probe.read_count(), 1);-}--#[tokio::test]-async fn unified_runtime_failure_still_falls_back_to_backend_reads() {- let backend = CountingFileSystem::new();- backend- .write("/fallback.txt", b"backend", 0, WriteFlag::Create)- .await- .unwrap();- let runtime = CacheRuntime::memory();- runtime.close().await.unwrap();- let cached = CachedFileSystem::with_runtime(- Box::new(backend),- runtime,- CacheNamespace::new("runtime-fail-open"),- CachePolicy::default(),- );-- assert_eq!(- cached.read("/fallback.txt", 0, 0).await.unwrap(),- b"backend"- );-}--#[tokio::test]-async fn default_tree_directory_delegates_to_backend() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend- .write("/docs/one.md", b"one", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- let entries = fs.tree_directory("/docs", false, None, None).await.unwrap();-- assert_eq!(entries.len(), 1);- assert_eq!(probe.tree_count(), 1);- assert_eq!(probe.read_dir_count(), 0);-}--#[tokio::test]-async fn default_grep_delegates_to_backend() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend- .write("/docs/one.md", b"needle", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- let result = fs- .grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- ..Default::default()- },- )- .await- .unwrap();-- assert_eq!(result.count, 1);- assert_eq!(probe.grep_count(), 1);- assert_eq!(probe.read_dir_count(), 0);- assert_eq!(probe.read_count(), 0);-}--#[tokio::test]-async fn cached_tree_traversal_reuses_directory_cache_after_warmup() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend.mkdir("/docs/sub", 0o755).await.unwrap();- backend- .write("/docs/one.md", b"one", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/docs/sub/two.md", b"two", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs_with_policy(- backend,- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- let first = fs.tree_directory("/docs", false, None, None).await.unwrap();- assert_eq!(first.len(), 3);- assert_eq!(probe.tree_count(), 0);- assert_eq!(probe.read_dir_count(), 2);-- let second = fs.tree_directory("/docs", false, None, None).await.unwrap();- assert_eq!(second.len(), 3);- assert_eq!(- probe.read_dir_count(),- 2,- "second tree should reuse cached read_dir entries"- );-}--#[tokio::test]-async fn cached_grep_traversal_reuses_directory_and_file_cache_after_warmup() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend.mkdir("/docs/sub", 0o755).await.unwrap();- backend- .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs_with_policy(- backend,- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- let first = fs- .grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- ..Default::default()- },- )- .await- .unwrap();- assert_eq!(first.count, 2);- assert_eq!(probe.grep_count(), 0);- assert_eq!(probe.read_dir_count(), 2);- assert_eq!(probe.read_count(), 2);- assert_eq!(probe.stat_count(), 1);-- let second = fs- .grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- ..Default::default()- },- )- .await- .unwrap();- assert_eq!(second.count, 2);- assert_eq!(- probe.read_dir_count(),- 2,- "second grep should reuse cached read_dir entries"- );- assert_eq!(- probe.read_count(),- 2,- "second grep should reuse cached file contents"- );- assert_eq!(- probe.stat_count(),- 2,- "second grep should only stat the query root, not every cached entry"- );-}--#[tokio::test]-async fn cached_grep_includes_context_from_cached_content() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend- .write(- "/docs/a.md",- b"first\nbefore\nneedle\nafter",- 0,- WriteFlag::Create,- )- .await- .unwrap();- let (fs, _) = cached_fs_with_policy(- backend,- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- let result = fs- .grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- before_context: 2,- after_context: 1,- ..Default::default()- },- )- .await- .unwrap();-- let matched = &result.matches[0];- assert_eq!(matched.before_context.as_ref().unwrap()[0].content, "first");- assert_eq!(- matched.before_context.as_ref().unwrap()[1].content,- "before"- );- assert_eq!(matched.after_context.as_ref().unwrap()[0].content, "after");-}--#[tokio::test]-async fn cached_grep_batches_generation_validation_after_warmup() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend.mkdir("/docs/sub", 0o755).await.unwrap();- backend- .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create)- .await- .unwrap();- let (fs, provider) = cached_fs_with_tracking_provider(- backend,- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- fs.grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- ..Default::default()- },- )- .await- .unwrap();- provider.reset_observed_reads();-- let result = fs- .grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- ..Default::default()- },- )- .await- .unwrap();-- assert_eq!(result.count, 2);- assert!(- provider.batch_get_count() > 0,- "warm cached grep should batch generation validation reads"- );-}--#[tokio::test]-async fn cached_grep_memoizes_generation_keys_within_one_traversal() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend.mkdir("/docs/sub", 0o755).await.unwrap();- backend- .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create)- .await- .unwrap();- let (fs, provider) = cached_fs_with_tracking_provider(- backend,- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- fs.grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- ..Default::default()- },- )- .await- .unwrap();- provider.reset_observed_reads();-- let result = fs- .grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- ..Default::default()- },- )- .await- .unwrap();-- assert_eq!(result.count, 2);- let subtree_keys = provider- .observed_read_keys()- .into_iter()- .filter(|key| key.contains(":subtree:"))- .collect::<Vec<_>>();- let unique = subtree_keys- .iter()- .collect::<std::collections::HashSet<_>>();- assert_eq!(- subtree_keys.len(),- unique.len(),- "one cached grep traversal should not re-read the same generation key"- );-}--#[tokio::test]-async fn cached_grep_scans_cached_files_with_bounded_concurrency() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- for index in 0..8 {- backend- .write(- &format!("/docs/{index}.md"),- b"needle\nplain",- 0,- WriteFlag::Create,- )- .await- .unwrap();- }- let provider = Arc::new(MemoryMockProvider::new().with_get_delay(Duration::from_millis(30)));- let (fs, provider) = cached_fs_with_tracking_provider_instance(- backend,- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- provider,- );-- fs.grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- ..Default::default()- },- )- .await- .unwrap();- provider.reset_observed_reads();-- let result = fs- .grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- ..Default::default()- },- )- .await- .unwrap();-- assert_eq!(result.count, 8);- let max_gets = provider.max_concurrent_gets();- assert!(- max_gets > 1,- "warm cached grep should scan cached file reads concurrently"- );- assert!(- max_gets <= 8,- "cached grep file scan concurrency should stay bounded"- );-}--#[tokio::test]-async fn cached_grep_traversal_matches_default_grep_semantics() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend.mkdir("/docs/sub", 0o755).await.unwrap();- backend.mkdir("/docs/skip", 0o755).await.unwrap();- backend- .write("/docs/a.md", b"Needle\nneedle", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/docs/sub/b.md", b"needle\nmiss", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/docs/skip/c.md", b"needle", 0, WriteFlag::Create)- .await- .unwrap();- let direct = backend.clone();- let (fs, _) = cached_fs_with_policy(- backend,- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- let cached = fs- .grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- case_insensitive: true,- node_limit: Some(2),- exclude_path: Some("/docs/skip"),- level_limit: Some(1),- ..Default::default()- },- )- .await- .unwrap();- let direct_result = direct- .grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- case_insensitive: true,- node_limit: Some(2),- exclude_path: Some("/docs/skip"),- level_limit: Some(1),- ..Default::default()- },- )- .await- .unwrap();-- let tuples = |result: &GrepResult| {- result- .matches- .iter()- .map(|m| (m.file.clone(), m.line, m.content.clone()))- .collect::<Vec<_>>()- };- assert_eq!(tuples(&cached), tuples(&direct_result));-}--#[tokio::test]-async fn cached_grep_traversal_falls_back_for_multiwrite_backend() {- let primary = CountingFileSystem::new();- primary.mkdir("/docs", 0o755).await.unwrap();- primary- .write("/docs/a.md", b"needle", 0, WriteFlag::Create)- .await- .unwrap();- let primary_probe = primary.clone();- let multiwrite = MultiWriteWrappedFS::builder(Arc::new(primary))- .build()- .unwrap();- let fs = CachedFileSystem::with_runtime(- Box::new(multiwrite),- CacheRuntime::memory(),- CacheNamespace::new("grep-multiwrite"),- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- let ctx = Arc::new(FsContextInner::new("acct"));- let result = FS_CTX- .scope(ctx, async {- fs.grep(- "/docs",- "needle",- GrepOptions {- recursive: true,- ..Default::default()- },- )- .await- .unwrap()- })- .await;-- assert_eq!(result.count, 1);- assert_eq!(- primary_probe.grep_count(),- 1,- "multi-write backends must keep their backend grep path"- );- assert_eq!(- primary_probe.read_dir_count(),- 0,- "cached traversal must not bypass multi-write grep semantics"- );-}--#[tokio::test]-async fn cached_tree_traversal_matches_default_tree_semantics() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend.mkdir("/docs/sub", 0o755).await.unwrap();- backend- .write("/docs/a.md", b"a", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/docs/.hidden.md", b"hidden", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/docs/sub/b.md", b"b", 0, WriteFlag::Create)- .await- .unwrap();- let direct = backend.clone();- let (fs, _) = cached_fs_with_policy(- backend,- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- let cached = fs- .tree_directory("/docs", false, None, Some(1))- .await- .unwrap();- let direct_entries = direct- .tree_directory("/docs", false, None, Some(1))- .await- .unwrap();- assert_eq!(- cached- .iter()- .map(|entry| entry.rel_path.as_str())- .collect::<Vec<_>>(),- direct_entries- .iter()- .map(|entry| entry.rel_path.as_str())- .collect::<Vec<_>>()- );-- let with_hidden = fs.tree_directory("/docs", true, None, None).await.unwrap();- assert!(- with_hidden- .iter()- .any(|entry| entry.rel_path == ".hidden.md"),- "show_hidden=true should include hidden files"- );-}--#[tokio::test]-async fn cached_tree_traversal_returns_oversized_directories_without_caching_them() {- let backend = CountingFileSystem::new();- backend.mkdir("/large", 0o755).await.unwrap();- for index in 0..4097 {- backend- .write(- &format!("/large/{index:04}.txt"),- b"x",- 0,- WriteFlag::Create,- )- .await- .unwrap();- }- let probe = backend.clone();- let (fs, _) = cached_fs_with_policy(- backend,- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- assert_eq!(- fs.tree_directory("/large", false, None, None)- .await- .unwrap()- .len(),- 4097- );- assert_eq!(- fs.tree_directory("/large", false, None, None)- .await- .unwrap()- .len(),- 4097- );- assert_eq!(- probe.read_dir_count(),- 2,- "oversized directories should not be cached by tree traversal"- );-}--#[tokio::test]-async fn cached_tree_traversal_falls_back_when_provider_is_unavailable() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend- .write("/docs/a.md", b"a", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let provider = Arc::new(MemoryMockProvider::new());- provider.set_unavailable(true);- let fs = CachedFileSystem::with_runtime(- Box::new(backend),- CacheRuntime::memory_with_provider(provider),- CacheNamespace::new("tree-unavailable"),- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- assert_eq!(- fs.tree_directory("/docs", false, None, None)- .await- .unwrap()- .len(),- 1- );- assert_eq!(- fs.tree_directory("/docs", false, None, None)- .await- .unwrap()- .len(),- 1- );- assert_eq!(probe.read_dir_count(), 2);- assert!(fs.metrics().snapshot().errors >= 1);-}--#[tokio::test]-async fn cached_tree_traversal_falls_back_for_multiwrite_backend() {- let primary = CountingFileSystem::new();- primary.mkdir("/docs", 0o755).await.unwrap();- primary- .write("/docs/a.md", b"a", 0, WriteFlag::Create)- .await- .unwrap();- let primary_probe = primary.clone();- let multiwrite = MultiWriteWrappedFS::builder(Arc::new(primary))- .build()- .unwrap();- let fs = CachedFileSystem::with_runtime(- Box::new(multiwrite),- CacheRuntime::memory(),- CacheNamespace::new("tree-multiwrite"),- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),- );-- let ctx = Arc::new(FsContextInner::new("acct"));- let entries = FS_CTX- .scope(ctx, async {- fs.tree_directory("/docs", false, None, None).await.unwrap()- })- .await;-- assert_eq!(entries.len(), 1);- assert_eq!(- primary_probe.tree_count(),- 1,- "multi-write backends must keep their backend tree_directory path"- );- assert_eq!(- primary_probe.read_dir_count(),- 0,- "cached traversal must not bypass multi-write tree semantics"- );-}--#[test]-fn cache_policy_supports_explicit_permission_sensitive_prefixes() {- let policy = CachePolicy::default().with_bypass_prefix("/private");-- assert!(!policy.cache_file("/private/secret.md", 10));- assert!(!policy.cache_directory("/private/docs"));- assert!(policy.cache_file("/public/.overview.md", 10));-}--#[test]-fn cache_policy_marks_high_value_objects_as_preferred() {- let policy = CachePolicy::default();-- assert_eq!(- policy.file_decision("/docs/.abstract.md", 128),- CacheDecision::Prefer- );- assert_eq!(- policy.file_decision("/docs/.overview.md", 128),- CacheDecision::Prefer- );- assert_eq!(- policy.file_decision("/docs/note.md", 128),- CacheDecision::Cache- );- assert_eq!(policy.directory_decision("/docs"), CacheDecision::Prefer);-}--#[tokio::test]-async fn full_file_reads_are_read_through_cached_but_range_reads_bypass() {- let backend = CountingFileSystem::new();- backend- .write("/note.md", b"hello world", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- assert_eq!(fs.read("/note.md", 0, 0).await.unwrap(), b"hello world");- assert_eq!(fs.read("/note.md", 0, 0).await.unwrap(), b"hello world");- assert_eq!(probe.read_count(), 1);-- assert_eq!(fs.read("/note.md", 6, 5).await.unwrap(), b"world");- assert_eq!(probe.read_count(), 2);-- let metrics = fs.metrics().snapshot();- assert_eq!(metrics.file_hits, 1);- assert_eq!(metrics.file_misses, 1);- assert_eq!(metrics.backend_fallbacks, 1);-}--#[tokio::test]-async fn read_dir_is_cached_and_parent_changes_invalidate_it() {- let backend = CountingFileSystem::new();- backend.mkdir("/docs", 0o755).await.unwrap();- backend- .write("/docs/one.md", b"one", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- assert_eq!(fs.read_dir("/docs").await.unwrap().len(), 1);- assert_eq!(fs.read_dir("/docs").await.unwrap().len(), 1);- assert_eq!(probe.read_dir_count(), 1);-- fs.write("/docs/two.md", b"two", 0, WriteFlag::Create)- .await- .unwrap();- let entries = fs.read_dir("/docs").await.unwrap();- assert_eq!(entries.len(), 2);- assert_eq!(probe.read_dir_count(), 2);-- let metrics = fs.metrics().snapshot();- assert_eq!(metrics.read_dir_hits, 1);- assert_eq!(metrics.read_dir_misses, 2);- assert!(metrics.invalidations >= 1);-}--#[tokio::test]-async fn oversized_directories_bypass_directory_cache() {- let cacheable = CountingFileSystem::new();- cacheable.mkdir("/small", 0o755).await.unwrap();- for index in 0..4096 {- cacheable- .write(- &format!("/small/{index:04}.txt"),- b"x",- 0,- WriteFlag::Create,- )- .await- .unwrap();- }- let cacheable_probe = cacheable.clone();- let (cacheable_fs, _) = cached_fs(cacheable);-- assert_eq!(cacheable_fs.read_dir("/small").await.unwrap().len(), 4096);- assert_eq!(cacheable_fs.read_dir("/small").await.unwrap().len(), 4096);- assert_eq!(cacheable_probe.read_dir_count(), 1);-- let oversized = CountingFileSystem::new();- oversized.mkdir("/large", 0o755).await.unwrap();- for index in 0..4097 {- oversized- .write(- &format!("/large/{index:04}.txt"),- b"x",- 0,- WriteFlag::Create,- )- .await- .unwrap();- }- let oversized_probe = oversized.clone();- let (oversized_fs, _) = cached_fs(oversized);-- assert_eq!(oversized_fs.read_dir("/large").await.unwrap().len(), 4097);- assert_eq!(oversized_fs.read_dir("/large").await.unwrap().len(), 4097);- assert_eq!(oversized_probe.read_dir_count(), 2);-}--#[tokio::test]-async fn all_directory_membership_mutations_invalidate_parent_entries() {- let backend = CountingFileSystem::new();- backend.mkdir("/root", 0o755).await.unwrap();- backend.mkdir("/root/old", 0o755).await.unwrap();- backend- .write("/root/file.txt", b"file", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);- assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);-- fs.mkdir("/root/new", 0o755).await.unwrap();- assert_eq!(fs.read_dir("/root").await.unwrap().len(), 3);-- fs.rename("/root/old", "/root/moved").await.unwrap();- let names: Vec<String> = fs- .read_dir("/root")- .await- .unwrap()- .into_iter()- .map(|entry| entry.name)- .collect();- assert!(names.contains(&"moved".to_string()));- assert!(!names.contains(&"old".to_string()));-- fs.remove("/root/file.txt").await.unwrap();- assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);-- fs.remove_all("/root/new").await.unwrap();- assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1);- assert_eq!(probe.read_dir_count(), 5);-}--#[tokio::test]-async fn writes_and_deletes_never_leave_stale_file_cache_entries() {- let backend = CountingFileSystem::new();- backend- .write("/value.txt", b"old", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"old");- fs.write("/value.txt", b"new", 0, WriteFlag::Truncate)- .await- .unwrap();- assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"new");-- fs.remove("/value.txt").await.unwrap();- assert!(fs.read("/value.txt", 0, 0).await.is_err());- assert_eq!(probe.read_count(), 2);-}--#[tokio::test]-async fn full_writes_populate_cache_before_returning() {- let backend = CountingFileSystem::new();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- fs.write("/fresh.md", b"fresh", 0, WriteFlag::Create)- .await- .unwrap();- assert_eq!(fs.read("/fresh.md", 0, 0).await.unwrap(), b"fresh");- assert_eq!(probe.read_count(), 0);-}--#[tokio::test]-async fn partial_writes_and_truncate_invalidate_cached_file_contents() {- let backend = CountingFileSystem::new();- backend- .write("/partial.txt", b"hello", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hello");- fs.write("/partial.txt", b"X", 1, WriteFlag::None)- .await- .unwrap();- assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hXllo");-- fs.truncate("/partial.txt", 2).await.unwrap();- assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hX");- assert_eq!(probe.read_count(), 3);-}--#[tokio::test]-async fn file_rename_invalidates_old_and_new_paths() {- let backend = CountingFileSystem::new();- backend- .write("/old.txt", b"old", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/new.txt", b"historical", 0, WriteFlag::Create)- .await- .unwrap();- let direct = backend.clone();- let (fs, _) = cached_fs(backend);-- assert_eq!(fs.read("/old.txt", 0, 0).await.unwrap(), b"old");- assert_eq!(fs.read("/new.txt", 0, 0).await.unwrap(), b"historical");- direct.remove("/new.txt").await.unwrap();-- fs.rename("/old.txt", "/new.txt").await.unwrap();-- assert!(fs.read("/old.txt", 0, 0).await.is_err());- assert_eq!(fs.read("/new.txt", 0, 0).await.unwrap(), b"old");-}--#[tokio::test]-async fn remove_all_generation_rejects_residual_descendant_cache_entries() {- let backend = CountingFileSystem::new();- backend.mkdir("/tree", 0o755).await.unwrap();- backend- .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create)- .await- .unwrap();- let direct = backend.clone();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- assert_eq!(fs.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old");- fs.remove_all("/tree").await.unwrap();-- direct.mkdir("/tree", 0o755).await.unwrap();- direct- .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create)- .await- .unwrap();-- assert_eq!(fs.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"new");- assert_eq!(probe.read_count(), 2);-}--#[tokio::test]-async fn remove_all_error_invalidates_cached_state_after_partial_mutation() {- let backend = CountingFileSystem::new().with_partial_remove_failure("/tree/deleted.txt");- backend.mkdir("/tree", 0o755).await.unwrap();- backend- .write("/tree/deleted.txt", b"deleted", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/tree/survivor.txt", b"survivor", 0, WriteFlag::Create)- .await- .unwrap();- backend- .write("/unrelated.txt", b"unrelated", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- assert_eq!(- fs.read("/tree/deleted.txt", 0, 0).await.unwrap(),- b"deleted"- );- assert_eq!(- fs.read("/tree/survivor.txt", 0, 0).await.unwrap(),- b"survivor"- );- assert_eq!(fs.read("/unrelated.txt", 0, 0).await.unwrap(), b"unrelated");- assert_eq!(fs.read_dir("/tree").await.unwrap().len(), 2);-- let error = fs.remove_all("/tree").await.unwrap_err();- assert!(error.to_string().contains("partially removed"));-- assert!(fs.read("/tree/deleted.txt", 0, 0).await.is_err());- assert_eq!(- fs.read("/tree/survivor.txt", 0, 0).await.unwrap(),- b"survivor"- );- assert_eq!(fs.read("/unrelated.txt", 0, 0).await.unwrap(), b"unrelated");- let entries = fs.read_dir("/tree").await.unwrap();- assert_eq!(entries.len(), 1);- assert_eq!(entries[0].name, "survivor.txt");- assert_eq!(probe.read_count(), 5);- assert_eq!(probe.read_dir_count(), 2);-}--#[tokio::test]-async fn shared_provider_generation_bump_invalidates_other_wrappers() {- let backend = CountingFileSystem::new();- backend.mkdir("/tree", 0o755).await.unwrap();- backend- .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create)- .await- .unwrap();- let direct = backend.clone();- let probe = backend.clone();- let runtime = CacheRuntime::memory();-- let first = CachedFileSystem::with_runtime(- Box::new(backend.clone()),- runtime.clone(),- CacheNamespace::new("shared"),- CachePolicy::default(),- );- let second = CachedFileSystem::with_runtime(- Box::new(backend),- runtime,- CacheNamespace::new("shared"),- CachePolicy::default(),- );-- assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old");- second.remove_all("/tree").await.unwrap();-- direct.mkdir("/tree", 0o755).await.unwrap();- direct- .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create)- .await- .unwrap();-- assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"new");- assert_eq!(probe.read_count(), 2);-}--#[tokio::test]-async fn provider_generation_eviction_after_restart_cannot_revive_old_descendants() {- let backend = CountingFileSystem::new();- backend.mkdir("/tree", 0o755).await.unwrap();- backend- .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create)- .await- .unwrap();- let direct = backend.clone();- let probe = backend.clone();- let provider = Arc::new(MemoryMockProvider::new());- let first_runtime = CacheRuntime::memory_with_provider(provider.clone());-- let first = CachedFileSystem::with_runtime(- Box::new(backend.clone()),- first_runtime.clone(),- CacheNamespace::new("restart"),- CachePolicy::default(),- );- assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old");- first.remove_all("/tree").await.unwrap();-- direct.mkdir("/tree", 0o755).await.unwrap();- direct- .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create)- .await- .unwrap();-- for key in provider.keys().await {- if key.contains(":subtree:") {- first_runtime.del(&[key]).await.unwrap();- }- }- drop(first);- drop(first_runtime);-- let restarted = CachedFileSystem::with_runtime(- Box::new(backend),- CacheRuntime::memory_with_provider(provider),- CacheNamespace::new("restart"),- CachePolicy::default(),- );- assert_eq!(- restarted.read("/tree/leaf.txt", 0, 0).await.unwrap(),- b"new"- );- assert_eq!(probe.read_count(), 2);-}--#[tokio::test]-async fn directory_rename_invalidates_old_and_historical_new_subtrees() {- let backend = CountingFileSystem::new();- backend.mkdir("/old", 0o755).await.unwrap();- backend- .write("/old/leaf.txt", b"moved", 0, WriteFlag::Create)- .await- .unwrap();- let direct = backend.clone();- let (fs, _) = cached_fs(backend);-- assert_eq!(fs.read("/old/leaf.txt", 0, 0).await.unwrap(), b"moved");- fs.rename("/old", "/new").await.unwrap();- assert!(fs.read("/old/leaf.txt", 0, 0).await.is_err());- assert_eq!(fs.read("/new/leaf.txt", 0, 0).await.unwrap(), b"moved");-- fs.remove_all("/new").await.unwrap();- direct.mkdir("/new", 0o755).await.unwrap();- direct- .write("/new/leaf.txt", b"replacement", 0, WriteFlag::Create)- .await- .unwrap();- assert_eq!(- fs.read("/new/leaf.txt", 0, 0).await.unwrap(),- b"replacement"- );-}--#[tokio::test]-async fn concurrent_misses_share_one_backend_read() {- let backend = CountingFileSystem::new().with_read_delay(Duration::from_millis(30));- backend- .write("/hot.md", b"hot", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- let mut tasks = Vec::new();- for _ in 0..12 {- let fs = fs.clone();- tasks.push(tokio::spawn(async move {- fs.read("/hot.md", 0, 0).await.unwrap()- }));- }- for task in tasks {- assert_eq!(task.await.unwrap(), b"hot");- }-- assert_eq!(probe.read_count(), 1);- let metrics = fs.metrics().snapshot();- assert_eq!(metrics.inflight_leaders, 1);- assert_eq!(metrics.inflight_followers, 11);- assert_eq!(metrics.inflight_backend_saved, 11);-}--#[tokio::test]-async fn cache_policy_bypasses_lock_and_control_files() {- let backend = CountingFileSystem::new();- backend- .write("/state.lock", b"first", 0, WriteFlag::Create)- .await- .unwrap();- backend.mkdir("/queue", 0o755).await.unwrap();- backend- .write("/queue/peek", b"live", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let (fs, _) = cached_fs(backend);-- fs.read("/state.lock", 0, 0).await.unwrap();- fs.read("/state.lock", 0, 0).await.unwrap();- fs.read("/queue/peek", 0, 0).await.unwrap();- fs.read("/queue/peek", 0, 0).await.unwrap();-- assert_eq!(probe.read_count(), 4);- assert_eq!(fs.metrics().snapshot().policy_bypasses, 4);-}--#[tokio::test]-async fn failed_invalidation_bypasses_cache_instead_of_serving_stale_data() {- let backend = CountingFileSystem::new();- backend- .write("/value.txt", b"old", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let provider = Arc::new(MemoryMockProvider::new());- provider.set_delete_failure(true);- let fs = CachedFileSystem::with_runtime(- Box::new(backend),- CacheRuntime::memory_with_provider(provider),- CacheNamespace::new("delete-failure"),- CachePolicy::default(),- );-- assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"old");- fs.write("/value.txt", b"new", 0, WriteFlag::Truncate)- .await- .unwrap();- assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"new");- assert_eq!(probe.read_count(), 2);-- let metrics = fs.metrics().snapshot();- assert!(metrics.errors >= 1);- assert!(metrics.policy_bypasses >= 1);-}--#[tokio::test]-async fn unavailable_provider_falls_back_to_backend_and_enters_bypass() {- let backend = CountingFileSystem::new();- backend- .write("/available.txt", b"backend", 0, WriteFlag::Create)- .await- .unwrap();- let probe = backend.clone();- let provider = Arc::new(MemoryMockProvider::new());- provider.set_unavailable(true);- let fs = CachedFileSystem::with_runtime(- Box::new(backend),- CacheRuntime::memory_with_provider(provider),- CacheNamespace::new("unavailable"),- CachePolicy::default(),- );-- assert_eq!(fs.read("/available.txt", 0, 0).await.unwrap(), b"backend");- assert_eq!(fs.read("/available.txt", 0, 0).await.unwrap(), b"backend");- assert_eq!(probe.read_count(), 2);-- let metrics = fs.metrics().snapshot();- assert!(metrics.errors >= 2);- assert!(metrics.policy_bypasses >= 1);-}--#[tokio::test]-async fn generation_backfill_is_bounded_and_continues_after_one_set_failure() {- let backend = CountingFileSystem::new();- let mut current = String::new();- for component in ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l"] {- current.push('/');- current.push_str(component);- backend.mkdir(¤t, 0o755).await.unwrap();- }- let path = "/a/b/c/d/e/f/g/h/i/j/k/l/file.md";- backend- .write(path, b"generation", 0, WriteFlag::Create)- .await- .unwrap();-- let provider = Arc::new(MemoryMockProvider::new().with_set_delay(Duration::from_millis(20)));- provider.fail_next_set_matching(":subtree:");- let fs = CachedFileSystem::with_runtime(- Box::new(backend),- CacheRuntime::memory_with_provider(Arc::clone(&provider)),- CacheNamespace::new("generation-set-failure"),- CachePolicy::default(),- );-- assert_eq!(fs.read(path, 0, 0).await.unwrap(), b"generation");-- let keys = provider.keys().await;- assert_eq!(- keys.iter().filter(|key| key.contains(":subtree:")).count(),- 13,- "one of the fourteen ancestor generation writes should fail"- );- assert_eq!(- keys.iter().filter(|key| key.contains(":file:")).count(),- 1,- "a generation write failure must not prevent the file cache fill"- );- assert_eq!(fs.metrics().snapshot().errors, 1);- assert!(- (2..=8).contains(&provider.max_concurrent_sets()),- "generation writes should run concurrently with a fixed upper bound"- );-}--#[tokio::test]-async fn metrics_cover_operations_bytes_latency_and_errors() {- let backend = CountingFileSystem::new();- backend- .write("/metrics.txt", b"metrics", 0, WriteFlag::Create)- .await- .unwrap();- let (fs, _) = cached_fs(backend);-- assert_eq!(fs.read("/metrics.txt", 0, 0).await.unwrap(), b"metrics");- assert_eq!(fs.read("/metrics.txt", 0, 0).await.unwrap(), b"metrics");- fs.write("/metrics.txt", b"updated", 0, WriteFlag::Truncate)- .await- .unwrap();-- let metrics = fs.metrics().snapshot();- assert_eq!(metrics.file_hits, 1);- assert_eq!(metrics.file_misses, 1);- assert_eq!(metrics.backend_fallbacks, 1);- assert!(metrics.puts >= 1);- assert!(metrics.deletes >= 1);- assert!(metrics.invalidations >= 1);- assert_eq!(metrics.backend_bytes, 7);- assert_eq!(metrics.cache_bytes, 7);- assert!(metrics.get_latency_ns > 0);- assert!(metrics.put_latency_ns > 0);- assert!(metrics.delete_latency_ns > 0);- assert_eq!(metrics.errors, 0);-}diff --git a/docs/en/api/01-overview.md b/docs/en/api/01-overview.mdindex 79c406d96..f4603bf3b 100644--- a/docs/en/api/01-overview.md+++ b/docs/en/api/01-overview.md@@ -359,10 +359,6 @@ This catalog follows the routes actually mounted by the server. Each group headi| GET | `/api/v1/system/status` | System status || POST | `/api/v1/system/wait` | Wait for background processing || POST | `/api/v1/system/consistency` | Check filesystem and vector-index consistency |-| POST | `/api/v1/system/backend/sync-status` | Query backend synchronization status |-| POST | `/api/v1/system/backend/sync-retry` | Retry backend synchronization |-| GET | `/api/v1/system/sync/{sync_path}` | Path-form compatibility endpoint for synchronization status |-| POST | `/api/v1/system/sync/{sync_path}/retry` | Path-form compatibility endpoint for synchronization retry |### [Resources](02-resources.md) and [Filesystem](03-filesystem.md)diff --git a/docs/en/api/07-system.md b/docs/en/api/07-system.mdindex 78a175fcc..d87aefa60 100644--- a/docs/en/api/07-system.md+++ b/docs/en/api/07-system.md@@ -1,6 +1,8 @@# System Status-The OpenViking System API provides health, readiness, consistency, and multi-write backend synchronization status. Component observers and Prometheus metrics are documented separately.+The OpenViking System API provides health, readiness, consistency, and+background-processing status. Component observers and Prometheus metrics are+documented separately.## API Reference@@ -417,99 +419,6 @@ ov system wait --timeout 60----### backend_sync_status()--Return multi-write backend synchronization status for a Viking URI subtree. This endpoint requires ROOT or ADMIN permission.--**HTTP API**--```http-POST /api/v1/system/backend/sync-status-Content-Type: application/json-```--```bash-curl -X POST http://localhost:1933/api/v1/system/backend/sync-status \- -H "Content-Type: application/json" \- -H "X-API-Key: your-admin-key" \- -d '{"uri":"viking://resources"}'-```--The URI-path form is also available:--```http-GET /api/v1/system/sync/{sync_path}-```--**CLI**--```bash-ov system backend sync-status viking://resources-```--**Response example**--```json-{- "status": "ok",- "result": {- "path": "viking://resources",- "entry_count": 12- }-}-```--`result` is supplied by the active filesystem backend. `path` identifies the queried scope and `entry_count` is the number of sync records in that scope. A backend may add diagnostics such as pending or failed records.--### backend_sync_retry()--Retry incomplete multi-write backend synchronization work for a URI subtree. This endpoint requires ROOT or ADMIN permission.--**HTTP API**--```http-POST /api/v1/system/backend/sync-retry-Content-Type: application/json-```--```bash-curl -X POST http://localhost:1933/api/v1/system/backend/sync-retry \- -H "Content-Type: application/json" \- -H "X-API-Key: your-admin-key" \- -d '{"uri":"viking://resources"}'-```--The URI-path form is:--```http-POST /api/v1/system/sync/{sync_path}/retry-```--**CLI**--```bash-ov system backend sync-retry viking://resources-```--**Response example**--```json-{- "status": "ok",- "result": {- "path": "viking://resources",- "retried": 2,- "failed": 0- }-}-```--`retried` is the number of records rescheduled by this request, and `failed` is the number that could not be scheduled. A backend may include additional diagnostic fields.--The public Python, TypeScript, and Go SDKs do not currently expose multi-write backend synchronization methods, so the sections above show only HTTP and CLI tabs.------<a id="reindex"></a><a id="observer-api"></a>## Related Documentationdiff --git a/docs/en/api/20-webdav.md b/docs/en/api/20-webdav.mdindex 161ba6d19..9526ff464 100644--- a/docs/en/api/20-webdav.md+++ b/docs/en/api/20-webdav.md@@ -17,7 +17,7 @@ The adapter supports the following scope:- Resources only. Memories, skills, sessions, and other namespaces are not exposed.- Text-first writes. `PUT` currently accepts UTF-8 text content only.- WebDAV subset only. `OPTIONS`, `PROPFIND`, `GET`, `HEAD`, `PUT`, `DELETE`, `MKCOL`, and `MOVE` are supported.-- Semantic sidecars and internal system files stay internal. Derived or internal files such as `.abstract.md`, `.overview.md`, `.path.ovlock`, `.redirect.json`, and `.sync_log.json` are hidden from listings and cannot be accessed directly through WebDAV.+- Semantic sidecars and internal system files stay internal. Derived or internal files such as `.abstract.md`, `.overview.md`, `.path.ovlock`, and multi-write metadata are hidden from listings and cannot be accessed directly through WebDAV.Behavior notes:@@ -25,7 +25,7 @@ Behavior notes:- Replacing an existing file through WebDAV refreshes related semantics and vectors, same as `write()`.- `PUT` does not create parent collections automatically. Create missing directories with `MKCOL` first.- User-created dot-directories and dot-files remain visible unless they match one of the reserved internal filenames above.-- When multi-write storage is enabled, files redirected to a backup are still exposed through the filesystem APIs as normal files; internal redirect and sync metadata never become visible to callers.+- With multi-write enabled, WebDAV reads use the primary. Backup replication is asynchronous.## API Referencediff --git a/docs/en/concepts/05-storage.md b/docs/en/concepts/05-storage.mdindex ccedc225e..b117329df 100644--- a/docs/en/concepts/05-storage.md+++ b/docs/en/concepts/05-storage.md@@ -54,10 +54,10 @@ AGFS provides POSIX-style file operations with multiple backend support.By default, AGFS uses a single backend for content storage. Once `storage.agfs.backups` is configured, OpenViking enters multi-write mode:-- Top-level `storage.agfs.backend` is the primary backend and remains the authoritative write target.-- `storage.agfs.backups.items[]` defines backup backends for replicas, migration, or read acceleration.+- Top-level `storage.agfs.backend` is the primary backend and the only direct read source.+- `storage.agfs.backups.items[]` defines asynchronously updated backup backends.- The Python SDK, HTTP API, and CLI filesystem interfaces stay unchanged.-- Multi-write uses `.redirect.json` and `.sync_log.json` internally to track redirect mappings and sync progress. These files are not visible to users.+- V2 metadata is partitioned and stored only on the primary.For the conceptual model, see [Multi-Write Storage](./14-multi-write-storage.md). For examples, see the [Multi-Write Storage Guide](../guides/13-multi-write-storage.md).@@ -161,5 +161,5 @@ client.mv(- [Architecture Overview](./01-architecture.md) - System architecture- [Context Layers](./03-context-layers.md) - L0/L1/L2 model- [Viking URI](./04-viking-uri.md) - URI specification-- [Multi-Write Storage](./14-multi-write-storage.md) - Primary/backup roles, routing, and consistency+- [Multi-Write Storage](./14-multi-write-storage.md) - V2 primary and backup consistency- [Retrieval Mechanism](./07-retrieval.md) - Retrieval process detailsdiff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.mdindex b6da7c6bb..224b96a9f 100644--- a/docs/en/concepts/12-metrics.md+++ b/docs/en/concepts/12-metrics.md@@ -289,8 +289,12 @@ seconds, for example `123 ns = 0.000000123 seconds`.| `openviking_ragfs_cache_bytes_total` | Counter | `source` | bytes from backend/cache || `openviking_ragfs_cache_operation_duration_seconds_total` | Counter | `operation` | cumulative get/put/delete duration || `openviking_ragfs_cache_inflight_events_total` | Counter | `event` | leader/follower/backend_saved counts |-| `openviking_ragfs_multiwrite_background_tasks` | Gauge | none | background tasks, including the retry loop |-| `openviking_ragfs_multiwrite_read_routes_total` | Counter | `route` | primary/backup/redirect/miss selections |+| `openviking_ragfs_multiwrite_background_tasks` | Gauge | none | active flush runtime, `0` or `1` per mount |+| `openviking_ragfs_multiwrite_pending_events` | Gauge | none | submitted events that are queued, executing, or awaiting retry |+| `openviking_ragfs_multiwrite_errors_total` | Counter | `worker` | final failures for `flush`, `checkpoint`, `catch_up`, or `gc` |+| `openviking_ragfs_multiwrite_backend_lag_events` | Gauge | `backend, partition` | sum of `tail - synced_seq` across accounts |+| `openviking_ragfs_multiwrite_protocol_version` | Gauge | `version` | mount count for the closed label `v2` |+| `openviking_ragfs_multiwrite_protocol_status` | Gauge | `status` | mount count for closed labels `migrating` and `stable` |These metrics have no `mount` or `account_id` label. Cache and multi-backendfamilies are absent when those capabilities are not enabled. `status` is@@ -298,6 +302,10 @@ families are absent when those capabilities are not enabled. `status` isHistogram buckets cover all outcomes, with finite bounds from 0.0001 to 10 seconds.Python `get_stats()` retains its microsecond fields.+The protocol family always contains both closed-label series. One unreadable+or invalid protocol, account, or segment manifest rejects the+whole scrape snapshot instead of publishing partial zero lag.+### Tasks and Task Tracker| Metric Family | Type | Common Labels | Meaning |diff --git a/docs/en/concepts/14-multi-write-storage.md b/docs/en/concepts/14-multi-write-storage.mdindex b5ffd735b..356cab111 100644--- a/docs/en/concepts/14-multi-write-storage.md+++ b/docs/en/concepts/14-multi-write-storage.md@@ -1,134 +1,51 @@# Multi-Write Storage-Multi-write storage lets OpenViking use one primary storage backend together with multiple backup backends under a unified filesystem abstraction. It supports replicas (including cross-region replicas), configurable read routing, and storage migration; it does not automatically promote a backup to primary.+V2 multi-write keeps one primary backend as the business source of truth and+replicates its files to configured backups.-From the API user's point of view, interfaces such as `read()`, `write()`, `ls()`, and `stat()` do not change. Multi-write logic lives inside RAGFS, so callers do not need to care which underlying backend ultimately stores a file.+## Read And Write Model-## Core Model+- All direct reads use the primary. Backups never serve application reads.+- A successful primary mutation returns immediately.+- Metadata persistence and backup application continue asynchronously.+- Backup lag does not change the result already returned for the primary.+- Multi-write metadata is stored only on the primary.-Multi-write storage consists of one primary backend and multiple backup backends:+This model is eventually consistent. A process crash between primary success+and event persistence can lose that replication event. Rebuild affected+backups from the primary after such a crash.-| Role | Configuration | Description |-| --- | --- | --- |-| primary | `storage.agfs.backend` | Authoritative write target and final read fallback |-| backup | `storage.agfs.backups.items[]` | Receives replicated writes and may optionally participate in reads |+## Metadata Model-If `backups` is not configured, OpenViking continues to use the original single-backend mode.+Each account has a fixed set of metadata partitions. Events are routed to one+partition, appended to a segment log, and consumed independently by each+backup. Checkpoints compact older history, and garbage collection removes+metadata that is no longer required.-## Write Path+`initial_partitions` controls the partition count when an account's metadata is+created. Existing accounts keep their persisted partition count and routes when+the configuration changes. Partition expansion and shrinking are not supported+in this release.-By default, writes land on the primary backend first and are then replicated to write-enabled backup backends.+## First Enable Import-```text-Client- -> OpenViking API- -> RAGFS MultiWrite- -> primary- -> backup1 / backup2 / ...-```+When `.multiwrite.json` is absent, startup creates it as `V2/migrating`,+starts foreground V2 event submission, and imports the current primary files+in the background. Successful import changes the status to `stable`. Extra+old files on backups are not removed.-If a backup does not define `operations`, it participates in writes by default. This keeps cold-backup setup simple.+Old multi-write protocols and old migration fields are rejected. They are not+converted or interpreted.-## Sync Modes+## Operations-Multi-write supports two consistency modes.--| Mode | Config value | Behavior | Suitable for |-| --- | --- | --- | --- |-| Async multi-write | `async` | Return as soon as the primary write succeeds; backups sync in the background | Low-latency writes, eventual consistency |-| Sync multi-write | `sync` | Wait for backup acknowledgements after the primary write succeeds | Stronger write confirmation when extra latency is acceptable |--In async mode, backups may lag behind the primary; recovery depends on backend availability and retry outcomes. In sync mode, `write_ack_count` and `write_ack_timeout_ms` control how many backup acknowledgements are required and how long the system waits.--If sync mode fails to reach the required acknowledgement count, the call returns an error even though the primary may already have committed; that write is not automatically rolled back. Backups that did not acknowledge are still retried in the background; to inspect sync state, see the [Multi-Write Storage Guide](../guides/13-multi-write-storage.md).--## Read Path--Reads do not hit every backup by default. Only backups that explicitly declare the `read` operation join the read route.--Read order:--```text-1. Read-enabled backups in ascending priority order-2. Fallback to the primary backend-3. If the file is redirected, access the redirect target-4. Return NotFound if the file is still missing-```--This avoids having cold-backup nodes participate in reads by default and reduces the risk of serving stale data.--## Redirect--Redirect means "certain files are not written to the primary backend and are written to a specified backup instead."--Common cases:--- Large files go to object storage.-- Specific file extensions go to a dedicated backend.-- The primary backend keeps standard content, while special files live elsewhere.--Redirect policies are configured on the primary backend. When a file matches a policy, OpenViking records the mapping in internal metadata. Calls such as `ls()`, `stat()`, and `read()` still expose a normal filesystem view.--## Exclude--Exclude means "a specific backup backend does not receive matching files."--Common cases:--- A memory or cache backend should not keep large files.-- One backup only stores text resources.-- A lower-cost backend excludes temporary or oversized files.--Exclude policies are configured on each backup and affect only whether that backup receives a write.--## Internal Metadata--Multi-write uses two internal metadata files:--| File | Purpose |-| --- | --- |-| `.redirect.json` | Records which backend stores each redirected file |-| `.sync_log.json` | Records per-file sync version and backup acknowledgement progress |--These files are hidden from normal users, do not appear in standard directory listings, and should not be read or written through public APIs.--If the primary backend enables at-rest encryption, these internal metadata files follow the same encryption policy.--## Encryption Relationship--Multi-write does not change OpenViking's transparent encryption model.--Rules:--- The Python layer and public APIs stay unaware of encryption details.-- The primary backend must be encrypted when global encryption is enabled.-- Each backup may decide independently whether to enable encryption.-- Internal metadata must go through the primary backend's encryption path.--That means enabling multi-write does not change how clients call the system; encryption remains a configuration concern per backend.--## Relationship with OVPack--Multi-write only handles new writes after it is enabled. It does not automatically sync historical files that already exist in the primary backend.--Recommended migration flow:--1. Pause writes or arrange a verifiable incremental catch-up window, then migrate existing data with OVPack or another controlled process.-2. Validate the target backend data.-3. Enable the multi-write configuration.-4. Catch up changes made during migration and verify sync state before resuming writes; multi-write replicates subsequent additions and updates.--## Limitations--- In async mode, backups may lag; reads from them cannot assume the latest write is visible.-- Historical files that existed before multi-write was enabled need a separate migration or backfill process.-- Redirected files rely on internal metadata to reconstruct the directory view.-- Concurrent writes from multiple processes to the same primary backend still need future distributed metadata locking.-- Hot directories may update internal metadata frequently and can introduce additional write amplification.+V2 does not expose synchronization status or manual retry APIs or CLI+commands. Use the multi-write Prometheus metrics to observe queue depth,+worker failures, backup lag, protocol version, and protocol status.## Related Documents- [Storage Architecture](./05-storage.md)+- [Metrics](./12-metrics.md)- [Configuration Guide](../guides/01-configuration.md)- [Multi-Write Storage Guide](../guides/13-multi-write-storage.md)-- [OVPack Import and Export](../guides/09-ovpack.md)diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex 0d47386b3..70ac61e23 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -1300,7 +1300,6 @@ Storage configuration for context data, including file storage (RAGFS) and vecto| `backend` | str | `"local"`, `"s3"`, or `"memory"` | `"local"` || `timeout` | float | Request timeout in seconds | `10.0` || `backups` | object | Multi-write storage configuration. When set, the top-level `backend` acts as the primary backend and `backups.items[]` defines backup backends | `null` |-| `redirects` | array | File redirect policies for multi-write storage. Matching files are written to the specified backup instead of the primary backend | `[]` || `queuefs` | object | QueueFS configuration. Controls the namespace mode, backend, and runtime options for `/queue` | `{ "mode": "shared", "backend": "sqlite", "recover_stale_sec": 0, "busy_timeout_ms": 5000 }` || `queue_db_path` | str (optional) | Legacy compatibility field for QueueFS sqlite DB path. Superseded by `storage.agfs.queuefs.db_path`. Defaults to `{storage.workspace}/_system/queue/queue.db` when not set. Useful when the workspace volume does not support sqlite (e.g. some network filesystems) | `null` || `s3` | object | S3 backend configuration (when backend is 's3') | - |@@ -1314,7 +1313,8 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro##### Multi-Write Storage Configuration-`storage.agfs.backups` enables multi-write storage. If it is not configured, OpenViking stays in single-backend mode.+`storage.agfs.backups` enables asynchronous V2 multi-write storage. The+top-level backend remains the only direct read source.```json{@@ -1322,26 +1322,16 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro"workspace": "./data","agfs": {"backend": "local",- "redirects": [- {- "type": "FileExtensionPolicy",- "extensions": ["(pdf|ppt|zip)"],- "target": ["s3-backup"]- }- ],"backups": {- "sync_type": "async",+ "initial_partitions": 16,+ "checkpoint_interval_secs": 86400,+ "provider": "filesystem","items": [{- "name": "s3-backup",- "backend": "s3",- "s3": {- "bucket": "openviking-backup",- "region": "cn-beijing",- "endpoint": "https://tos-s3-cn-beijing.volces.com",- "access_key": "your-ak",- "secret_key": "your-sk",- "prefix": "multi-write"+ "name": "local-backup",+ "backend": "local",+ "params": {+ "workspace": "./data/backup"}}]@@ -1355,36 +1345,14 @@ Common `backups` fields:| Parameter | Type | Description | Default ||-----------|------|-------------|---------|-| `sync_type` | str | Multi-write sync mode. Supports `"async"` or `"sync"` | `"async"` |-| `write_ack_count` | int | Number of backup acknowledgements required before a `sync` write returns | all backups |-| `write_ack_timeout_ms` | int | Timeout in milliseconds while waiting for backup acknowledgements in `sync` mode | `null` |-| `write_concurrency` | int | Maximum async backup write concurrency | `null` |-| `items` | array | Backup backend list. Each item reuses normal backend configuration and adds fields such as `name`, `operations`, `excludes`, and `encryption` | `[]` |--Common `redirects` fields:--| Parameter | Type | Description | Default |-|-----------|------|-------------|---------|-| `type` | str | Policy type. Supports `"FileExtensionPolicy"` or `"FileOverSizePolicy"` | required |-| `extensions` | array | Extension regex list used by `FileExtensionPolicy`, for example `["(pdf\\|ppt)"]` | `[]` |-| `max_size_mb` | int | File size threshold in MB used by `FileOverSizePolicy` | `null` |-| `target` | array | Backup `name` list that receives matched files | required |--File-size redirect example:--```json-{- "type": "FileOverSizePolicy",- "max_size_mb": 100,- "target": ["s3-backup"]-}-```--Notes:+| `initial_partitions` | int | Initial metadata partitions; changes do not affect existing accounts | `16` |+| `checkpoint_interval_secs` | int | Checkpoint discovery interval; minimum `60` | `86400` |+| `provider` | str | Metadata provider: `filesystem` or top-level CacheRuntime-backed `cache` | `"filesystem"` |+| `items` | array | Backups with `name`, `backend`, `params`, and optional `encryption` | `[]` |-- `redirects` is configured at top-level `storage.agfs` and defines redirect policies for the primary backend.-- `target` must reference an existing backup `name` from `backups.items[]`.-- Files matched by redirect still appear as normal readable and listable files through the filesystem APIs.+The Python configuration layer still accepts legacy sync, retry, operations,+excludes, and redirects fields. It warns and removes them before calling Rust;+the fields have no effect.See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more examples.@@ -1452,9 +1420,9 @@ The top-level `cache` section is a sibling of `storage`. Its public shape is Pro}```-The canonical configuration has no global `cache.enabled`. CacheRuntime is initialized when CacheFS or QueueFS selects `backend=cache`, or when PathLock selects `provider=cache`. Cache-backed PathLock currently requires `cache.provider=redis`; DynamicProvider is not supported for PathLock. When all modules use local providers, `cache.params` is not parsed and no Provider connection is opened.+The canonical configuration has no global `cache.enabled`. CacheRuntime is initialized when CacheFS or QueueFS selects `backend=cache`, PathLock selects `provider=cache`, or multi-write selects `backups.provider=cache`. Cache-backed PathLock currently requires `cache.provider=redis`; DynamicProvider is not supported for PathLock. When all modules use local providers, `cache.params` is not parsed and no Provider connection is opened.-This is a breaking configuration change. `storage.agfs.cache`, `storage.agfs.queuefs.backend="redis"`, and `storage.agfs.queuefs.redis` are rejected. Move Provider settings to top-level `cache.provider/cache.params`, select `cachefs.backend="cache"` or `queuefs.backend="cache"`, use Redis `mode="standalone"` instead of `singleton`, and use `rediss://` instead of `tls_enabled`.+This is a breaking configuration change. `storage.agfs.cache`, `storage.agfs.queuefs.backend="redis"`, `storage.agfs.queuefs.redis`, and `storage.agfs.backups.provider="redis"` are rejected. Move Provider settings to top-level `cache.provider/cache.params`, select `cachefs.backend="cache"`, `queuefs.backend="cache"`, or `backups.provider="cache"`, use Redis `mode="standalone"` instead of `singleton`, and use `rediss://` instead of `tls_enabled`.##### QueueFS Configurationdiff --git a/docs/en/guides/08-encryption.md b/docs/en/guides/08-encryption.mdindex c10ad8926..ea971ce81 100644--- a/docs/en/guides/08-encryption.md+++ b/docs/en/guides/08-encryption.md@@ -21,7 +21,7 @@ Rules:- When global `encryption.enabled=true`, the primary backend must be encrypted.- Each backup backend may control its own encryption through `encryption.enabled`.-- Multi-write internal metadata such as `.redirect.json` and `.sync_log.json` follows the primary backend's encryption policy.+- V2 multi-write metadata is stored on the raw primary and bypasses `EncryptionWrappedFS`.- OpenViking does not expose and does not need public encryption APIs for operating on these internal files.See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more multi-write configuration details.diff --git a/docs/en/guides/09-ovpack.md b/docs/en/guides/09-ovpack.mdindex eab78f7a7..8b117348f 100644--- a/docs/en/guides/09-ovpack.md+++ b/docs/en/guides/09-ovpack.md@@ -38,26 +38,13 @@ control files, and `.relations.json` are outside the OVPack migration scope.## Working with Multi-Write Storage-Multi-write storage only replicates writes that happen after it is enabled. It-does not automatically copy historical files that already existed before-`storage.agfs.backups` was turned on.+V2 startup imports current V1 primary files into partitioned metadata and+removes V1 sidecars. It does not remove extra old files from backups. Pause+all instances and take a complete backup before the first V2 startup.-To seed the primary and replicas together, enable multi-write on the **empty target environment before restoring** the package:--1. Pause application writes and export or back up each source account.-2. Configure the target primary and backup backends, including their write policies, then start the target server and provision its restore identity.-3. Restore or import through that server so historical content passes through the configured write fanout.-4. Check synchronization status for every restored scope and verify the expected files in each replica before switching traffic. Asynchronous replication can still be pending after restore returns.-5. Resume writes after validating content and index integrity. Keep the source and backup until verification is complete.--For example, using the target account's admin key:--```bash-ov system backend sync-status viking://resources-ov system backend sync-status viking://user-```--Restoring first and enabling backups afterwards only replicates subsequent writes; it does not seed historical content into replicas.+Once import starts, in-place rollback is unsupported. Restore the complete+pre-import backup to return to V1. After startup, verify primary content with+normal file APIs and monitor backup lag through Prometheus metrics.For more details, see the [Multi-Write Storage Guide](./13-multi-write-storage.md).diff --git a/docs/en/guides/13-multi-write-storage.md b/docs/en/guides/13-multi-write-storage.mdindex 2c6c8f318..4a30c18ab 100644--- a/docs/en/guides/13-multi-write-storage.md+++ b/docs/en/guides/13-multi-write-storage.md@@ -1,20 +1,10 @@# Multi-Write Storage Guide-This guide explains how to configure multi-write storage in OpenViking. Multi-write storage lets one primary backend replicate writes to multiple backup backends for replica storage (including cross-region replicas), optional read routing, and migration. It does not automatically promote a backup when the primary fails.--Multi-write lives inside RAGFS. The Python SDK, HTTP API, and CLI usage remain unchanged.--## Prerequisites--- You already have a working `ov.conf`.-- The primary backend has been verified to read and write correctly.-- If you plan to use S3-compatible storage, prepare the bucket, endpoint, and access credentials first.-- For existing data, follow the [OVPack multi-write migration workflow](./09-ovpack.md#working-with-multi-write-storage); enabling backups alone does not copy historical files.+V2 multi-write returns after the primary succeeds and updates metadata and+backups asynchronously. The primary is the only direct read source.## Minimal Configuration-Merge these excerpts into your existing model/server configuration. The following example uses a local directory as the primary backend and replicates writes to another local directory.-```json{"storage": {@@ -22,13 +12,15 @@ Merge these excerpts into your existing model/server configuration. The followin"agfs": {"backend": "local","backups": {- "sync_type": "async",+ "initial_partitions": 16,+ "checkpoint_interval_secs": 86400,+ "provider": "filesystem","items": [{"name": "local-backup","backend": "local",- "local": {- "workspace": "./data/backup"+ "params": {+ "workspace": "./data/backup"}}]@@ -38,385 +30,62 @@ Merge these excerpts into your existing model/server configuration. The followin}```-Notes:--- The top-level `backend` is the primary backend.-- `backups.items[]` is the backup backend list.-- `name` is the stable identity of a backup; later sync metadata refers to it.-- A backup with `backend = "local"` uses `local.workspace` to point to its local directory.-- If `sync_type` is omitted, treat it as async by default.--## Configuring Multiple Backups+The three top-level V2 settings are:-You can configure more than one backup. The following example writes to both a local replica and S3-compatible object storage.--```json-{- "storage": {- "workspace": "./data",- "agfs": {- "backend": "local",- "backups": {- "sync_type": "async",- "items": [- {- "name": "local-az2",- "backend": "local",- "local": {- "workspace": "./data/local-az2"- }- },- {- "name": "object-store",- "backend": "s3",- "s3": {- "bucket": "openviking-backup",- "region": "us-east-1",- "endpoint": "https://s3.example.com",- "access_key": "your-access-key",- "secret_key": "your-secret-key",- "prefix": "openviking",- "directory_marker_mode": "none"- }- }- ]- }- }- }-}-```--Recommendations:--- Do not use unstable hostnames or temporary IDs for `name`.-- The backup path or bucket should not point to the same physical location as the primary backend.-- Changing a backup `name` affects historical sync metadata recognition, so treat that as a production change.--### S3-Compatible Storage Notes--Match these `s3` fields to the endpoint and object-store configuration:--| Field | Required? | Description |+| Field | Default | Meaning || --- | --- | --- |-| `use_path_style` | Optional; default `true` | Use path-style URLs (`http://host/bucket/key`); select the style your endpoint supports. |-| `directory_marker_mode` | Yes for backup items | `none` creates no marker, `empty` creates a zero-byte marker, and `nonempty` creates a marker with content. Set it explicitly in each S3 backup item; if omitted, startup fails with `AGFSConfigError: configuration error: invalid directory_marker_mode: null (valid: none, empty, nonempty)`. |-| `use_ssl` | Optional | Set to `false` for HTTP endpoints (e.g. `http://localhost:9000`). |--**Minimal S3-compatible example (RustFS/MinIO):**--```json-{- "name": "s3-backup",- "backend": "s3",- "s3": {- "bucket": "my-bucket",- "endpoint": "http://localhost:9000",- "access_key": "your-access-key",- "secret_key": "your-secret-key",- "prefix": "openviking",- "use_ssl": false,- "use_path_style": true,- "directory_marker_mode": "none"- }-}-```--The example uses `none`, which suits S3-compatible services that do not use directory markers (RustFS, MinIO, Ceph, etc.). Backup items do not get a default for this field. The top-level `storage` S3 configuration defaults to `empty`.--### Docker Networking for S3 Backup--When running OpenViking in Docker with an S3 backup on the same host:--- **Linux Docker**: Use `--network host` or the host's LAN IP. Docker bridge network can reach the host's LAN via gateway IP (e.g. `172.17.0.1:9000`).-- **macOS/Windows Docker Desktop**: Use `host.docker.internal` to reach the host from the container. Host networking is available as an opt-in feature in Docker Desktop 4.34 and later; see [Docker host networking](https://docs.docker.com/engine/network/drivers/host/).--If the server crashes silently on startup with S3 backup enabled, check the Docker networking first. The RAGFS Rust binding will produce `dispatch failure` if the S3 endpoint is unreachable from inside the container.--## Choosing a Sync Mode--### Async Mode--Async mode fits most deployments.--```json-{- "backups": {- "sync_type": "async",- "items": []- }-}-```--Characteristics:--- Returns immediately after the primary write succeeds.-- Backup writes run in the background.-- Low write latency.-- Lag has no fixed bound while a backend is unavailable; monitor synchronization and recovery.--Suitable for:--- Write throughput first.-- Backups mainly used for disaster recovery.-- Eventual consistency is acceptable.--### Sync Mode--Sync mode waits for backup acknowledgements.--```json-{- "backups": {- "sync_type": "sync",- "write_ack_count": 1,- "write_ack_timeout_ms": 5000,- "items": [{"name": "local-backup", "backend": "local", "local": {"workspace": "./backup-data"}}]- }-}-```--Parameters:--| Parameter | Description |-| --- | --- |-| `write_ack_count` | Minimum number of backup acknowledgements required before the write returns |-| `write_ack_timeout_ms` | Timeout in milliseconds while waiting for backup acknowledgements |--Characteristics:--- Stronger write confirmation.-- Write latency depends on backup responsiveness.-- Unconfirmed backups continue to be retried in the background.-- The client may still see an error after the primary write has already succeeded when the required backup acknowledgements are not met.--Suitable for:--- Narrowing the confirmation window between the primary and backups.-- Backup latency is predictable.-- The caller can accept the extra latency of synchronous writes.--## Configuring Read Acceleration--Backups do not participate in reads by default. To let a backup serve reads, explicitly configure `operations`.--```json-{- "name": "cache-backend",- "backend": "memfs",- "operations": [- {"operation": "write", "priority": 0},- {- "operation": "read",- "priority": 10- }- ]-}-```--Read priority rules:--- Lower `priority` values are tried first.-- Only backups with `read` configured join the read route.-- The primary backend always remains the final fallback.-- Cold-backup nodes usually should not be read-enabled.--If a backup defines only `read` but not `write`, it does not receive normal multi-write replication. Use that only when you explicitly control how the backend gets its data.+| `initial_partitions` | `16` | Metadata partitions created for an account |+| `checkpoint_interval_secs` | `86400` | Checkpoint discovery interval; minimum `60` |+| `provider` | `"filesystem"` | Metadata segment provider: `filesystem` or `cache` |-## Redirect Configuration+Each `items[]` entry needs `name`, `backend`, and backend-specific `params`.+`encryption` is optional. Backup names must be unique and cannot be+`primary`.-Redirect sends matching files to a specified backup instead of the primary backend.+When `provider` is `cache`, multi-write reuses the top-level CacheRuntime.+The top-level `cache.provider` selects the implementation; multi-write has no+separate Cache Provider connection.-Redirect by file extension:+The Python configuration layer still accepts legacy V1 sync, acknowledgement,+retry, operation, exclude, and redirect fields. It warns and removes them+before calling Rust; the fields have no effect.-```json-{- "storage": {- "agfs": {- "backend": "local",- "redirects": [- {- "type": "FileExtensionPolicy",- "extensions": ["(pdf|ppt|zip)"],- "target": ["object-store"]- }- ],- "backups": {- "items": [- {- "name": "object-store",- "backend": "s3",- "s3": {- "bucket": "openviking-large-files",- "endpoint": "https://s3.example.com"- }- }- ]- }- }- }-}-```--Redirect by file size:--```json-{- "type": "FileOverSizePolicy",- "max_size_mb": 100,- "target": ["object-store"]-}-```--Notes:--- `target` must reference an existing backup `name`.-- Redirected files still appear as normal readable, listable, and queryable files through the public APIs.-- Redirect mappings are stored in internal metadata on the primary backend.--## Exclude Configuration--Exclude makes one backup skip matching files.--```json-{- "name": "cache-backend",- "backend": "memfs",- "excludes": [- {- "type": "FileOverSizePolicy",- "max_size_mb": 50- },- {- "type": "FileExtensionPolicy",- "extensions": ["(mp4|zip)"]- }- ]-}-```--Common uses:--- Exclude large files from cache backends.-- Exclude file types that do not need to be preserved on a lower-cost backup.-- Keep one backup focused on text or configuration resources only.--If the target backup of a redirect also excludes the same file, the configuration is self-contradictory. Fix the configuration instead of expecting the system to guess another target.--## Encryption Configuration--Multi-write storage reuses OpenViking's transparent at-rest encryption.--Example with global encryption enabled:--```json-{- "encryption": {- "enabled": true,- "provider": "local",- "local": {- "key_file": "~/.openviking/master.key"- }- },- "storage": {- "workspace": "./data",- "agfs": {- "backend": "local",- "backups": {- "items": [- {- "name": "plain-cache",- "backend": "memfs",- "encryption": {- "enabled": false- }- },- {- "name": "encrypted-backup",- "backend": "local",- "local": {- "workspace": "./data/encrypted-backup"- },- "encryption": {- "enabled": true- }- }- ]- }- }- }-}-```--Rules:--- When global `encryption.enabled=true`, the primary backend must be encrypted.-- Each backup may independently control encryption through `encryption.enabled`.-- The Python SDK, HTTP API, and CLI do not need to handle encryption or decryption.-- Internal metadata such as `.redirect.json` and `.sync_log.json` follows the primary backend's encryption policy.+## Runtime Behavior-## Migrating Existing Data--Multi-write only replicates writes that happen after it is enabled. It does not automatically copy historical files.--For OVPack migration, follow [Working with Multi-Write Storage](./09-ovpack.md#working-with-multi-write-storage): configure multi-write on the clean target before restoring through that server, then verify each replica before resuming writes. That guide also covers target account initialization and restore conflict handling.--## Verifying the Configuration--Before startup, it is recommended to run:--```bash-openviking-server doctor-```--After startup, verify with ordinary file APIs:--```bash-ov write viking://resources/multiwrite-check.txt \- --content "multi-write check"--ov read viking://resources/multiwrite-check.txt-```--If you use a local backup, you can also inspect the backup directory directly. Also run `ov system backend sync-status viking://resources` and check every intended replica. A successful ordinary read may come only from the primary and does not prove replication.--## FAQ--### Why is a backup not serving reads?--Backups are write-only by default. To make a backup serve reads, configure:--```json-{- "operations": [- {"operation": "write", "priority": 0},- {- "operation": "read",- "priority": 10- }- ]-}-```+- The primary is the only direct read source and business source of truth.+- Primary success returns without waiting for metadata or backups.+- Multi-write metadata is stored only on the primary.+- Backups converge in the background.+- There is no synchronization status or manual retry API or CLI command.-### Why do historical files not appear in the backup after enabling multi-write?+A crash between primary success and event persistence can lose one replication+event. Rebuild an affected backup from the primary.-Multi-write only handles new writes after it is enabled. For historical data, follow the [OVPack migration workflow](./09-ovpack.md#working-with-multi-write-storage); enabling backups does not backfill it.+## Partition Count-### Can async mode guarantee that the newest data is immediately readable from a backup?+`initial_partitions` applies only when an account's partition manifest is first+created. Existing accounts continue to use their persisted partition count and+routes after configuration changes. Partition expansion and shrinking are not+supported in this release.-No. Background retries need a functioning backend to catch up. A stale but successful backup read does not fall back to the primary; route reads to the primary when current data is required.+## First Enable And Rollback-### Will internal metadata files appear in normal user listings?+When `.multiwrite.json` is absent, the mount starts foreground V2 event+submission and imports current primary files in the background. Extra old+files on backups are not removed.-No. `.redirect.json` and `.sync_log.json` are internal files and are hidden from ordinary directory listings.+Old multi-write protocols and old migration fields are rejected. Take a+complete backup before enabling multi-write. Restore that backup before+rolling back the binary.-### If sync mode returns a failure, does that mean the primary backend definitely did not write the data?+## Verification-No. The primary write may already have succeeded while the required backup acknowledgements were not met. In that case the client can see a failure even though the data already exists on the primary backend, and lagging backups will continue to be repaired in the background.+Use normal file APIs to verify primary reads and writes. Use the metrics in+[Metrics](../concepts/12-metrics.md) to monitor queue depth, worker failures,+backup lag, protocol version, and protocol status.## Related Documents- [Multi-Write Storage](../concepts/14-multi-write-storage.md)-- [Storage Architecture](../concepts/05-storage.md)- [Configuration Guide](./01-configuration.md)-- [Encryption Guide](./08-encryption.md)- [OVPack Import and Export](./09-ovpack.md)diff --git a/docs/scripts/check-api-reference.mjs b/docs/scripts/check-api-reference.mjsindex 6a1a4432e..d83dcb212 100644--- a/docs/scripts/check-api-reference.mjs+++ b/docs/scripts/check-api-reference.mjs@@ -178,11 +178,9 @@ function findRoute(method, documentedPath) {const candidateMethod = candidate.slice(0, separator)const candidatePath = candidate.slice(separator + 1)if (candidateMethod !== method) continue- const placeholderPattern =- candidatePath.startsWith('/webdav/resources/{}') ||- candidatePath.startsWith('/api/v1/system/sync/{}')- ? '.+'- : '[^/]+'+ const placeholderPattern = candidatePath.startsWith('/webdav/resources/{}')+ ? '.+'+ : '[^/]+'const pattern = candidatePath.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\\\{\\\}/g, placeholderPattern)@@ -351,10 +349,7 @@ function routeIsCovered(route, coverage) {.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\\\{\\\}/g,- routePath.startsWith('/webdav/resources/{}') ||- routePath.startsWith('/api/v1/system/sync/{}')- ? '.+'- : '[^/]+'+ routePath.startsWith('/webdav/resources/{}') ? '.+' : '[^/]+')for (const reference of coverage) {const referenceSeparator = reference.indexOf(' ')diff --git a/docs/zh/api/01-overview.md b/docs/zh/api/01-overview.mdindex 39d6406b7..8e278afd1 100644--- a/docs/zh/api/01-overview.md+++ b/docs/zh/api/01-overview.md@@ -354,10 +354,6 @@ JSON 输出 - 错误:| GET | `/api/v1/system/status` | 系统状态 || POST | `/api/v1/system/wait` | 等待后台处理完成 || POST | `/api/v1/system/consistency` | 文件系统与向量索引一致性检查 |-| POST | `/api/v1/system/backend/sync-status` | 查询后端同步状态 |-| POST | `/api/v1/system/backend/sync-retry` | 重试后端同步 |-| GET | `/api/v1/system/sync/{sync_path}` | 路径形式的同步状态兼容接口 |-| POST | `/api/v1/system/sync/{sync_path}/retry` | 路径形式的同步重试兼容接口 |### [资源](02-resources.md)与[文件系统](03-filesystem.md)diff --git a/docs/zh/api/07-system.md b/docs/zh/api/07-system.mdindex 9281b3fb1..420fac892 100644--- a/docs/zh/api/07-system.md+++ b/docs/zh/api/07-system.md@@ -1,6 +1,7 @@# 系统状态-OpenViking 系统 API 提供健康检查、就绪检查、一致性检查和主备后端同步状态。组件级观测和 Prometheus 指标分别提供独立文档。+OpenViking 系统 API 提供健康检查、就绪检查、一致性检查和后台处理状态。+组件级观测和 Prometheus 指标分别提供独立文档。## API 参考@@ -409,99 +410,6 @@ ov system wait --timeout 60----### backend_sync_status()--查询指定 Viking URI 子树在主备存储后端之间的同步状态。该接口要求 ROOT 或 ADMIN 权限。--**HTTP API**--```http-POST /api/v1/system/backend/sync-status-Content-Type: application/json-```--```bash-curl -X POST http://localhost:1933/api/v1/system/backend/sync-status \- -H "Content-Type: application/json" \- -H "X-API-Key: your-admin-key" \- -d '{"uri":"viking://resources"}'-```--也可以使用 URI 路径形式:--```http-GET /api/v1/system/sync/{sync_path}-```--**CLI**--```bash-ov system backend sync-status viking://resources-```--**响应示例**--```json-{- "status": "ok",- "result": {- "path": "viking://resources",- "entry_count": 12- }-}-```--`result` 由当前文件系统后端返回;`path` 标识查询范围,`entry_count` 表示该范围内的同步记录数。具体后端可能附加待同步、失败记录等诊断字段。--### backend_sync_retry()--重试指定 URI 子树中尚未完成的主备后端同步工作。该接口要求 ROOT 或 ADMIN 权限。--**HTTP API**--```http-POST /api/v1/system/backend/sync-retry-Content-Type: application/json-```--```bash-curl -X POST http://localhost:1933/api/v1/system/backend/sync-retry \- -H "Content-Type: application/json" \- -H "X-API-Key: your-admin-key" \- -d '{"uri":"viking://resources"}'-```--URI 路径形式为:--```http-POST /api/v1/system/sync/{sync_path}/retry-```--**CLI**--```bash-ov system backend sync-retry viking://resources-```--**响应示例**--```json-{- "status": "ok",- "result": {- "path": "viking://resources",- "retried": 2,- "failed": 0- }-}-```--`retried` 是本次重新调度的记录数,`failed` 是重试调度失败的记录数;具体后端可能附加额外诊断字段。--公共 Python、TypeScript 和 Go SDK 当前没有主备后端同步方法,因此以上小节只展示 HTTP 和 CLI Tab。------<a id="reindex"></a><a id="observer-api"></a>## 相关文档diff --git a/docs/zh/api/20-webdav.md b/docs/zh/api/20-webdav.mdindex 215d1b65c..d6cf424ba 100644--- a/docs/zh/api/20-webdav.md+++ b/docs/zh/api/20-webdav.md@@ -17,7 +17,7 @@ OpenViking Server 也提供了一个面向资源文件的精简 WebDAV 适配层- 仅开放 `resources` 命名空间,不暴露 memories、skills、sessions 等其他空间。- 以文本写入为主,当前 `PUT` 只接受 UTF-8 文本内容。- 只实现一小部分 WebDAV 方法:`OPTIONS`、`PROPFIND`、`GET`、`HEAD`、`PUT`、`DELETE`、`MKCOL`、`MOVE`。-- 语义侧边文件和系统内部文件保持内部可见。`.abstract.md`、`.overview.md`、`.path.ovlock`、`.redirect.json`、`.sync_log.json` 这些派生或内部文件不会出现在 WebDAV 列表中,也不能被直接访问。+- 语义侧边文件和系统内部文件保持内部可见。`.abstract.md`、`.overview.md`、`.path.ovlock` 和多写 metadata 不会出现在 WebDAV 列表中,也不能被直接访问。行为说明:@@ -25,7 +25,7 @@ OpenViking Server 也提供了一个面向资源文件的精简 WebDAV 适配层- 通过 WebDAV 覆盖已有文件时,会像 `write()` 一样刷新相关语义和向量。- `PUT` 不会自动创建父目录。缺失的目录需要先用 `MKCOL` 创建。- 用户自己创建的点目录或点文件仍然可见,只有上面列出的保留内部文件名会被隐藏。-- 启用主备存储时,被 redirect 到 backup 的文件仍会通过文件系统 API 呈现为普通文件;内部 redirect 和同步元数据不会暴露给调用方。+- 启用多写时,WebDAV 读取只访问 primary,backup 在后台异步复制。## API 参考diff --git a/docs/zh/concepts/05-storage.md b/docs/zh/concepts/05-storage.mdindex 710c964e1..1ab4eeae4 100644--- a/docs/zh/concepts/05-storage.md+++ b/docs/zh/concepts/05-storage.md@@ -54,10 +54,10 @@ AGFS 提供 POSIX 风格的文件操作,支持多种后端。默认情况下,AGFS 使用一个后端作为内容存储。配置 `storage.agfs.backups` 后,OpenViking 会启用主备模式:-- 顶层 `storage.agfs.backend` 是 primary,作为权威写入目标。-- `storage.agfs.backups.items[]` 是 backup,用于副本、迁移或读加速。+- 顶层 `storage.agfs.backend` 是 primary,也是唯一直接读取来源。+- `storage.agfs.backups.items[]` 是后台异步更新的 backup。- Python SDK、HTTP API 和 CLI 的文件系统接口保持不变。-- 主备存储内部使用 `.redirect.json` 和 `.sync_log.json` 维护 redirect 映射与同步进度,这些文件对用户不可见。+- V2 metadata 分区存储,并且只存储在 primary。更多概念说明见 [主备存储](./14-multi-write-storage.md),配置示例见 [主备存储指南](../guides/13-multi-write-storage.md)。@@ -158,5 +158,5 @@ client.mv(- [架构概述](./01-architecture.md) - 系统整体架构- [上下文层级](./03-context-layers.md) - L0/L1/L2 模型- [Viking URI](./04-viking-uri.md) - URI 规范-- [主备存储](./14-multi-write-storage.md) - primary/backup、主备路由与一致性+- [多写存储](./14-multi-write-storage.md) - V2 primary 与 backup 一致性- [检索机制](./07-retrieval.md) - 检索流程详解diff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.mdindex 0ccd621fa..29c021a12 100644--- a/docs/zh/concepts/12-metrics.md+++ b/docs/zh/concepts/12-metrics.md@@ -288,8 +288,12 @@ RAGFS 通过一次原生 `metrics()` 调用读取文件系统、Cache、multi-ba| `openviking_ragfs_cache_bytes_total` | Counter | `source` | 来自后端和缓存的字节数 || `openviking_ragfs_cache_operation_duration_seconds_total` | Counter | `operation` | get/put/delete 累计耗时 || `openviking_ragfs_cache_inflight_events_total` | Counter | `event` | leader/follower/backend_saved 次数 |-| `openviking_ragfs_multiwrite_background_tasks` | Gauge | 无 | 后台任务数,包含重试循环 |-| `openviking_ragfs_multiwrite_read_routes_total` | Counter | `route` | primary/backup/redirect/miss 路由选择次数 |+| `openviking_ragfs_multiwrite_background_tasks` | Gauge | 无 | 每个 mount 的活跃 flush runtime,值为 `0` 或 `1` |+| `openviking_ragfs_multiwrite_pending_events` | Gauge | 无 | 已提交且正在排队、执行中或重试中的事件数 |+| `openviking_ragfs_multiwrite_errors_total` | Counter | `worker` | `flush`、`checkpoint`、`catch_up`、`gc` 的最终失败次数 |+| `openviking_ragfs_multiwrite_backend_lag_events` | Gauge | `backend, partition` | 所有 account 的 `tail - synced_seq` 之和 |+| `openviking_ragfs_multiwrite_protocol_version` | Gauge | `version` | 闭集标签 `v2` 对应的 mount 数 |+| `openviking_ragfs_multiwrite_protocol_status` | Gauge | `status` | 闭集标签 `migrating`、`stable` 对应的 mount 数 |这些指标不带 `mount` 和 `account_id` 标签。未启用 Cache 或 multi-backend时,不生成对应指标族。`status` 为 `success` 或 `error`。@@ -297,6 +301,9 @@ RAGFS 通过一次原生 `metrics()` 调用读取文件系统、Cache、multi-baHistogram 包含全部操作结果,有限桶边界范围为 0.0001 至 10 秒。Python `get_stats()` 保留原有微秒字段。+protocol 指标始终包含闭集中的两条 series。任何 protocol、account 或 segment+manifest 无法读取或校验时,本次完整 snapshot 失败,不会发布部分 lag=0。+### 任务与 Task Tracker| 指标族 | 类型 | 常见标签 | 含义 |diff --git a/docs/zh/concepts/14-multi-write-storage.md b/docs/zh/concepts/14-multi-write-storage.mdindex 5464b5bb1..d7d0dbdc3 100644--- a/docs/zh/concepts/14-multi-write-storage.md+++ b/docs/zh/concepts/14-multi-write-storage.md@@ -1,134 +1,45 @@# 主备存储-主备存储(multi-write)让 OpenViking 在一个统一的文件系统抽象下,同时使用一个主存储和多个备份存储。它可用于保存副本(包括跨地域副本)、配置读路由和迁移存储;不会自动把 backup 提升为 primary。+V2 多写以一个 primary backend 作为业务真相,并把文件复制到配置的 backup。-从 API 使用者视角看,`read()`、`write()`、`ls()`、`stat()` 等接口不变。主备存储逻辑位于 RAGFS 内部,调用方不需要关心文件最终落在哪个底层后端。+## 读写模型-## 核心模型+- 所有直接读取只访问 primary。backup 不参与业务读取。+- primary 变更成功后立即返回。+- metadata 持久化和 backup 应用在后台异步执行。+- backup 落后不会改变 primary 已返回的结果。+- 多写 metadata 只存储在 primary。-主备存储由一个 primary 和多个 backup 组成:+该模型是最终一致。进程可能在 primary 成功和事件持久化之间崩溃,导致该次+复制事件丢失。发生此情况后,需要从 primary 重建受影响的 backup。-| 角色 | 配置位置 | 说明 |-| --- | --- | --- |-| primary | `storage.agfs.backend` | 权威写入目标,也是读取兜底 |-| backup | `storage.agfs.backups.items[]` | 接收复制写入,可选参与读取 |+## Metadata 模型-没有配置 `backups` 时,OpenViking 继续使用原有单后端模式。+每个 account 使用固定数量的 metadata partition。事件路由到一个 partition,+追加到 segment log,再由每个 backup 独立消费。Checkpoint 压缩旧历史,GC+清理不再需要的 metadata。-## 写入路径+`initial_partitions` 仅在 account metadata 首次创建时决定 partition 数量。+已有 account 始终使用持久化的 partition 数量和 routes,修改配置不会改变+现有布局。本期不支持扩容或缩容。-默认情况下,写入先落到 primary,再复制到 write-enabled backup。+## 首次启用导入-```text-Client- -> OpenViking API- -> RAGFS MultiWrite- -> primary- -> backup1 / backup2 / ...-```+`.multiwrite.json` 不存在时,启动流程创建 `V2/migrating` 状态,先启用前台+V2 事件提交,再在后台导入 primary 当前文件。导入成功后切换为 `stable`。+导入不会删除 backup 中多余的旧文件。-backup 未配置 `operations` 时默认参与写入。这样可以用最少配置得到冷备能力。+旧多写协议和旧迁移字段直接拒绝,不执行转换或解释。-## 同步模式+## 运维-主备存储支持两种一致性模式。--| 模式 | 配置值 | 行为 | 适用场景 |-| --- | --- | --- | --- |-| 异步复制 | `async` | primary 写成功后立即返回,backup 后台同步 | 低延迟写入、最终一致 |-| 同步复制 | `sync` | primary 写成功后等待 backup 确认 | 更强写入确认、可接受额外延迟 |--异步模式下,backup 可能落后于 primary,恢复时间取决于后端可用性和重试结果。同步模式下,可以通过 `write_ack_count` 和 `write_ack_timeout_ms` 控制需要等待多少 backup 确认,以及等待多久。--同步模式未达到所需确认数时会返回错误,但 primary 可能已经写入成功,不会因此自动回滚。未确认的 backup 仍会在后台重试;同步状态的检查方法见[主备存储指南](../guides/13-multi-write-storage.md)。--## 读取路径--读取不会默认访问所有 backup。只有显式声明 `read` 操作的 backup 才会进入读路由。--读取顺序如下:--```text-1. 按 priority 升序访问 read-enabled backup-2. 回退到 primary-3. 如果文件被 redirect,则访问 redirect target-4. 仍未命中则返回 NotFound-```--这种设计避免冷备节点默认参与读取,降低读到旧数据的风险。--## Redirect--Redirect 表示“某些文件不写入 primary,而是写入指定 backup”。--常见用途:--- 大文件进入对象存储。-- 特定后缀文件进入专门 backend。-- 主存储只保存常规内容,特殊文件由其他 backend 保存。--Redirect 策略配置在 primary 上。命中策略后,OpenViking 会把映射记录到内部元数据中。用户执行 `ls()`、`stat()`、`read()` 时仍能看到正常的文件系统视图。--## Exclude--Exclude 表示“某个 backup 不接收匹配的文件”。--常见用途:--- 内存或缓存 backend 不保存大文件。-- 某个 backup 只保存文本类资源。-- 某个低成本 backend 排除临时或超大文件。--Exclude 策略配置在 backup 上,只影响该 backup 是否接收写入。--## 内部元数据--主备存储使用两个内部元数据文件:--| 文件 | 作用 |-| --- | --- |-| `.redirect.json` | 记录 redirect 文件对应的目标 backend |-| `.sync_log.json` | 记录每个文件的同步版本和 backup 确认进度 |--这些文件对普通用户不可见,不会出现在常规列表结果中,也不应通过公开 API 直接读写。--如果 primary 开启静态数据加密,这些内部元数据也会跟随 primary 加密策略写入。--## 加密关系--主备存储不会改变 OpenViking 的透明加密模型。--规则如下:--- Python 层和公共 API 不感知加密实现。-- primary 在全局加密开启时必须加密。-- backup 可以独立决定是否加密。-- 内部元数据必须走 primary 的加密入口。--这意味着启用主备存储后,调用方式仍然不变;只需要通过配置决定每个 backend 的加密策略。--## 与 OVPack 的关系--主备存储只负责“启用之后的新写入”。它不会自动同步启用之前已经存在于 primary 中的历史文件。--如果要迁移存量数据,推荐流程是:--1. 暂停写入或安排可核对的增量补齐窗口,再使用 OVPack 或其他受控方式迁移存量。-2. 校验目标 backend 数据。-3. 启用主备存储配置。-4. 补齐迁移期间的变更并验证同步状态后恢复写入;后续新增和修改由主备存储复制。--## 限制--- 异步模式下 backup 可能落后;读取 backup 时不能假设已经看到最新写入。-- 启用主备存储前的历史文件需要单独迁移或回填。-- redirect 文件依赖内部元数据恢复目录视图。-- 多进程同时写同一 primary 时,需要未来的分布式元数据锁能力。-- 热点目录会频繁更新内部元数据,可能带来额外写放大。+V2 不提供同步状态或手工重试 API 和 CLI 命令。请使用多写 Prometheus 指标+观察队列深度、worker 失败、backup lag、协议版本和协议状态。## 相关文档- [存储架构](./05-storage.md)+- [指标](./12-metrics.md)- [配置指南](../guides/01-configuration.md)-- [主备存储指南](../guides/13-multi-write-storage.md)+- [多写存储指南](../guides/13-multi-write-storage.md)- [OVPack 导入导出](../guides/09-ovpack.md)diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex 12dd465be..66df58985 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -1290,8 +1290,7 @@ Glob 引擎配置,用于路径模式匹配。这些设置为服务端配置,|------|------|------|--------|| `backend` | str | `"local"`、`"s3"` 或 `"memory"` | `"local"` || `timeout` | float | 请求超时时间(秒) | `10.0` |-| `backups` | object | 主备存储配置。配置后顶层 `backend` 作为 primary,`backups.items[]` 作为 backup | `null` |-| `redirects` | array | 主备存储的文件重定向策略。命中后文件写入指定 backup,而不是 primary | `[]` |+| `backups` | object | 多写存储配置。配置后顶层 `backend` 作为 primary,`backups.items[]` 作为 backup | `null` || `queuefs` | object | QueueFS 配置。控制 `/queue` 的命名空间模式、后端和运行时参数 | `{ "mode": "shared", "backend": "sqlite", "recover_stale_sec": 0, "busy_timeout_ms": 5000 }` || `queue_db_path` | str(可选)| 旧版兼容字段,用于覆盖 QueueFS 的 sqlite 数据库文件路径。已被 `storage.agfs.queuefs.db_path` 取代。未设置时默认为 `{storage.workspace}/_system/queue/queue.db`。适用于 workspace 卷不支持 sqlite 的场景(例如某些网络文件系统) | `null` || `s3` | object | S3 后端配置(`backend=s3` 时使用) | - |@@ -1306,7 +1305,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系##### 主备存储配置-`storage.agfs.backups` 用于启用主备存储。未配置时,OpenViking 保持单 backend 模式。+`storage.agfs.backups` 启用异步 V2 多写。顶层 backend 始终是唯一直接读取来源。```json{@@ -1314,26 +1313,16 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系"workspace": "./data","agfs": {"backend": "local",- "redirects": [- {- "type": "FileExtensionPolicy",- "extensions": ["(pdf|ppt|zip)"],- "target": ["s3-backup"]- }- ],"backups": {- "sync_type": "async",+ "initial_partitions": 16,+ "checkpoint_interval_secs": 86400,+ "provider": "filesystem","items": [{- "name": "s3-backup",- "backend": "s3",- "s3": {- "bucket": "openviking-backup",- "region": "cn-beijing",- "endpoint": "https://tos-s3-cn-beijing.volces.com",- "access_key": "your-ak",- "secret_key": "your-sk",- "prefix": "multi-write"+ "name": "local-backup",+ "backend": "local",+ "params": {+ "workspace": "./data/backup"}}]@@ -1347,36 +1336,13 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系| 参数 | 类型 | 说明 | 默认值 ||------|------|------|--------|-| `sync_type` | str | 主备同步模式,支持 `"async"` 或 `"sync"` | `"async"` |-| `write_ack_count` | int | `sync` 模式下返回前需要的 backup 确认数 | 全部 backup |-| `write_ack_timeout_ms` | int | `sync` 模式下等待 backup 确认的超时时间,单位毫秒 | `null` |-| `write_concurrency` | int | 异步 backup 写入并发上限 | `null` |-| `items` | array | backup backend 列表,每个 item 复用普通 backend 配置并增加 `name`、`operations`、`excludes`、`encryption` 等字段 | `[]` |--`redirects` 常用字段:--| 参数 | 类型 | 说明 | 默认值 |-|------|------|------|--------|-| `type` | str | 策略类型,支持 `"FileExtensionPolicy"` 或 `"FileOverSizePolicy"` | 必填 |-| `extensions` | array | `FileExtensionPolicy` 使用的扩展名正则列表,例如 `["(pdf\|ppt)"]` | `[]` |-| `max_size_mb` | int | `FileOverSizePolicy` 使用的文件大小阈值,单位 MB | `null` |-| `target` | array | 命中策略后写入的 backup `name` 列表 | 必填 |--按文件大小重定向示例:--```json-{- "type": "FileOverSizePolicy",- "max_size_mb": 100,- "target": ["s3-backup"]-}-```--注意:+| `initial_partitions` | int | account 首次初始化的 metadata partition 数;修改不影响已有 account | `16` |+| `checkpoint_interval_secs` | int | Checkpoint 扫描间隔,最小值为 `60` | `86400` |+| `provider` | str | Metadata provider:`filesystem` 或复用顶层 CacheRuntime 的 `cache` | `"filesystem"` |+| `items` | array | backup 列表,包含 `name`、`backend`、`params` 和可选 `encryption` | `[]` |-- `redirects` 配置在顶层 `storage.agfs`,表示 primary 的重定向策略。-- `target` 必须引用 `backups.items[]` 中已经定义的 backup `name`。-- 命中 redirect 的文件仍会通过普通文件系统 API 呈现为可读、可列举的文件。+Python 配置层仍接受旧 sync、retry、operations、excludes 和 redirects+字段,输出 warning 后会在调用 Rust 前删除,字段不生效。更多配置示例见 [主备存储指南](./13-multi-write-storage.md)。@@ -1444,9 +1410,9 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系}```-标准配置没有全局 `cache.enabled`。当 CacheFS 或 QueueFS 选择 `backend=cache`,或 PathLock 选择 `provider=cache` 时初始化 CacheRuntime。Cache PathLock 当前只支持 `cache.provider=redis`,不支持 DynamicProvider。全部模块使用本地 Provider 时不解析 `cache.params`,也不连接 Provider。+标准配置没有全局 `cache.enabled`。当 CacheFS 或 QueueFS 选择 `backend=cache`、PathLock 选择 `provider=cache`,或多写选择 `backups.provider=cache` 时初始化 CacheRuntime。Cache PathLock 当前只支持 `cache.provider=redis`,不支持 DynamicProvider。全部模块使用本地 Provider 时不解析 `cache.params`,也不连接 Provider。-这是一次配置破坏性变更:`storage.agfs.cache`、`storage.agfs.queuefs.backend="redis"` 和 `storage.agfs.queuefs.redis` 已删除并会被拒绝。请把 Provider 参数迁移到顶层 `cache.provider/cache.params`,业务模块改为 `cachefs.backend="cache"` 或 `queuefs.backend="cache"`;Redis 的 `singleton` 改为 `standalone`,`tls_enabled` 改为使用 `rediss://` endpoint。+这是一次配置破坏性变更:`storage.agfs.cache`、`storage.agfs.queuefs.backend="redis"`、`storage.agfs.queuefs.redis` 和 `storage.agfs.backups.provider="redis"` 已删除并会被拒绝。请把 Provider 参数迁移到顶层 `cache.provider/cache.params`,业务模块改为 `cachefs.backend="cache"`、`queuefs.backend="cache"` 或 `backups.provider="cache"`;Redis 的 `singleton` 改为 `standalone`,`tls_enabled` 改为使用 `rediss://` endpoint。##### QueueFS 配置diff --git a/docs/zh/guides/08-encryption.md b/docs/zh/guides/08-encryption.mdindex 674571b08..4c1efffc2 100644--- a/docs/zh/guides/08-encryption.md+++ b/docs/zh/guides/08-encryption.md@@ -21,7 +21,7 @@ OpenViking 可以按账户加密存储中新写入的文件。加解密由存储- 全局 `encryption.enabled=true` 时,primary backend 必须加密。- backup backend 可以通过自己的 `encryption.enabled` 控制是否加密。-- `.redirect.json` 和 `.sync_log.json` 等主备存储内部元数据跟随 primary 加密策略。+- V2 多写 metadata 存储在 raw primary,不经过 `EncryptionWrappedFS`。- OpenViking 不提供也不需要公开的加解密 API 来操作这些内部文件。更多主备存储配置见 [主备存储指南](./13-multi-write-storage.md)。diff --git a/docs/zh/guides/09-ovpack.md b/docs/zh/guides/09-ovpack.mdindex a611267db..9533748a8 100644--- a/docs/zh/guides/09-ovpack.md+++ b/docs/zh/guides/09-ovpack.md@@ -33,22 +33,12 @@ Session 通过 user 命名空间一起迁移,路径为## 与主备存储配合-主备存储只复制启用之后的新写入,不会自动同步启用之前已经存在的历史文件。如需同时为 primary 和副本写入存量,应在**空目标环境恢复之前**启用主备存储:+V2 启动会把当前 V1 primary 文件导入分区 metadata,并删除 V1 sidecar。+导入不会删除 backup 中多余的旧文件。首次 V2 启动前,应停服所有实例并创建+完整备份。-1. 暂停业务写入,按源 account 分别导出或备份。-2. 配置目标 primary、backup backend 及其写策略,启动目标服务并创建恢复身份。-3. 通过该服务恢复或导入,使存量内容经过已配置的主备复制分发。-4. 对每个恢复的 scope 检查同步状态,并在切流前验证各副本中的文件。恢复返回时,异步复制仍可能未完成。-5. 验证内容和索引完整性后再恢复业务写入;验证完成前保留源数据和备份。--例如,使用目标 account 的 admin key 检查:--```bash-ov system backend sync-status viking://resources-ov system backend sync-status viking://user-```--先恢复再启用 backups 只会复制后续写入,不能把历史内容补到副本。+导入开始后不支持原地回滚。要恢复 V1,必须恢复导入前的完整备份。启动后使用+普通文件 API 验证 primary 内容,并通过 Prometheus 指标观察 backup lag。更多说明见 [主备存储指南](./13-multi-write-storage.md)。diff --git a/docs/zh/guides/13-multi-write-storage.md b/docs/zh/guides/13-multi-write-storage.mdindex 14d5e2dc1..3211082db 100644--- a/docs/zh/guides/13-multi-write-storage.md+++ b/docs/zh/guides/13-multi-write-storage.md@@ -1,20 +1,10 @@-# 主备存储指南+# 多写存储指南-本指南介绍如何配置 OpenViking 的主备存储能力。主备存储允许一个 primary 后端同时复制写入多个 backup 后端,用于保存副本(包括跨地域副本)、配置读路由和存储迁移。primary 故障时不会自动提升 backup。--主备存储能力位于 RAGFS 内部。OpenViking 的 Python SDK、HTTP API 和 CLI 使用方式保持不变。--## 前置条件--- 已有可用的 `ov.conf`。-- 已确认 primary backend 可以正常读写。-- 如果要接入 S3 兼容存储,已准备好 bucket、endpoint 和访问凭据。-- 如需迁移已有数据,请遵循 [OVPack 主备存储迁移流程](./09-ovpack.md#与主备存储配合);仅启用 backups 不会复制历史文件。+V2 多写在 primary 成功后返回,metadata 和 backup 在后台异步更新。primary+是唯一直接读取来源。## 最小配置-将下列片段合并到现有模型和服务端配置。下面示例使用本地目录作为 primary,并把写入复制到另一个本地目录。-```json{"storage": {@@ -22,13 +12,15 @@"agfs": {"backend": "local","backups": {- "sync_type": "async",+ "initial_partitions": 16,+ "checkpoint_interval_secs": 86400,+ "provider": "filesystem","items": [{"name": "local-backup","backend": "local",- "local": {- "workspace": "./data/backup"+ "params": {+ "workspace": "./data/backup"}}]@@ -38,385 +30,55 @@}```-说明:--- 顶层 `backend` 是 primary。-- `backups.items[]` 是 backup 列表。-- `name` 是 backup 的稳定身份,后续同步元数据会引用它。-- `backend = "local"` 的 backup 使用 `local.workspace` 指定本地目录。-- `sync_type` 不配置时默认按异步模式理解。--## 多 Backup 配置+V2 的三个顶层配置是:-可以配置多个 backup。下面示例同时写入本地副本和 S3 兼容对象存储。--```json-{- "storage": {- "workspace": "./data",- "agfs": {- "backend": "local",- "backups": {- "sync_type": "async",- "items": [- {- "name": "local-az2",- "backend": "local",- "local": {- "workspace": "./data/local-az2"- }- },- {- "name": "object-store",- "backend": "s3",- "s3": {- "bucket": "openviking-backup",- "region": "us-east-1",- "endpoint": "https://s3.example.com",- "access_key": "your-access-key",- "secret_key": "your-secret-key",- "prefix": "openviking",- "directory_marker_mode": "none"- }- }- ]- }- }- }-}-```--建议:--- `name` 不要使用会频繁变化的机器名或临时编号。-- backup 的底层路径或 bucket 应避免与 primary 指向同一物理位置。-- 修改 backup `name` 会影响历史同步元数据的识别,生产环境应谨慎变更。--### S3 兼容存储注意事项--按实际 endpoint 和对象存储配置选择以下 `s3` 字段:--| 字段 | 是否必填 | 说明 |+| 字段 | 默认值 | 含义 || --- | --- | --- |-| `use_path_style` | 可选,默认 `true` | 使用路径风格 URL(`http://host/bucket/key`),按 endpoint 支持的方式选择。 |-| `directory_marker_mode` | backup 项必填 | `none` 不创建标记,`empty` 创建零字节标记,`nonempty` 创建带内容的标记。每个 S3 backup 项都要显式设置;省略时启动会报 `AGFSConfigError: configuration error: invalid directory_marker_mode: null (valid: none, empty, nonempty)`。 |-| `use_ssl` | 可选 | HTTP 端点(如 `http://localhost:9000`)需要设置为 `false`。 |--**S3 兼容存储最小示例(RustFS/MinIO):**--```json-{- "name": "s3-backup",- "backend": "s3",- "s3": {- "bucket": "my-bucket",- "endpoint": "http://localhost:9000",- "access_key": "your-access-key",- "secret_key": "your-secret-key",- "prefix": "openviking",- "use_ssl": false,- "use_path_style": true,- "directory_marker_mode": "none"- }-}-```--上面的示例使用 `none`,适用于不使用目录标记的 S3 兼容服务(RustFS、MinIO、Ceph 等)。backup 项的这个字段没有默认值;顶层 `storage` 的 S3 配置默认使用 `empty`。--### Docker 网络配置--在 Docker 中运行 OpenViking 并配置同主机的 S3 备份时,需要注意:--- **Linux Docker**:使用 `--network host` 或宿主机局域网 IP。Docker bridge 网络可通过网关 IP(如 `172.17.0.1:9000`)访问宿主机局域网。-- **macOS/Windows Docker Desktop**:从容器访问宿主机时可使用 `host.docker.internal`。Docker Desktop 4.34 及之后的版本也提供可选的 host networking,启用方式见 [Docker 官方说明](https://docs.docker.com/engine/network/drivers/host/)。--如果启用 S3 备份后服务静默崩溃,请优先排查 Docker 网络。RAGFS Rust binding 在容器内无法访问 S3 端点时会报 `dispatch failure` 错误。--## 同步模式选择--### 异步模式--异步模式适合大多数场景。--```json-{- "backups": {- "sync_type": "async",- "items": []- }-}-```--特点:--- primary 写入成功后立即返回。-- backup 写入在后台执行。-- 写入延迟低。-- 后端不可用期间,落后时间没有固定上限,需观察同步和恢复状态。--适合:--- 写入吞吐优先。-- backup 主要用于灾备。-- 可以接受最终一致性。--### 同步模式--同步模式会等待 backup 确认。--```json-{- "backups": {- "sync_type": "sync",- "write_ack_count": 1,- "write_ack_timeout_ms": 5000,- "items": [{"name": "local-backup", "backend": "local", "local": {"workspace": "./backup-data"}}]- }-}-```--参数说明:--| 参数 | 说明 |-| --- | --- |-| `write_ack_count` | 写入返回前至少需要多少个 backup 确认 |-| `write_ack_timeout_ms` | 等待 backup 确认的超时时间,单位毫秒 |--特点:--- 写入确认更强。-- 写入延迟受 backup 影响。-- 未确认的 backup 会继续由后台重试修复。-- primary 已写成功但 backup 未达确认数时,客户端可能收到错误;此时 primary 中可能已经存在数据。--适合:--- 希望尽量减少 primary 与 backup 的确认窗口。-- backup 延迟可控。-- 调用方能接受同步写入带来的额外延迟。--## 配置读加速--backup 默认不参与读取。要让 backup 服务读取,需要显式配置 `operations`。--```json-{- "name": "cache-backend",- "backend": "memfs",- "operations": [- {"operation": "write", "priority": 0},- {- "operation": "read",- "priority": 10- }- ]-}-```--读取优先级规则:--- `priority` 越小越优先。-- 只有声明 `read` 的 backup 才参与读取。-- primary 始终作为最终兜底。-- 冷备 backup 不建议配置读能力。--如果一个 backup 只配置了 `read`,没有配置 `write`,它不会接收普通复制写入。只有在你明确知道该 backend 的数据来源时,才应使用这种配置。+| `initial_partitions` | `16` | 为 account 创建的 metadata partition 数量 |+| `checkpoint_interval_secs` | `86400` | Checkpoint 扫描间隔,最小值为 `60` |+| `provider` | `"filesystem"` | Metadata segment provider:`filesystem` 或 `cache` |-## Redirect 配置+每个 `items[]` 需要 `name`、`backend` 和 backend 专用 `params`。+`encryption` 可选。backup 名称必须唯一,且不能为 `primary`。-Redirect 用于把匹配的文件写入指定 backup,而不是写入 primary。+`provider` 为 `cache` 时,多写复用顶层 CacheRuntime。实际实现由顶层+`cache.provider` 选择;多写没有独立 Cache Provider 连接配置。-按扩展名重定向:+Python 配置层仍接受 V1 的 sync、ack、retry、operations、excludes 和+redirects 字段,输出 warning 后会在调用 Rust 前删除,字段不生效。-```json-{- "storage": {- "agfs": {- "backend": "local",- "redirects": [- {- "type": "FileExtensionPolicy",- "extensions": ["(pdf|ppt|zip)"],- "target": ["object-store"]- }- ],- "backups": {- "items": [- {- "name": "object-store",- "backend": "s3",- "s3": {- "bucket": "openviking-large-files",- "endpoint": "https://s3.example.com"- }- }- ]- }- }- }-}-```--按大小重定向:--```json-{- "type": "FileOverSizePolicy",- "max_size_mb": 100,- "target": ["object-store"]-}-```--注意:--- `target` 必须引用已有 backup 的 `name`。-- redirect 文件仍会通过普通 API 呈现为可读、可列举、可查询状态。-- redirect 映射保存在 primary 的内部元数据中。--## Exclude 配置--Exclude 用于让某个 backup 跳过匹配文件。--```json-{- "name": "cache-backend",- "backend": "memfs",- "excludes": [- {- "type": "FileOverSizePolicy",- "max_size_mb": 50- },- {- "type": "FileExtensionPolicy",- "extensions": ["(mp4|zip)"]- }- ]-}-```--常见用法:--- 缓存 backend 排除大文件。-- 低成本备份排除无需保存的文件类型。-- 某个 backup 只保存文本或配置类资源。--如果 redirect 的目标 backup 同时 exclude 了该文件,说明配置互相冲突。请优先修正配置,不要依赖系统自动猜测其他目标。--## 加密配置--主备存储复用 OpenViking 的透明静态加密能力。--全局加密开启示例:--```json-{- "encryption": {- "enabled": true,- "provider": "local",- "local": {- "key_file": "~/.openviking/master.key"- }- },- "storage": {- "workspace": "./data",- "agfs": {- "backend": "local",- "backups": {- "items": [- {- "name": "plain-cache",- "backend": "memfs",- "encryption": {- "enabled": false- }- },- {- "name": "encrypted-backup",- "backend": "local",- "local": {- "workspace": "./data/encrypted-backup"- },- "encryption": {- "enabled": true- }- }- ]- }- }- }-}-```--规则:--- 全局 `encryption.enabled=true` 时,primary 必须加密。-- backup 可以通过 `encryption.enabled` 单独控制是否加密。-- Python SDK、HTTP API 和 CLI 不需要处理加解密。-- `.redirect.json` 和 `.sync_log.json` 等内部元数据会跟随 primary 加密策略。+## 运行行为-## 存量数据迁移--主备存储只复制启用之后的新写入,不会自动复制历史文件。--使用 OVPack 迁移时,请遵循 [与主备存储配合](./09-ovpack.md#与主备存储配合):先在干净目标配置主备存储,再通过该服务恢复,验证各副本后恢复业务写入。目标 account 初始化和恢复冲突处理也在该流程中说明。--## 验证配置--启动前建议运行:--```bash-openviking-server doctor-```--启动后可以用普通文件 API 验证:--```bash-ov write viking://resources/multiwrite-check.txt \- --content "multi-write check"--ov read viking://resources/multiwrite-check.txt-```--如果使用本地 backup,可以直接检查 backup 目录中是否出现对应文件。还应运行 `ov system backend sync-status viking://resources`,逐个核对预期副本。普通读取成功可能只读到了 primary,不能证明复制完成。--## 常见问题--### 为什么 backup 没有参与读取?--backup 默认只参与写入,不参与读取。需要在 backup 上显式配置:--```json-{- "operations": [- {"operation": "write", "priority": 0},- {- "operation": "read",- "priority": 10- }- ]-}-```+- primary 是唯一直接读取来源和业务真相。+- primary 成功后立即返回,不等待 metadata 或 backup。+- 多写 metadata 只存储在 primary。+- backup 在后台最终收敛。+- 不提供同步状态或手工重试 API 和 CLI 命令。-### 为什么启用主备存储后历史文件没有出现在 backup?+进程可能在 primary 成功和事件持久化之间崩溃,导致一次复制事件丢失。此时+需要从 primary 重建受影响的 backup。-主备存储只处理启用后的新写入。历史文件请按 [OVPack 迁移流程](./09-ovpack.md#与主备存储配合) 处理;启用 backups 不会自动补齐。+## Partition 数量-### 异步模式下能否保证立即读到 backup 的最新数据?+`initial_partitions` 仅在 account 的 partition manifest 首次创建时生效。+已有 account 在配置变化后继续使用持久化的 partition 数量和 routes。+本期不支持扩容或缩容。-不能。后台重试需要后端恢复可用才能追上进度。backup 返回旧数据但读取成功时,不会自动回退 primary;需要最新数据时,应让读路由使用 primary。+## 首次启用与回滚-### 内部元数据文件会出现在用户列表里吗?+`.multiwrite.json` 不存在时,mount 先启用前台 V2 事件提交,再在后台导入+primary 当前文件。导入不会删除 backup 中多余的旧文件。-不会。`.redirect.json` 和 `.sync_log.json` 是内部文件,会被普通目录列表隐藏。+旧多写协议和旧迁移字段直接拒绝。启用多写前必须创建完整备份;回滚二进制+前必须先恢复该备份。-### sync 模式返回失败是否表示 primary 一定没写入?+## 验证-不是。primary 写成功但 backup 未达到确认数时,客户端可能收到失败。此时 primary 数据可能已经存在,落后的 backup 会由后台重试修复。+使用普通文件 API 验证 primary 读写。使用[指标](../concepts/12-metrics.md)+监控队列深度、worker 失败、backup lag、协议版本和协议状态。## 相关文档-- [主备存储](../concepts/14-multi-write-storage.md)-- [存储架构](../concepts/05-storage.md)+- [多写存储](../concepts/14-multi-write-storage.md)- [配置指南](./01-configuration.md)-- [加密指南](./08-encryption.md)- [OVPack 导入导出](./09-ovpack.md)diff --git a/examples/claude-code-memory-plugin/.claude-plugin/plugin.json b/examples/claude-code-memory-plugin/.claude-plugin/plugin.jsonindex 627d750e5..c839bd3b2 100644--- a/examples/claude-code-memory-plugin/.claude-plugin/plugin.json+++ b/examples/claude-code-memory-plugin/.claude-plugin/plugin.json@@ -1,6 +1,6 @@{"name": "openviking-memory",- "version": "0.7.10",+ "version": "0.7.11","description": "Long-term semantic memory for Claude Code, powered by OpenViking. Auto-recall relevant memories at session start and capture important information during conversations.","author": {"name": "OpenViking",@@ -17,4 +17,4 @@],"mcpServers": "./.mcp.json","types": "./mods/usage/types.d.ts"-}+}\ No newline at end of filediff --git a/examples/claude-code-memory-plugin/package.json b/examples/claude-code-memory-plugin/package.jsonindex 71eb2f10f..52205300a 100644--- a/examples/claude-code-memory-plugin/package.json+++ b/examples/claude-code-memory-plugin/package.json@@ -1,6 +1,6 @@{"name": "claude-code-openviking-memory",- "version": "0.7.10",+ "version": "0.7.11","description": "OpenViking memory plugin for Claude Code — semantic long-term memory","type": "module","scripts": {diff --git a/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md b/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.mdindex 684d8172a..04c78ecc9 100644--- a/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md+++ b/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md@@ -159,7 +159,7 @@ port is bound only after initialization finishes, so "connection refused"during startup is normal; `/health` 200 says nothing about embedding or VLM.`GET /ready` (no auth, 200 or 503):-`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…,"multiwrite_sync":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`.+`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`.`ok`, `not_configured` and `not_supported` count as healthy; `embedding` is areal embed call with a 10s cap. `503 {"status":"not_ready","reason":"initializing"}`while booting; 404 on servers that predate the endpoint. The official dockerdiff --git a/examples/codex-memory-plugin/.codex-plugin/plugin.json b/examples/codex-memory-plugin/.codex-plugin/plugin.jsonindex 2a3f34ac2..4870aa30a 100644--- a/examples/codex-memory-plugin/.codex-plugin/plugin.json+++ b/examples/codex-memory-plugin/.codex-plugin/plugin.json@@ -1,6 +1,6 @@{"name": "openviking-memory",- "version": "0.11.9",+ "version": "0.11.10","description": "Long-term semantic memory for Codex, powered by OpenViking. Recall on UserPromptSubmit, incremental add_message on Stop (per turn), commit on SessionEnd (Codex >= 0.145) and on PreCompact, with an idle/ended sweep on SessionStart as the fallback for signals, crashes and older Codex builds. Includes a local stdio MCP proxy that reads OpenViking credentials from env/ovcli.conf.","author": {"name": "OpenViking",diff --git a/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md b/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.mdindex 63b6a73c8..6e36ee300 100644--- a/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md+++ b/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md@@ -152,7 +152,7 @@ port is bound only after initialization finishes, so "connection refused"during startup is normal; `/health` 200 says nothing about embedding or VLM.`GET /ready` (no auth, 200 or 503):-`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…,"multiwrite_sync":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`.+`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`.`ok`, `not_configured` and `not_supported` count as healthy; `embedding` is areal embed call with a 10s cap. `503 {"status":"not_ready","reason":"initializing"}`while booting; 404 on servers that predate the endpoint. The official dockerdiff --git a/examples/memory-plugin-shared/doctor-core.test.mjs b/examples/memory-plugin-shared/doctor-core.test.mjsindex 8ff2b844c..86d8bf55b 100644--- a/examples/memory-plugin-shared/doctor-core.test.mjs+++ b/examples/memory-plugin-shared/doctor-core.test.mjs@@ -237,7 +237,7 @@ test("lintServerConf covers every harness, not just the two with a legacy ov.con});test("assessReady interprets the readiness checks", () => {- const ok = { ok: true, status: 200, json: { status: "ready", checks: { agfs: { status: "ok", checks: { filesystem: "ok", multiwrite_sync: "not_supported" } }, vectordb: "ok", api_key_manager: "not_configured", embedding: "ok", ollama: "not_configured" } } };+ const ok = { ok: true, status: 200, json: { status: "ready", checks: { agfs: { status: "ok", checks: { filesystem: "ok" } }, vectordb: "ok", api_key_manager: "not_configured", embedding: "ok", ollama: "not_configured" } } };let report = createReport();assert.equal(assessReady(report, ok).ready, true);assert.equal(report.exitCode(), 0);diff --git a/examples/ov.conf.example b/examples/ov.conf.exampleindex 5b81cb297..8650d0b0c 100644--- a/examples/ov.conf.example+++ b/examples/ov.conf.example@@ -77,6 +77,17 @@"agfs": {"backend": "local","timeout": 10,+ "backups_example": {+ "_comment": "Rename this key to backups to enable V2 multi-write.",+ "initial_partitions": 16,+ "checkpoint_interval_secs": 86400,+ "provider": "cache",+ "items": [{+ "name": "local-backup",+ "backend": "local",+ "params": {"workspace": "./data/backup"}+ }]+ },"cachefs": {"backend": "local","namespace": "openviking",diff --git a/openviking/pyagfs/async_client.py b/openviking/pyagfs/async_client.pyindex ebfa1d563..4c6657e8c 100644--- a/openviking/pyagfs/async_client.py+++ b/openviking/pyagfs/async_client.py@@ -349,18 +349,6 @@ class AsyncAGFSClient:ctx=_fs_ctx_or_default(path, fs_ctx),)- async def system_sync_status(- self, path: str, *, fs_ctx: Dict[str, str] | None = None- ) -> Dict[str, Any]:- """Return multi-write sync status for a file or directory path."""- return await self.run("system_sync_status", path, ctx=_fs_ctx_or_default(path, fs_ctx))-- async def system_sync_retry(- self, path: str, *, fs_ctx: Dict[str, str] | None = None- ) -> Dict[str, Any]:- """Retry pending multi-write sync work for a file or directory path."""- return await self.run("system_sync_retry", path, ctx=_fs_ctx_or_default(path, fs_ctx))-# -- pathlock async wrappers ------------------------------------------------async def pathlock_acquire_exact(diff --git a/openviking/pyagfs/protocols.py b/openviking/pyagfs/protocols.pyindex 020151cb8..5c2a484b2 100644--- a/openviking/pyagfs/protocols.py+++ b/openviking/pyagfs/protocols.py@@ -118,9 +118,3 @@ class AGFSSyncClientProtocol(Protocol):continuation_token: str | None = None,) -> Dict[str, Any]:"""Return one page of flat glob matches for the given AGFS directory."""-- def system_sync_status(self, path: str) -> Dict[str, Any]:- """Return multi-write sync status for a file or directory path."""-- def system_sync_retry(self, path: str) -> Dict[str, Any]:- """Retry pending multi-write sync work for a file or directory path."""diff --git a/openviking/server/auth/plugins/api_key.py b/openviking/server/auth/plugins/api_key.pyindex 686444728..75d44bb92 100644--- a/openviking/server/auth/plugins/api_key.py+++ b/openviking/server/auth/plugins/api_key.py@@ -24,8 +24,6 @@ _API_KEY_ROOT_ALLOWED_PREFIXES = ("/api/v1/observer","/api/v1/console","/api/v1/tasks",- "/api/v1/system/backend",- "/api/v1/system/sync",)diff --git a/openviking/server/routers/system.py b/openviking/server/routers/system.pyindex 0ef245f79..78b62f4d6 100644--- a/openviking/server/routers/system.py+++ b/openviking/server/routers/system.py@@ -38,19 +38,13 @@ def _is_ready_check_ok(value) -> bool:async def _probe_agfs_readiness() -> dict[str, object]:- """Return structured AGFS readiness, including multi-write sync health when available."""+ """Return structured AGFS filesystem readiness."""viking_fs = get_viking_fs()checks: dict[str, object] = {}await viking_fs.ls("viking://", ctx=None)checks["filesystem"] = "ok"- try:- await viking_fs.system_sync_status("viking://", ctx=None)- checks["multiwrite_sync"] = "ok"- except (AGFSInvalidOperationError, AGFSNotSupportedError):- checks["multiwrite_sync"] = "not_supported"-return {"status": "ok", "checks": checks}@@ -126,7 +120,7 @@ async def readiness_check(request: Request):checks = {}- # 1. AGFS: probe filesystem access and multi-write sync health+ # 1. AGFS: probe filesystem accesstry:checks["agfs"] = await _probe_agfs_readiness()except Exception as e:@@ -220,12 +214,6 @@ class ConsistencyRequest(BaseModel):uri: str-class BackendSyncRequest(BaseModel):- """Request model for backend sync status and retry operations."""-- uri: str--@router.post("/api/v1/system/wait", tags=["system"])async def wait_processed(request: WaitRequest,@@ -250,51 +238,3 @@ async def check_consistency(ctx=ctx,)return Response(status="ok", result=result)--[email protected]("/api/v1/system/backend/sync-status", tags=["system"])-async def backend_sync_status(- request: BackendSyncRequest,- ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),-):- """Return multi-write backend sync status for a Viking URI subtree."""- service = get_service()- uri = validate_request_viking_uri(resolve_path_variables(request.uri), ctx)- result = await service.fs.system_sync_status(uri, ctx=ctx)- return Response(status="ok", result=result)--[email protected]("/api/v1/system/backend/sync-retry", tags=["system"])-async def backend_sync_retry(- request: BackendSyncRequest,- ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),-):- """Retry pending multi-write backend sync work for a Viking URI subtree."""- service = get_service()- uri = validate_request_viking_uri(resolve_path_variables(request.uri), ctx)- result = await service.fs.system_sync_retry(uri, ctx=ctx)- return Response(status="ok", result=result)--[email protected]("/api/v1/system/sync/{sync_path:path}", tags=["system"])-async def admin_sync_status(- sync_path: str,- ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),-):- """Return multi-write backend sync status for one URI subtree through the admin API."""- service = get_service()- uri = validate_request_viking_uri(resolve_path_variables(sync_path), ctx)- result = await service.fs.system_sync_status(uri, ctx=ctx)- return Response(status="ok", result=result)--[email protected]("/api/v1/system/sync/{sync_path:path}/retry", tags=["system"])-async def admin_sync_retry(- sync_path: str,- ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),-):- """Retry pending multi-write backend sync work for one URI subtree through the admin API."""- service = get_service()- uri = validate_request_viking_uri(resolve_path_variables(sync_path), ctx)- result = await service.fs.system_sync_retry(uri, ctx=ctx)- return Response(status="ok", result=result)diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.pyindex 163e75e66..ee32386a7 100644--- a/openviking/service/fs_service.py+++ b/openviking/service/fs_service.py@@ -1062,16 +1062,6 @@ class FSService:viking_fs = self._ensure_initialized()await viking_fs._ensure_access(uri, ctx, action=AclAction.WRITE)- async def system_sync_status(self, uri: str, ctx: RequestContext) -> Dict[str, Any]:- """Return multi-write sync status for one Viking URI subtree."""- viking_fs = self._ensure_initialized()- return await viking_fs.system_sync_status(uri, ctx=ctx)-- async def system_sync_retry(self, uri: str, ctx: RequestContext) -> Dict[str, Any]:- """Retry multi-write sync work for one Viking URI subtree."""- viking_fs = self._ensure_initialized()- return await viking_fs.system_sync_retry(uri, ctx=ctx)-async def read(self, uri: str, ctx: RequestContext, offset: int = 0, limit: int = -1) -> str:"""Read file content. Accepts a Viking URI or a 32-char hex vector record id."""viking_fs = self._ensure_initialized()diff --git a/openviking/storage/viking_fs/_snapshot.py b/openviking/storage/viking_fs/_snapshot.pyindex 92dc4df79..c5f561947 100644--- a/openviking/storage/viking_fs/_snapshot.py+++ b/openviking/storage/viking_fs/_snapshot.py@@ -121,30 +121,6 @@ class _SnapshotMixin:def _restore_reindex_context(ctx: RequestContext) -> RequestContext:return replace(ctx, bypass_acl=True)- async def system_sync_status(- self, uri: str, ctx: Optional[RequestContext] = None- ) -> Dict[str, Any]:- """Return multi-write sync status for one Viking URI subtree."""- await self._ensure_access(uri, ctx)- real_ctx = self._ctx_or_default(ctx)- path = self._uri_to_path(uri, ctx=ctx)- return await self._async_agfs.system_sync_status(- path,- fs_ctx={"account_id": real_ctx.account_id},- )-- async def system_sync_retry(- self, uri: str, ctx: Optional[RequestContext] = None- ) -> Dict[str, Any]:- """Retry multi-write sync for one Viking URI subtree."""- await self._ensure_access(uri, ctx, action=AclAction.WRITE)- real_ctx = self._ctx_or_default(ctx)- path = self._uri_to_path(uri, ctx=ctx)- return await self._async_agfs.system_sync_retry(- path,- fs_ctx={"account_id": real_ctx.account_id},- )-async def get_gitignore(self, ctx: Optional[RequestContext] = None) -> str:"""Return the account-level .ovgitignore content, or an empty string if absent.diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.pyindex 6f6fc7b83..6acc259e1 100644--- a/openviking/utils/agfs_utils.py+++ b/openviking/utils/agfs_utils.py@@ -41,7 +41,13 @@ class RagfsBindingConfig:cachefs_backend = getattr(cachefs, "backend", "local")pathlock = self.agfs.pathlock.model_dump(mode="json", exclude_none=True)pathlock_uses_cache = pathlock.get("provider") == "cache"- uses_runtime = cachefs_backend == "cache" or queue_backend == "cache" or pathlock_uses_cache+ backups = getattr(self.agfs, "backups", None)+ uses_runtime = (+ cachefs_backend == "cache"+ or queue_backend == "cache"+ or pathlock_uses_cache+ or _get_config_value(backups, "provider", "filesystem") == "cache"+ )if (pathlock_uses_cacheand self.cache is not None@@ -60,8 +66,8 @@ class RagfsBindingConfig:else:if uses_runtime:raise ValueError(- "top-level cache config is required when CacheFS, QueueFS, or cache-backed PathLock "- "uses the shared Runtime"+ "top-level cache config is required when CacheFS, QueueFS, cache-backed PathLock, "+ "or cache-backed multi-write uses the shared Runtime")cache_config = _disabled_cache_config(cachefs)binding_config: Dict[str, Any] = {@@ -301,10 +307,8 @@ def _generate_plugin_config(# Check for multi-write configurationbackups_config = getattr(agfs_config, "backups", None)redirects_config = getattr(agfs_config, "redirects", None)- if redirects_config is not None and backups_config is None:- raise ValueError(- "redirects requires backups; single-backend mode does not support redirects"- )+ if redirects_config is not None:+ logger.warning("storage.agfs.redirects is deprecated and ignored")# Build primary backend plugin configprimary_plugin_config: Dict[str, Any] = {}@@ -326,12 +330,6 @@ def _generate_plugin_config(mount_config["server_encryption_enabled"] = server_encryption_enabledmount_config["primary_encryption_enabled"] = server_encryption_enabled- # Serialize redirect policies- if redirects_config is not None:- mount_config["primary_redirects"] = [- _serialize_redirect_policy(p) for p in redirects_config- ]-# Determine the plugin type name for the primary backendif backend == "local":plugin_name = "localfs"@@ -370,6 +368,9 @@ def _get_config_value(config: Any, key: str, default: Any = None) -> Any:def _get_backend_specific_params(item: Any) -> Any:"""Return backend-specific nested params from one backup item using the backend name key."""+ params = _get_config_value(item, "params")+ if params is not None:+ return paramsbackend_params = _get_config_value(item, "backend_params")if backend_params is not None:return backend_params@@ -482,21 +483,42 @@ def _normalize_backup_item(item: Any, data_path: Path) -> Dict[str, Any]:def _serialize_backups_config(backups_config: Any, data_path: Path) -> Dict[str, Any]:- """Serialize raw multi-write backups config with top-level passthrough and normalized items."""+ """Serialize V2 backups while warning about and stripping legacy fields."""result = _dump_config_object(backups_config)- result.setdefault("sync_type", "async")- result["items"] = [- _normalize_backup_item(item, data_path)- for item in _get_config_value(backups_config, "items", [])- ]+ deprecated_fields = []+ for field_name in (+ "sync_type",+ "write_ack_count",+ "write_ack_timeout_ms",+ "write_concurrency",+ "retry_interval_ms",+ "retry_backoff_base_ms",+ "retry_max_retries_per_round",+ "retry_quarantine_after_failures",+ "read_probe_cache_ttl_ms",+ ):+ if field_name in result:+ deprecated_fields.append(field_name)+ result.pop(field_name)++ items = []+ for item in _get_config_value(backups_config, "items", []):+ normalized = _normalize_backup_item(item, data_path)+ for field_name in ("timeout", "operations", "excludes"):+ if field_name in normalized:+ deprecated_fields.append(f"items.{field_name}")+ normalized.pop(field_name)+ items.append(normalized)+ result["items"] = items++ if deprecated_fields:+ logger.warning(+ "Deprecated backups configuration fields are ignored: %s",+ ", ".join(dict.fromkeys(deprecated_fields)),+ )return result-def _serialize_redirect_policy(policy: Any) -> Dict[str, Any]:- """Serialize one raw redirect/exclude policy object to a dict."""- return _dump_config_object(policy)--def _build_git_config_dict(git_config: Any, storage_path: Path) -> Dict[str, Any]:"""Build an in-memory git config dict consumed by the ragfs binding.diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.pyindex a9afb07aa..fb8de031b 100644--- a/openviking_cli/utils/config/agfs_config.py+++ b/openviking_cli/utils/config/agfs_config.py@@ -446,7 +446,7 @@ class AGFSConfig(BaseModel):default=None, description="Multi-write backups configuration. None = single backend mode.")redirects: Optional[List[dict[str, Any]]] = Field(- default=None, description="Primary redirect policies."+ default=None, description="[Deprecated] Primary redirect policies. Ignored by V2.")@model_validator(mode="after")@@ -494,9 +494,38 @@ class AGFSConfig(BaseModel):"db_path/queue_db_path will be ignored.")- if self.redirects is not None and self.backups is None:- raise ValueError(- "redirects requires backups; single-backend mode does not support redirects"- )+ if self.backups is not None:+ initial_partitions = self.backups.get("initial_partitions", 16)+ if (+ not isinstance(initial_partitions, int)+ or isinstance(initial_partitions, bool)+ or not 1 <= initial_partitions <= 1024+ ):+ raise ValueError("backups initial_partitions must be between 1 and 1024")++ checkpoint_interval_secs = self.backups.get("checkpoint_interval_secs", 86400)+ if (+ not isinstance(checkpoint_interval_secs, int)+ or isinstance(checkpoint_interval_secs, bool)+ or checkpoint_interval_secs < 60+ ):+ raise ValueError("backups checkpoint_interval_secs must be at least 60")++ if self.backups.get("provider", "filesystem") not in {"filesystem", "cache"}:+ raise ValueError("backups provider must be 'filesystem' or 'cache'")++ names = set()+ for item in self.backups.get("items", []):+ if not isinstance(item, dict):+ raise ValueError("backups items must be objects")+ name = item.get("name")+ if not isinstance(name, str) or not name.strip():+ raise ValueError("backup name must not be empty")+ name = name.strip()+ if name == "primary":+ raise ValueError("backup name 'primary' is reserved")+ if name in names:+ raise ValueError("backup names must be unique")+ names.add(name)return selfdiff --git a/tests/agfs/test_async_client.py b/tests/agfs/test_async_client.pyindex 9c8b025d4..cb0897771 100644--- a/tests/agfs/test_async_client.py+++ b/tests/agfs/test_async_client.py@@ -6,10 +6,17 @@ import threadingimport pytest-from openviking.pyagfs import AsyncAGFSClient+from openviking.pyagfs import AsyncAGFSClient, RAGFSBindingClientfrom openviking.storage.viking_vector_index_backend import _AsyncVectorAdapter+def test_removed_sync_methods_are_not_exposed():+ """Removed sync methods must stay absent from Python and native clients."""+ for client_type in (AsyncAGFSClient, RAGFSBindingClient):+ assert not hasattr(client_type, "system_sync_status")+ assert not hasattr(client_type, "system_sync_retry")++@pytest.mark.asyncio@pytest.mark.parametrize("backend", ["agfs", "vector"])async def test_async_agfs_client_hides_threadpool(backend):diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.pyindex 76a73cbb7..a632835c8 100644--- a/tests/misc/test_config_validation.py+++ b/tests/misc/test_config_validation.py@@ -440,59 +440,62 @@ def test_generate_plugin_config_forwards_queuefs_runtime_options():assert plugins["queuefs"]["config"]["busy_timeout_ms"] == 1234-def test_agfs_redirects_require_backups():- """Single-backend mode must reject redirect policies during config validation."""- with pytest.raises(ValueError, match="redirects requires backups"):- AGFSConfig(- path="/tmp/ov-test",- backend="local",- redirects=[- {- "type": "FileExtensionPolicy",- "extensions": ["(md)"],- "target": ["s3-backup"],- }- ],- )+def test_agfs_redirects_without_backups_are_parse_only():+ """Legacy redirects must remain constructible without enabling multi-write."""+ redirects = [+ {+ "type": "FileExtensionPolicy",+ "extensions": ["(md)"],+ "target": ["missing-backup"],+ }+ ]++ config = AGFSConfig(+ path="/tmp/ov-test",+ backend="local",+ redirects=redirects,+ )+ assert config.redirects == redirects-def test_generate_plugin_config_rejects_redirects_without_backups(tmp_path):- """Runtime plugin config generation must also reject redirect-only configs."""- config = type(- "RedirectOnlyConfig",- (),++def test_generate_plugin_config_ignores_redirects_without_backups(tmp_path):+ """Legacy redirects must not alter a single-backend mount config."""+ redirects = [{- "backend": "local",- "s3": None,- "backups": None,- "redirects": [- type(- "RedirectPolicy",- (),- {- "type": "FileExtensionPolicy",- "extensions": ["(md)"],- "target": ["s3-backup"],- },- )()- ],- "queuefs": type(- "QueueConfig",- (),- {- "mode": "shared",- "backend": "sqlite",- "db_path": None,- "recover_stale_sec": 0,- "busy_timeout_ms": 5000,- },- )(),- "queue_db_path": None,- },- )()+ "type": "FileExtensionPolicy",+ "extensions": ["(md)"],+ "target": ["missing-backup"],+ }+ ]+ config = AGFSConfig(path=str(tmp_path), backend="local", redirects=redirects)+ plain_config = AGFSConfig(path=str(tmp_path), backend="local")++ mount_config = _generate_plugin_config(config, tmp_path)["localfs"]["config"]+ plain_mount_config = _generate_plugin_config(plain_config, tmp_path)["localfs"]["config"]++ assert mount_config == plain_mount_config+++def test_generate_plugin_config_ignores_redirects_with_backups(tmp_path):+ """Legacy redirects must remain accepted without reaching Rust."""+ redirects = [+ {+ "type": "FileExtensionPolicy",+ "extensions": ["(md)"],+ "target": ["missing-backup"],+ }+ ]+ config = AGFSConfig(+ path=str(tmp_path),+ backend="local",+ backups={"items": [{"name": "mem-backup", "backend": "memory"}]},+ redirects=redirects,+ )++ mount_config = _generate_plugin_config(config, tmp_path)["localfs"]["config"]- with pytest.raises(ValueError, match="redirects requires backups"):- _generate_plugin_config(config, tmp_path)+ assert "primary_redirects" not in mount_configdef test_generate_plugin_config_passes_multiwrite_encryption_flag(tmp_path):@@ -517,20 +520,85 @@ def test_generate_plugin_config_passes_multiwrite_encryption_flag(tmp_path):assert mount_config["primary_encryption_enabled"] is True-def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path):- """Plugin config generation should normalize backup params while preserving top-level multi-write fields."""+def test_generate_plugin_config_preserves_v2_backups_without_sync_type(tmp_path):+ """V2 backup fields must pass through without injecting a V1 sync mode."""+ config = AGFSConfig(+ path=str(tmp_path),+ backend="local",+ backups={+ "initial_partitions": 32,+ "checkpoint_interval_secs": 120,+ "provider": "filesystem",+ "items": [{"name": "mem-backup", "backend": "memory"}],+ },+ )++ backups = _generate_plugin_config(config, tmp_path)["localfs"]["config"]["backups"]++ assert backups["initial_partitions"] == 32+ assert backups["checkpoint_interval_secs"] == 120+ assert backups["provider"] == "filesystem"+ assert "sync_type" not in backups++[email protected]("provider", ["filesystem", "cache"])+def test_agfs_config_accepts_supported_v2_backup_providers(provider):+ """AGFS config must accept each specified metadata provider."""+ config = AGFSConfig(+ path="/tmp/ov-test",+ backend="local",+ backups={"provider": provider, "items": []},+ )++ assert config.backups["provider"] == provider+++ ("backups", "expected"),+ [+ ({"initial_partitions": 0, "items": []}, "initial_partitions"),+ ({"initial_partitions": 1025, "items": []}, "initial_partitions"),+ ({"checkpoint_interval_secs": 59, "items": []}, "checkpoint_interval_secs"),+ ({"provider": "memory", "items": []}, "provider"),+ ({"provider": "redis", "items": []}, "provider"),+ ({"items": [{"name": " ", "backend": "memory"}]}, "name"),+ ({"items": [{"name": "primary", "backend": "memory"}]}, "primary"),+ (+ {+ "items": [+ {"name": "backup-a", "backend": "memory"},+ {"name": "backup-a", "backend": "memory"},+ ]+ },+ "unique",+ ),+ ],+)+def test_agfs_config_rejects_invalid_v2_backups(backups, expected):+ """AGFS config must reject invalid V2 backup fields before Rust startup."""+ with pytest.raises(ValueError, match=expected):+ AGFSConfig(path="/tmp/ov-test", backend="local", backups=backups)+++def test_generate_plugin_config_strips_legacy_multiwrite_fields(tmp_path):+ """Plugin config generation should strip legacy fields before calling Rust."""explicit_backup_dir = tmp_path / "backup-local-no-mkdir"config = AGFSConfig(path=str(tmp_path),backend="local",backups={- "retry_interval_ms": 1234,- "retry_backoff_base_ms": 55,+ "sync_type": "async",+ "write_ack_count": 1,+ "write_ack_timeout_ms": 1000,+ "write_concurrency": 2,"items": [{"name": "local-explicit","backend": "local",- "local": {"workspace": str(explicit_backup_dir)},+ "params": {"workspace": str(explicit_backup_dir)},+ "timeout": 10,+ "operations": [{"operation": "write", "priority": 1}],+ "excludes": [{"type": "FileExtensionPolicy", "extensions": [".tmp"]}],},{"name": "local-default",@@ -558,12 +626,17 @@ def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path):plugins = _generate_plugin_config(config, tmp_path)mount_backups = plugins["localfs"]["config"]["backups"]- assert mount_backups["retry_interval_ms"] == 1234- assert mount_backups["retry_backoff_base_ms"] == 55+ assert {+ "sync_type",+ "write_ack_count",+ "write_ack_timeout_ms",+ "write_concurrency",+ }.isdisjoint(mount_backups)explicit_local, default_local, s3_backup = mount_backups["items"]assert explicit_local["backend"] == "localfs"assert explicit_local["params"]["local_dir"] == str(explicit_backup_dir)+ assert {"timeout", "operations", "excludes"}.isdisjoint(explicit_local)assert not explicit_backup_dir.exists()assert default_local["backend"] == "localfs"@@ -705,6 +778,23 @@ def test_ragfs_binding_enables_runtime_for_queuefs_cache_backend(tmp_path):assert binding["cache"]["runtime_enabled"] is True+def test_ragfs_binding_enables_runtime_for_multiwrite_cache_provider(tmp_path):+ """Multi-write cache metadata should enable Runtime without CacheFS."""+ agfs_config = AGFSConfig(+ path=str(tmp_path),+ backend="local",+ backups={"provider": "cache", "items": []},+ )++ binding = RagfsBindingConfig(+ agfs=agfs_config,+ cache=CacheConfig(provider="redis", params={}),+ ).to_binding_dict()++ assert binding["cache"]["enabled"] is False+ assert binding["cache"]["runtime_enabled"] is True++def test_agfs_pathlock_config_validates_provider_and_expiry(tmp_path):"""PathLock config validates Redis namespace and expiry boundaries."""config = AGFSConfig(diff --git a/tests/server/test_admin_api.py b/tests/server/test_admin_api.pyindex 739f97899..365a7f6f1 100644--- a/tests/server/test_admin_api.py+++ b/tests/server/test_admin_api.py@@ -34,7 +34,7 @@ from openviking.server.user_config import (user_config_backup_uri,)from openviking.service.core import OpenVikingService-from openviking.service.deletion import setup_deletion+from openviking.service.deletion import DeletionService, setup_deletionfrom openviking.service.task_store import (SYSTEM_TASK_ACCOUNT_ID,SYSTEM_TASK_USER_ID,@@ -311,6 +311,28 @@ async def _wait_for_task(client: httpx.AsyncClient, task_id: str) -> dict:# ---- Account CRUD ----+async def test_delete_account_files_disables_automatic_pathlock(monkeypatch):+ """Account cleanup must bypass the deleted tree's automatic PathLock."""+ path = "/local/account-to-delete"+ agfs = Mock()+ agfs.rm = AsyncMock()+ agfs.stat = AsyncMock(side_effect=AGFSNotFoundError(path))+ service = Mock()+ service.viking_fs._async_agfs = agfs+ tracker = Mock()+ monkeypatch.setattr("openviking.service.deletion.get_task_tracker", lambda: tracker)+ deletion = DeletionService(+ service=service,+ manager=Mock(),+ service_loop=asyncio.get_running_loop(),+ )++ await deletion._delete_account_files("account-to-delete")++ agfs.rm.assert_awaited_once_with(path, recursive=True, auto_pathlock=False)+ tracker.forget_account_tasks.assert_called_once_with("account-to-delete")++@pytest_asyncio.fixtureasync def template_account(lightweight_admin_client):account_id = _uid()@@ -324,17 +346,31 @@ async def template_account(lightweight_admin_client):async def test_studio_file_configuration_permissions_revision_and_overrides(- lightweight_admin_client, lightweight_admin_app, template_account, tmp_path, monkeypatch,+ lightweight_admin_client,+ lightweight_admin_app,+ template_account,+ tmp_path,+ monkeypatch,):from openviking.config.scope import ConfigScopefrom openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingletonaccount_id, admin_headers = template_accountconfig_path = tmp_path / "ov.conf"- config_path.write_text(json.dumps({"vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "file-secret"}}))+ config_path.write_text(+ json.dumps({"vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "file-secret"}})+ )monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file", config_path)manager = lightweight_admin_app.state.fake_service.runtime_config_manager- await manager.patch_account(account_id, {"vlm": {"model": "override-model", "credentials": [{"provider": "openai", "api_key": "account-secret"}]}})+ await manager.patch_account(+ account_id,+ {+ "vlm": {+ "model": "override-model",+ "credentials": [{"provider": "openai", "api_key": "account-secret"}],+ }+ },+ )url = "/api/v1/admin/configuration"params = {"source": "file", "account_id": account_id}denied = await lightweight_admin_client.get(url, params=params, headers=admin_headers)@@ -355,15 +391,28 @@ async def test_studio_file_configuration_permissions_revision_and_overrides(assert override_response.json()["result"]["overrides"]["cluster"] == ["query_planner"]content = json.loads(result["content"])content["rerank"] = {- "provider": "jev", "api_key": "jev-secret", "mode": "choice", "threshold": 0+ "provider": "jev",+ "api_key": "jev-secret",+ "mode": "choice",+ "threshold": 0,}body = {"revision": result["revision"], "content": json.dumps(content)}- assert (await lightweight_admin_client.patch(url, params={"source": "file"}, headers=admin_headers, json=body)).status_code == 403- saved = await lightweight_admin_client.patch(url, params={"source": "file"}, headers=root_headers(), json=body)+ assert (+ await lightweight_admin_client.patch(+ url, params={"source": "file"}, headers=admin_headers, json=body+ )+ ).status_code == 403+ saved = await lightweight_admin_client.patch(+ url, params={"source": "file"}, headers=root_headers(), json=body+ )assert saved.status_code == 200, saved.textassert saved.json()["result"]["restart_required"]assert json.loads(config_path.read_text())["rerank"]["provider"] == "jev"- assert (await lightweight_admin_client.patch(url, params={"source": "file"}, headers=root_headers(), json=body)).status_code == 400+ assert (+ await lightweight_admin_client.patch(+ url, params={"source": "file"}, headers=root_headers(), json=body+ )+ ).status_code == 400settings = await manager.get_settings(ConfigScope.account(account_id))assert settings["vlm"]["model"] == "override-model"@@ -987,12 +1036,10 @@ async def test_account_memory_templates_range_expressions_reach_file_body(fs = lightweight_admin_app.state.fake_service.viking_fssnapshot = await resolve_account_memory_registry(fs, account_id, MemoryTypeRegistry())schema = snapshot.get("events")- context = ExtractContext(- [- Message(id="m1", role="user", parts=[TextPart("Selected source message")]),- Message(id="m2", role="user", parts=[TextPart("Unrelated source message")]),- ]- )+ context = ExtractContext([+ Message(id="m1", role="user", parts=[TextPart("Selected source message")]),+ Message(id="m2", role="user", parts=[TextPart("Unrelated source message")]),+ ])rendered = MemoryFileUtils.write(MemoryFile(memory_type="events", extra_fields={"ranges": "0", "summary": "summary"}),content_template=schema.content_template,@@ -1315,33 +1362,29 @@ async def test_account_memory_templates_locked_fields_cannot_be_modified(for key, value in defaults.items()if key not in editable]- invalid.extend(- [- {"unknown": "ignored?"},- {"Description": "wrong key"},- {"_updated_at": "forged"},- {"_account_content_template": False},- {"deployment_content_template": "forged"},- {"enabled": 1},- {"fields": [{"name": "renamed_field", "description": "changed"}]},- {"description": None},- {"description": " \n"},- {"fields": [None]},- ]- )+ invalid.extend([+ {"unknown": "ignored?"},+ {"Description": "wrong key"},+ {"_updated_at": "forged"},+ {"_account_content_template": False},+ {"deployment_content_template": "forged"},+ {"enabled": 1},+ {"fields": [{"name": "renamed_field", "description": "changed"}]},+ {"description": None},+ {"description": " \n"},+ {"fields": [None]},+ ])for field in defaults["fields"]:invalid.extend({"fields": [{"name": field["name"], key: None if value is not None else "changed"}]}for key, value in field.items()if key not in {"name", "description"})- invalid.extend(- [- {"fields": [{"name": field["name"], "unknown": "ignored?"}]},- {"fields": [{"name": field["name"], "description": ""}]},- {"fields": [{"name": field["name"]}, {"name": field["name"]}]},- ]- )+ invalid.extend([+ {"fields": [{"name": field["name"], "unknown": "ignored?"}]},+ {"fields": [{"name": field["name"], "description": ""}]},+ {"fields": [{"name": field["name"]}, {"name": field["name"]}]},+ ])if memory_type == "events":invalid.extend({"fields": [{"name": name, "description": "changed"}]} for name in ("goal", "ranges")@@ -2131,6 +2174,33 @@ async def test_list_users_for_default_account_without_users_file_returns_empty_lassert resp.json()["result"] == []+async def test_list_accounts_with_watcher_uses_memory(+ lightweight_admin_client: httpx.AsyncClient,+ lightweight_admin_app: FastAPI,+ monkeypatch: pytest.MonkeyPatch,+):+ """A running watcher keeps list requests on the in-memory fast path."""+ loop = asyncio.get_running_loop()+ watch_task = loop.create_future()++ async def _unexpected_read(_path: str):+ raise AssertionError("list accounts should not read AGFS while watcher is running")++ manager = lightweight_admin_app.state.api_key_manager+ manager._store._watch_task = watch_task+ monkeypatch.setattr(manager._store, "_read_json", _unexpected_read)+ try:+ resp = await lightweight_admin_client.get("/api/v1/admin/accounts", headers=root_headers())+ users_resp = await lightweight_admin_client.get(+ "/api/v1/admin/accounts/default/users", headers=root_headers()+ )+ finally:+ watch_task.cancel()++ assert resp.status_code == 200, resp.text+ assert users_resp.status_code == 200, users_resp.text++async def test_identity_settings_refreshes_a_stale_registry_on_demand(admin_client: httpx.AsyncClient,admin_app: FastAPI,@@ -2308,22 +2378,18 @@ async def test_delete_account(account_id=SYSTEM_TASK_ACCOUNT_ID,user_id=SYSTEM_TASK_USER_ID,)- await queue.enqueue(- {- "task_id": user_cleanup.task_id,- "account_id": acct,- "user_id": "alice",- "target": {"account_id": acct, "user_id": "bob"},- }- )- await queue.enqueue(- {- "task_id": late_task_id,- "account_id": SYSTEM_TASK_ACCOUNT_ID,- "user_id": SYSTEM_TASK_USER_ID,- "target": {"account_id": acct, "user_id": "bob"},- }- )+ await queue.enqueue({+ "task_id": user_cleanup.task_id,+ "account_id": acct,+ "user_id": "alice",+ "target": {"account_id": acct, "user_id": "bob"},+ })+ await queue.enqueue({+ "task_id": late_task_id,+ "account_id": SYSTEM_TASK_ACCOUNT_ID,+ "user_id": SYSTEM_TASK_USER_ID,+ "target": {"account_id": acct, "user_id": "bob"},+ })skipped = await _wait_for_task(admin_client, late_task_id)assert skipped["status"] == "completed"assert skipped["result"] == {"deleted": not recreated, "stale": True}@@ -3129,10 +3195,7 @@ async def test_trusted_mode_admin_can_register_user_in_own_account(headers=trusted_headers(account=acct, user="alice", include_api_key=True),)assert users.status_code == 200- assert all(- user["api_key_available"] is False- for user in users.json()["result"]- )+ assert all(user["api_key_available"] is False for user in users.json()["result"])rotated = await trusted_admin_client.post(f"/api/v1/admin/accounts/{acct}/users/bob/key",@@ -3449,12 +3512,10 @@ async def test_server_restart_requires_root_and_valid_revision(server_overrides = {"host": cli_host}lightweight_admin_app.state.server_config_overrides = server_overridespath.write_text(- json.dumps(- {- "server": server,- "vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "test-key"},- }- )+ json.dumps({+ "server": server,+ "vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "test-key"},+ }))monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file", path)monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file_content", path.read_bytes())diff --git a/tests/server/test_auth.py b/tests/server/test_auth.pyindex 06c941a65..dfe92bb2b 100644--- a/tests/server/test_auth.py+++ b/tests/server/test_auth.py@@ -16,6 +16,7 @@ from starlette.requests import Requestfrom openviking.server.api_keys import APIKeyManagerfrom openviking.server.app import create_appfrom openviking.server.auth import get_request_context, resolve_identity+from openviking.server.auth.plugins.api_key import _api_key_root_can_access_pathfrom openviking.server.auth.registry import get_registryfrom openviking.server.config import ServerConfig, _is_localhost, validate_server_configfrom openviking.server.dependencies import set_service@@ -81,6 +82,20 @@ def _set_fake_task_tracker():ROOT_KEY = "root-secret-key-for-testing-only-1234567890abcdef"+ "path",+ [+ "/api/v1/system/backend/sync-status",+ "/api/v1/system/backend/sync-retry",+ "/api/v1/system/sync/viking://resources",+ "/api/v1/system/sync/viking://resources/retry",+ ],+)+def test_root_key_allowlist_rejects_removed_sync_paths(path: str):+ """Removed sync paths must stay outside the API-key root allowlist."""+ assert not _api_key_root_can_access_path(path)++def _make_request(path: str,headers: dict[str, str] | None = None,@@ -390,46 +405,6 @@ async def test_auth_on_multiple_endpoints(auth_client: httpx.AsyncClient):assert tenant_resp.json()["error"]["code"] == "PERMISSION_DENIED"-async def test_admin_sync_route_accepts_root_key(auth_client: httpx.AsyncClient, auth_service):- """ROOT keys should be allowed to call the system sync admin route."""- calls: list[str] = []-- async def _fake_system_sync_status(uri: str, ctx):- calls.append(uri)- return {"path": uri, "entry_count": 1}-- auth_service.fs.system_sync_status = _fake_system_sync_status-- resp = await auth_client.get(- "/api/v1/system/sync/viking://resources",- headers={"X-API-Key": ROOT_KEY},- )- assert resp.status_code == 200- assert resp.json()["result"] == {"path": "viking://resources", "entry_count": 1}- assert calls == ["viking://resources"]---async def test_admin_sync_route_rejects_user_key(auth_client: httpx.AsyncClient, user_key: str):- """Regular user keys must not access the system sync admin route."""- resp = await auth_client.get(- "/api/v1/system/sync/viking://resources",- headers={"X-API-Key": user_key},- )- assert resp.status_code == 403- assert resp.json()["error"]["code"] == "PERMISSION_DENIED"-- tenant_resp = await auth_client.get(- "/api/v1/fs/ls?uri=viking://",- headers={- "X-API-Key": ROOT_KEY,- "X-OpenViking-Account": "default",- "X-OpenViking-User": "default",- },- )- assert tenant_resp.status_code == 403- assert tenant_resp.json()["error"]["code"] == "PERMISSION_DENIED"--async def test_task_endpoints_require_auth():"""Task endpoints must reject unauthenticated callers before lookup/filtering."""set_task_tracker(None)@@ -1113,6 +1088,45 @@ async def test_invalid_trusted_identity_does_not_block_later_registration(await plugin.shutdown()+async def test_trusted_identity_registration_skips_identity_loaded_from_registry(auth_service):+ """A restarted plugin must not queue an identity already loaded from storage."""+ from openviking.server.auth.plugins import TrustedAuthPlugin++ config = ServerConfig(auth_mode="trusted", root_api_key=ROOT_KEY)+ app = FastAPI()+ app.state.config = config+ first_plugin = TrustedAuthPlugin()+ await first_plugin.initialize(app, auth_service, config)+ try:+ await first_plugin._api_key_manager.ensure_trusted_identities({"acme": {"alice"}})+ finally:+ await first_plugin.shutdown()++ restarted_app = FastAPI()+ restarted_app.state.config = config+ restarted_plugin = TrustedAuthPlugin()+ await restarted_plugin.initialize(restarted_app, auth_service, config)+ try:+ request = _make_request(+ "/api/v1/resources",+ headers={+ "X-OpenViking-Account": "acme",+ "X-OpenViking-User": "alice",+ },+ auth_enabled=False,+ auth_mode="trusted",+ )+ request.app.state.config = config++ await restarted_plugin.resolve_identity(+ request, x_openviking_account="acme", x_openviking_user="alice"+ )++ assert restarted_plugin._pending == {}+ finally:+ await restarted_plugin.shutdown()++@pytest.mark.parametrize("interval,path",[(0, "/api/v1/resources"), (0, "/api/v1/admin/accounts"), (300, "/api/v1/admin/accounts")],@@ -1210,7 +1224,14 @@ async def test_trusted_identity_registration_retries_failed_batch_without_exceedawait plugin.shutdown()-def test_trusted_identity_registration_config_rejects_invalid_limits():+def test_trusted_identity_registration_config_allows_disabling():+ """A zero interval disables registration; negative values remain invalid."""+ assert (+ ServerConfig(+ trusted_identity_flush_interval_seconds=0+ ).trusted_identity_flush_interval_seconds+ == 0+ )with pytest.raises(ValueError):ServerConfig(trusted_identity_flush_interval_seconds=-1)with pytest.raises(ValueError):diff --git a/tests/server/test_server_health.py b/tests/server/test_server_health.pyindex d0e78b960..d02d4a7c2 100644--- a/tests/server/test_server_health.py+++ b/tests/server/test_server_health.py@@ -150,82 +150,24 @@ async def test_system_status(client: httpx.AsyncClient):assert body["result"]["initialized"] is True-async def test_backend_sync_status_endpoint(client: httpx.AsyncClient, service):- calls: list[str] = []-- async def _fake_system_sync_status(uri: str, ctx):- calls.append(uri)- assert ctx is not None- return {"path": uri, "entry_count": 1}-- service.fs.system_sync_status = _fake_system_sync_status-- resp = await client.post(- "/api/v1/system/backend/sync-status",- json={"uri": "viking://resources"},- )- assert resp.status_code == 200- body = resp.json()- assert body["status"] == "ok"- assert body["result"] == {"path": "viking://resources", "entry_count": 1}- assert calls == ["viking://resources"]---async def test_backend_sync_retry_endpoint(client: httpx.AsyncClient, service):- calls: list[str] = []-- async def _fake_system_sync_retry(uri: str, ctx):- calls.append(uri)- assert ctx is not None- return {"path": uri, "retried": 2, "failed": 0}-- service.fs.system_sync_retry = _fake_system_sync_retry-- resp = await client.post(- "/api/v1/system/backend/sync-retry",- json={"uri": "viking://resources"},- )- assert resp.status_code == 200- body = resp.json()- assert body["status"] == "ok"- assert body["result"] == {"path": "viking://resources", "retried": 2, "failed": 0}- assert calls == ["viking://resources"]---async def test_admin_sync_status_route(client: httpx.AsyncClient, service):- calls: list[str] = []-- async def _fake_system_sync_status(uri: str, ctx):- calls.append(uri)- assert ctx is not None- return {"path": uri, "entry_count": 3}-- service.fs.system_sync_status = _fake_system_sync_status-- resp = await client.get("/api/v1/system/sync/viking://resources")- assert resp.status_code == 200- body = resp.json()- assert body["status"] == "ok"- assert body["result"] == {"path": "viking://resources", "entry_count": 3}- assert calls == ["viking://resources"]---async def test_admin_sync_retry_route(client: httpx.AsyncClient, service):- calls: list[str] = []-- async def _fake_system_sync_retry(uri: str, ctx):- calls.append(uri)- assert ctx is not None- return {"path": uri, "retried": 4, "failed": 1}-- service.fs.system_sync_retry = _fake_system_sync_retry-- resp = await client.post("/api/v1/system/sync/viking://resources/retry")- assert resp.status_code == 200- body = resp.json()- assert body["status"] == "ok"- assert body["result"] == {"path": "viking://resources", "retried": 4, "failed": 1}- assert calls == ["viking://resources"]+ ("method", "path", "json"),+ [+ ("POST", "/api/v1/system/backend/sync-status", {"uri": "viking://resources"}),+ ("POST", "/api/v1/system/backend/sync-retry", {"uri": "viking://resources"}),+ ("GET", "/api/v1/system/sync/viking://resources", None),+ ("POST", "/api/v1/system/sync/viking://resources/retry", None),+ ],+)+async def test_removed_sync_routes_return_404(+ client: httpx.AsyncClient,+ method: str,+ path: str,+ json: dict | None,+):+ """Removed V1 sync routes must stay unregistered."""+ resp = await client.request(method, path, json=json)+ assert resp.status_code == 404async def test_process_time_header(client: httpx.AsyncClient):@@ -253,15 +195,16 @@ async def test_404_for_unknown_route(client: httpx.AsyncClient):async def test_lifespan_shutdown_ignores_cancelled_service_close(monkeypatch):class _Service:+ def __init__(self):+ self.viking_fs = object()+ self._queue_manager = None+async def initialize(self):passasync def apply_agent_evolution_config(self):pass- viking_fs = None- _queue_manager = None-async def close(self):raise asyncio.CancelledError("shutdown")@@ -323,9 +266,6 @@ async def test_ready_returns_200_after_initialized(monkeypatch):async def ls(self, path, ctx=None):return []- async def system_sync_status(self, uri, ctx=None):- return {"path": uri, "entry_count": 0}-def _get_vector_store(self):class MockVectorStore:async def health_check(self):@@ -344,6 +284,10 @@ async def test_ready_returns_200_after_initialized(monkeypatch):"openviking_cli.utils.ollama.detect_ollama_in_config",lambda config: (False, None, None),)+ monkeypatch.setattr(+ "openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance",+ lambda: SimpleNamespace(embedding=SimpleNamespace(get_embedder=lambda: None)),+ )app = create_app(config=ServerConfig(), service=service)transport = httpx.ASGITransport(app=app)@@ -354,7 +298,7 @@ async def test_ready_returns_200_after_initialized(monkeypatch):assert body["status"] == "ready"assert body["checks"]["agfs"]["status"] == "ok"assert body["checks"]["agfs"]["checks"]["filesystem"] == "ok"- assert body["checks"]["agfs"]["checks"]["multiwrite_sync"] == "ok"+ assert body["checks"]["agfs"]["checks"] == {"filesystem": "ok"}async def test_slow_init_does_not_block_health(monkeypatch):@@ -386,6 +330,7 @@ async def test_initialize_runtime_state_loads_api_key_manager(monkeypatch):def __init__(self):self._initialized = Falseself.viking_fs = object()+ self._queue_manager = Noneasync def initialize(self):self._initialized = Truediff --git a/tests/test_config_loader.py b/tests/test_config_loader.pyindex 7e8484014..08444234c 100644--- a/tests/test_config_loader.py+++ b/tests/test_config_loader.py@@ -256,6 +256,14 @@ def test_generic_code_hosting_domains_include_supported_platforms():]+def test_example_config_loads_as_runtime_config():+ example_path = Path(__file__).resolve().parents[1] / "examples" / "ov.conf.example"++ config = OpenVikingConfig.from_dict(load_json_config(example_path))++ assert config.storage.agfs.backend == "local"++def test_example_code_hosting_domains_match_runtime_defaults():example_path = Path(__file__).resolve().parents[1] / "examples" / "ov.conf.example"example_text = example_path.read_text(encoding="utf-8")