Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feature(storage): support multi write storage (#2466)
baojun-zhang7ee1481e1dc1
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rsindex 288db767a..70fdd09a9 100644--- a/crates/ov_cli/src/client.rs+++ b/crates/ov_cli/src/client.rs@@ -234,6 +234,20 @@ impl HttpClient {self.post("/api/v1/system/consistency", &body).await}+ pub async fn backend_sync_status(&self, uri: &str) -> Result<serde_json::Value> {+ let body = serde_json::json!({+ "uri": uri,+ });+ self.post("/api/v1/system/backend/sync-status", &body).await+ }++ pub async fn backend_sync_retry(&self, uri: &str) -> Result<serde_json::Value> {+ let body = serde_json::json!({+ "uri": uri,+ });+ self.post("/api/v1/system/backend/sync-retry", &body).await+ }+/// Download file as raw bytespub async fn get_bytes(&self, uri: &str) -> Result<Vec<u8>> {let url = format!("{}/api/v1/content/download", self.base.base_url);diff --git a/crates/ov_cli/src/commands/system.rs b/crates/ov_cli/src/commands/system.rsindex 037cfae04..5ad24fff8 100644--- a/crates/ov_cli/src/commands/system.rs+++ b/crates/ov_cli/src/commands/system.rs@@ -81,6 +81,28 @@ pub async fn consistency(Ok(())}+pub async fn backend_sync_status(+ client: &HttpClient,+ uri: &str,+ output_format: OutputFormat,+ compact: bool,+) -> Result<()> {+ let response: serde_json::Value = client.backend_sync_status(uri).await?;+ output_success(&response, output_format, compact);+ Ok(())+}++pub async fn backend_sync_retry(+ client: &HttpClient,+ uri: &str,+ output_format: OutputFormat,+ compact: bool,+) -> Result<()> {+ let response: serde_json::Value = client.backend_sync_retry(uri).await?;+ output_success(&response, output_format, compact);+ Ok(())+}+fn output_consistency_table(response: &serde_json::Value, compact: bool) {let summary = json!({"ok": response.get("ok").and_then(|v| v.as_bool()).unwrap_or(false),diff --git a/crates/ov_cli/src/error_ui.rs b/crates/ov_cli/src/error_ui.rsindex 97260c046..ec5752f0a 100644--- a/crates/ov_cli/src/error_ui.rs+++ b/crates/ov_cli/src/error_ui.rs@@ -437,6 +437,12 @@ fn system_help_command(program: &str, tokens: &[&str]) -> String {Some("init-key") => format!("{program} system crypto init-key --help"),_ => format!("{program} system crypto --help"),},+ Some("backend") => match tokens.get(3).copied() {+ Some("sync-status" | "sync-retry") => {+ format!("{program} system backend {} --help", tokens[3])+ }+ _ => format!("{program} system backend --help"),+ },Some("wait" | "status" | "health" | "consistency") => {format!("{program} system {} --help", tokens[2])}diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rsindex e89e7e17a..f1e9d1039 100644--- a/crates/ov_cli/src/handlers.rs+++ b/crates/ov_cli/src/handlers.rs@@ -224,7 +224,7 @@ pub async fn handle_restore(.await}-use crate::SystemCommands;+use crate::{SystemBackendCommands, SystemCommands};pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> {let client = ctx.get_client();@@ -249,6 +249,16 @@ pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> {commands::system::consistency(&client, &uri, ctx.output_format, ctx.compact).await}SystemCommands::Crypto { action } => commands::crypto::handle_crypto(action).await,+ SystemCommands::Backend { action } => match action {+ SystemBackendCommands::SyncStatus { uri } => {+ commands::system::backend_sync_status(&client, &uri, ctx.output_format, ctx.compact)+ .await+ }+ SystemBackendCommands::SyncRetry { uri } => {+ commands::system::backend_sync_retry(&client, &uri, ctx.output_format, ctx.compact)+ .await+ }+ },}}diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rsindex 8b78eb31c..eb420daba 100644--- a/crates/ov_cli/src/help_ui.rs+++ b/crates/ov_cli/src/help_ui.rs@@ -2096,7 +2096,7 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[},CommandHelpSpec {path: &["system"],- purpose: "Run server utility, health, consistency, and crypto commands.",+ purpose: "Run server utility, health, consistency, backend sync, and crypto commands.",usage: "ov system <subcommand>",examples: &[HelpItem {@@ -2107,6 +2107,10 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[label: "ov system consistency viking://projects/acme",description: "Check filesystem/vector consistency.",},+ HelpItem {+ label: "ov system backend sync-status viking://projects/acme",+ description: "Inspect multi-write backend sync lag.",+ },],arguments: &[],common_options: &[],@@ -2120,6 +2124,10 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[label: "consistency <uri>",description: "Check subtree consistency.",},+ HelpItem {+ label: "backend <subcommand>",+ description: "Inspect or repair multi-write backend sync state.",+ },HelpItem {label: "crypto <subcommand>",description: "Key management commands.",@@ -2130,6 +2138,38 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[description: "Use the standard status view.",}],},+ CommandHelpSpec {+ path: &["system", "backend"],+ purpose: "Inspect and repair multi-write backend sync state.",+ usage: "ov system backend <subcommand>",+ examples: &[+ HelpItem {+ label: "ov system backend sync-status viking://resources",+ description: "Show pending and acknowledged backend sync state.",+ },+ HelpItem {+ label: "ov system backend sync-retry viking://resources",+ description: "Retry lagging backend sync targets.",+ },+ ],+ arguments: &[],+ common_options: &[],+ advanced_options: &[],+ subcommands: &[+ HelpItem {+ label: "sync-status <uri>",+ description: "Show multi-write backend sync status for a subtree.",+ },+ HelpItem {+ label: "sync-retry <uri>",+ description: "Retry pending multi-write backend sync work.",+ },+ ],+ next_steps: &[HelpItem {+ label: "ov system backend sync-status <uri>",+ description: "Inspect the subtree again after retry.",+ }],+ },CommandHelpSpec {path: &["reindex"],purpose: "Reindex semantic/vector artifacts for a URI.",diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rsindex e84c4d218..a24d7536c 100644--- a/crates/ov_cli/src/main.rs+++ b/crates/ov_cli/src/main.rs@@ -745,6 +745,27 @@ enum SystemCommands {#[command(subcommand)]action: commands::crypto::CryptoCommands,},+ /// Backend sync inspection and repair commands+ Backend {+ #[command(subcommand)]+ action: SystemBackendCommands,+ },+}++#[derive(Subcommand)]+enum SystemBackendCommands {+ /// Show multi-write backend sync status for a URI subtree+ #[command(name = "sync-status")]+ SyncStatus {+ /// Viking URI to inspect+ uri: String,+ },+ /// Retry pending multi-write backend sync work for a URI subtree+ #[command(name = "sync-retry")]+ SyncRetry {+ /// Viking URI to repair+ uri: String,+ },}#[derive(Subcommand)]@@ -1507,6 +1528,10 @@ fn known_task_command_requires_config(tokens: &[String]) -> bool {fn known_system_command_requires_config(tokens: &[String]) -> bool {match tokens.get(1).map(String::as_str) {Some("wait" | "status" | "health" | "consistency") => true,+ Some("backend") => matches!(+ tokens.get(2).map(String::as_str),+ None | Some("sync-status" | "sync-retry")+ ),Some("crypto") => matches!(tokens.get(2).map(String::as_str), None | Some("init-key")),_ => false,}@@ -2327,7 +2352,7 @@ mod tests {preprocess_privacy_args,};use crate::config::{Config, DEFAULT_CUSTOM_URL};- use crate::handlers;+ use crate::{SystemBackendCommands, SystemCommands, handlers};use crate::output::OutputFormat;use clap::{CommandFactory, Parser};use std::ffi::OsString;@@ -2372,6 +2397,23 @@ mod tests {}}+ #[test]+ fn cli_parses_system_backend_sync_status() {+ let cli = Cli::try_parse_from(["ov", "system", "backend", "sync-status", "viking://a"])+ .expect("system backend sync-status should parse");++ match cli.command {+ Commands::System { action } => match action {+ SystemCommands::Backend { action } => match action {+ SystemBackendCommands::SyncStatus { uri } => assert_eq!(uri, "viking://a"),+ _ => panic!("expected backend sync-status command"),+ },+ _ => panic!("expected system backend command"),+ },+ _ => panic!("expected system command"),+ }+ }+#[test]fn server_commands_require_existing_cli_config() {let cli = Cli::try_parse_from(["ov", "ls"]).expect("ls should parse");@@ -2510,6 +2552,7 @@ mod tests {&["ov", "config", "validate"],&["ov", "config", "show"],&["ov", "system", "consistency"],+ &["ov", "system", "backend", "sync-status"],&["ov", "admin", "list-accounts"],] {assert!(@@ -2588,6 +2631,9 @@ mod tests {&["ov", "system", "status"],&["ov", "system", "health"],&["ov", "system", "consistency"],+ &["ov", "system", "backend"],+ &["ov", "system", "backend", "sync-status"],+ &["ov", "system", "backend", "sync-retry"],&["ov", "system", "crypto"],&["ov", "system", "crypto", "init-key"],&["ov", "session", "new"],@@ -2970,6 +3016,10 @@ mod tests {vec!["ov", "system", "consistency", "help"],"ov system consistency --help",),+ (+ vec!["ov", "system", "backend", "sync-status", "help"],+ "ov system backend sync-status --help",+ ),] {let misuse = plain_help_misuse(&os_args(&args)).unwrap_or_else(|| panic!("{args:?} should be plain-help misuse"));diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rsindex 02beed13e..5f580485c 100644--- a/crates/ragfs-python/src/lib.rs+++ b/crates/ragfs-python/src/lib.rs@@ -12,9 +12,10 @@ use std::future::Future;use std::sync::Arc;use std::time::UNIX_EPOCH;+use ragfs::core::builder::EncryptionConfig;use ragfs::core::{- build_default_stack, ConfigValue, EncryptionConfig, FileInfo, FileSystem, FilesystemStats,- FsContext, FsContextInner, FsOperation, GrepResult, MountableFS, OperationStats, PluginConfig,+ build_default_stack, ConfigValue, FileInfo, FileSystem, FilesystemStats, FsContext,+ FsContextInner, FsOperation, GrepResult, MountableFS, OperationStats, PluginConfig,RagfsConfig, TreeEntry, WriteFlag, FS_CTX,};@@ -72,6 +73,8 @@ fn to_py_err(e: ragfs::core::Error) -> PyErr {ragfs::core::Error::Serialization(_) => new_py_err("AGFSSerializationError", msg),ragfs::core::Error::Network(_) => new_py_err("AGFSNetworkError", msg),ragfs::core::Error::Timeout(_) => new_py_err("AGFSTimeoutError", msg),+ ragfs::core::Error::SyncWriteQuorum { .. } => new_py_err("AGFSInternalError", msg),+ ragfs::core::Error::ContextMissing(_) => new_py_err("AGFSInternalError", msg),ragfs::core::Error::Internal(_) => new_py_err("AGFSInternalError", msg),}}@@ -232,25 +235,86 @@ fn days_to_ymd(days: u64) -> (u64, u64, u64) {(y, m, d)}-/// Convert a Python dict to HashMap<String, ConfigValue>+/// Convert a Python dict to HashMap<String, ConfigValue>, with recursive nesting support.fn py_dict_to_config(dict: &Bound<'_, PyDict>) -> PyResult<HashMap<String, ConfigValue>> {let mut params = HashMap::new();for (k, v) in dict.iter() {let key: String = k.extract()?;- let value = if let Ok(s) = v.extract::<String>() {- ConfigValue::String(s)- } else if let Ok(b) = v.extract::<bool>() {- ConfigValue::Bool(b)- } else if let Ok(i) = v.extract::<i64>() {- ConfigValue::Int(i)- } else {- ConfigValue::String(v.str()?.to_string())- };+ let value = py_any_to_config_value(&v)?;params.insert(key, value);}Ok(params)}+/// Convert a single Python value to ConfigValue, recursing into dicts/lists.+fn py_any_to_config_value(v: &Bound<'_, PyAny>) -> PyResult<ConfigValue> {+ if let Ok(s) = v.extract::<String>() {+ Ok(ConfigValue::String(s))+ } else if let Ok(b) = v.extract::<bool>() {+ Ok(ConfigValue::Bool(b))+ } else if let Ok(i) = v.extract::<i64>() {+ Ok(ConfigValue::Int(i))+ } else if let Ok(list) = v.downcast::<PyList>() {+ let mut strings = Vec::new();+ let mut has_nested = false;+ for item in list.iter() {+ if let Ok(s) = item.extract::<String>() {+ strings.push(s);+ } else {+ has_nested = true;+ break;+ }+ }+ if has_nested || strings.is_empty() {+ // Convert to serde_json::Value for complex lists+ let json_val = py_to_json_value(v)?;+ Ok(ConfigValue::Json(json_val))+ } else {+ Ok(ConfigValue::StringList(strings))+ }+ } else if v.is_instance_of::<PyDict>() {+ let json_val = py_to_json_value(v)?;+ Ok(ConfigValue::Json(json_val))+ } else {+ // Fallback: try string representation+ Ok(ConfigValue::String(v.str()?.to_string()))+ }+}++/// Convert a Python value to serde_json::Value recursively.+fn py_to_json_value(v: &Bound<'_, PyAny>) -> PyResult<serde_json::Value> {+ if v.is_none() {+ Ok(serde_json::Value::Null)+ } else if let Ok(b) = v.extract::<bool>() {+ Ok(serde_json::Value::Bool(b))+ } else if let Ok(i) = v.extract::<i64>() {+ Ok(serde_json::Value::Number(i.into()))+ } else if let Ok(f) = v.extract::<f64>() {+ if let Some(n) = serde_json::Number::from_f64(f) {+ Ok(serde_json::Value::Number(n))+ } else {+ Ok(serde_json::Value::String(f.to_string()))+ }+ } else if let Ok(s) = v.extract::<String>() {+ Ok(serde_json::Value::String(s))+ } else if let Ok(list) = v.downcast::<PyList>() {+ let mut arr = Vec::new();+ for item in list.iter() {+ arr.push(py_to_json_value(&item)?);+ }+ Ok(serde_json::Value::Array(arr))+ } else if let Ok(dict) = v.downcast::<PyDict>() {+ let mut map = serde_json::Map::new();+ for (k, val) in dict.iter() {+ let key: String = k.extract()?;+ map.insert(key, py_to_json_value(&val)?);+ }+ Ok(serde_json::Value::Object(map))+ } else {+ Ok(serde_json::Value::String(v.str()?.to_string()))+ }+}+/// Build an immutable `FsContext` from an optional Python `{"account_id": str}` dict.////// A context is always constructed (even when the dict is absent or lacks `account_id`), so the@@ -462,42 +526,6 @@ impl RAGFSBindingClient {self.read(py, path, offset, size, stream, ctx)}- /// Read raw bytes straight from the mount layer, bypassing the encryption layer.- ///- /// Used by cp / persist whole-blob copies (ciphertext is moved verbatim, never decrypted).- #[pyo3(signature = (path, offset=0, size=-1))]- fn read_raw(- &self,- py: Python<'_>,- path: String,- offset: i64,- size: i64,- ) -> PyResult<Py<PyAny>> {- let mountable = self.mountable.clone();- let off = if offset < 0 { 0u64 } else { offset as u64 };- let sz = if size < 0 { 0u64 } else { size as u64 };- let data = py_detach_blocking(py, move || {- self.rt- .block_on(async move { mountable.read(&path, off, sz).await })- })- .map_err(to_py_err)?;- Python::attach(|py| Ok(PyBytes::new(py, &data).into()))- }-- /// Write raw bytes straight to the mount layer, bypassing the encryption layer.- ///- /// Counterpart to `read_raw` for cp / persist whole-blob copies.- fn write_raw(&self, py: Python<'_>, path: String, data: Vec<u8>) -> PyResult<String> {- let mountable = self.mountable.clone();- let len = data.len();- py_detach_blocking(py, move || {- self.rt- .block_on(async move { mountable.write(&path, &data, 0, WriteFlag::Create).await })- })- .map_err(to_py_err)?;- Ok(format!("Written {} bytes", len))- }-/// Write data to file.////// Args:@@ -660,6 +688,28 @@ impl RAGFSBindingClient {Ok(m)}+ /// Attempt a same-mount verbatim copy and report whether the fast-path was used.+ #[pyo3(signature = (src_path, dst_path, ctx=None))]+ fn copy_within_mount(+ &self,+ py: Python<'_>,+ src_path: String,+ dst_path: String,+ ctx: Option<HashMap<String, String>>,+ ) -> PyResult<HashMap<String, bool>> {+ let fs_ctx = build_fs_context(ctx);+ let mountable = self.mountable.clone();+ let performed = self+ .run_scoped(py, fs_ctx, move || async move {+ mountable.copy_within_mount(&src_path, &dst_path).await+ })+ .map_err(to_py_err)?;++ let mut result = HashMap::new();+ result.insert("performed".to_string(), performed);+ Ok(result)+ }+/// Change file permissions.#[pyo3(signature = (path, mode, ctx=None))]fn chmod(@@ -737,7 +787,11 @@ impl RAGFSBindingClient {/// Args:/// fstype: Filesystem type (e.g., "memfs", "sqlfs", "kvfs", "queuefs")/// path: Mount path- /// config: Plugin configuration as dict+ /// config: Plugin configuration as dict. For multi-write, include:+ /// - "backups": nested dict matching BackendsConfig schema+ /// - "server_encryption_enabled": bool+ /// - "primary_encryption_enabled": bool+ /// - "primary_redirects": list of redirect policy dicts#[pyo3(signature = (fstype, path, config=None))]fn mount(&self,@@ -750,12 +804,8 @@ impl RAGFSBindingClient {Some(dict) => py_dict_to_config(dict)?,None => HashMap::new(),};-- let plugin_config = PluginConfig {- name: fstype.clone(),- mount_path: path.clone(),- params,- };+ let plugin_config = PluginConfig::from_raw_parts(fstype.clone(), path.clone(), params)+ .map_err(to_py_err)?;let mountable = self.mountable.clone();py_detach_blocking(py, move || {@@ -927,6 +977,58 @@ impl RAGFSBindingClient {})}+ /// Query multi-write sync status under a file or directory path.+ ///+ /// Args:+ /// path: Absolute AGFS path rooted at the mounted namespace+ /// ctx: Optional FsContext dict (e.g. {"account_id": ...})+ ///+ /// Returns:+ /// A JSON-like dict describing pending and acknowledged backup sync state.+ #[pyo3(signature = (path, ctx=None))]+ fn system_sync_status(+ &self,+ py: Python<'_>,+ path: String,+ ctx: Option<HashMap<String, String>>,+ ) -> PyResult<Py<PyAny>> {+ let fs_ctx = build_fs_context(ctx);+ let mountable = self.mountable.clone();+ let result = self+ .run_scoped(py, fs_ctx, move || async move {+ mountable.system_sync_status(&path).await+ })+ .map_err(to_py_err)?;++ Python::attach(|py| serde_json_to_py(py, &result))+ }++ /// Manually retry lagging multi-write backup sync operations under a path.+ ///+ /// Args:+ /// path: Absolute AGFS path rooted at the mounted namespace+ /// ctx: Optional FsContext dict (e.g. {"account_id": ...})+ ///+ /// Returns:+ /// A JSON-like dict summarizing retry results per target backend.+ #[pyo3(signature = (path, ctx=None))]+ fn system_sync_retry(+ &self,+ py: Python<'_>,+ path: String,+ ctx: Option<HashMap<String, String>>,+ ) -> PyResult<Py<PyAny>> {+ let fs_ctx = build_fs_context(ctx);+ let mountable = self.mountable.clone();+ let result = self+ .run_scoped(py, fs_ctx, move || async move {+ mountable.system_sync_retry(&path).await+ })+ .map_err(to_py_err)?;++ Python::attach(|py| serde_json_to_py(py, &result))+ }+/// Calculate file digest (not yet implemented in ragfs).#[pyo3(signature = (path, algorithm="xxh3"))]fn digest(&self, path: String, algorithm: &str) -> PyResult<HashMap<String, String>> {@@ -1025,7 +1127,9 @@ mod tests {Python::attach(|py| {let ty = py.get_type::<RAGFSBindingClient>();let kwargs = PyDict::new(py);- kwargs.set_item("config_path", "/tmp/legacy-ov.conf").unwrap();+ kwargs+ .set_item("config_path", "/tmp/legacy-ov.conf")+ .unwrap();let instance = ty.call((), Some(&kwargs));assert!(instance.is_ok());});diff --git a/crates/ragfs/src/core/builder.rs b/crates/ragfs/src/core/builder.rsindex 88ae2c063..da0c3e7c0 100644--- a/crates/ragfs/src/core/builder.rs+++ b/crates/ragfs/src/core/builder.rs@@ -1,12 +1,14 @@//! Standard RAGFS stack builder.//!//! Centralizes "register all built-in plugins + assemble the wrapper stack" so the binding has a-//! single construction path. Whether the encryption layer is present is decided *here, at-//! construction time*, by whether `RagfsConfig::encryption` is set (see §2.4/§2.6 of the design).+//! single construction path. Encryption is now applied per-backend inside `MountableFS::mount()`+//! (single backend) or `MountableFS::build_multi_write_fs()` (multi-write), so the global+//! `EncryptionWrappedFS` wrapper is removed from this builder.+//!+//! The top layer is always `StatsWrappedFS` so end-to-end timing (including crypto) is captured.use std::sync::Arc;-use super::encryption_wrapper::EncryptionWrappedFS;use super::filesystem::FileSystem;use super::mountable::MountableFS;use super::stats_wrapper::StatsWrappedFS;@@ -23,7 +25,7 @@ use crate::plugins::{/// `build_default_stack`'s signature.#[derive(Default)]pub struct RagfsConfig {- /// Encryption section: `None` → plaintext stack; `Some` → wrap an `EncryptionWrappedFS`.+ /// Encryption section: `None` → plaintext stack; `Some` → per-backend encryption wrapping.pub encryption: Option<EncryptionConfig>,}@@ -39,29 +41,29 @@ pub struct EncryptionConfig {pub struct RagfsStack {/// Mount manager (mount/unmount/list/stats/register_plugin live here).pub mountable: Arc<MountableFS>,- /// Data entry point: `Stats(Encryption(Mountable))` or `Stats(Mountable)`.+ /// Data entry point: `Stats(Mountable)` — encryption is per-backend inside mount.pub top: Arc<dyn FileSystem>,}/// Build the standard RAGFS stack.///-/// `config.encryption == None` → no encryption layer (plaintext); `Some` → insert-/// `EncryptionWrappedFS` in the middle. The top is always `StatsWrappedFS` so end-to-end timing-/// (including crypto) is captured.+/// Encryption config is forwarded to `MountableFS` so it can wrap individual backends+/// with `EncryptionWrappedFS` during `mount()`. The top is always `StatsWrappedFS`.pub async fn build_default_stack(config: RagfsConfig) -> RagfsStack {let mountable = Arc::new(MountableFS::new());register_builtin_plugins(&mountable).await;- let data_fs: Arc<dyn FileSystem> = match config.encryption {- Some(enc) => Arc::new(EncryptionWrappedFS::new(- mountable.clone() as Arc<dyn FileSystem>,- enc.root_key,- enc.provider_type,- )),- None => mountable.clone() as Arc<dyn FileSystem>,- };+ // Forward encryption config to MountableFS for per-backend wrapping.+ if let Some(enc) = &config.encryption {+ mountable+ .set_encryption_config(Some(enc.root_key), Some(enc.provider_type))+ .await;+ }- let top: Arc<dyn FileSystem> = Arc::new(StatsWrappedFS::with_arc(data_fs));+ // MountableFS is the data entry point; encryption is applied inside mount().+ let top: Arc<dyn FileSystem> = Arc::new(StatsWrappedFS::with_arc(+ mountable.clone() as Arc<dyn FileSystem>+ ));RagfsStack { mountable, top }}@@ -81,9 +83,11 @@ pub async fn register_builtin_plugins(fs: &MountableFS) {mod tests {use super::*;use crate::core::context::{FsContextInner, FS_CTX};- use crate::core::{PluginConfig, WriteFlag};+ use crate::core::{ConfigValue, PluginConfig, Result, WriteFlag};use crate::crypto;use std::collections::HashMap;+ use std::fs;+ use tempfile::TempDir;fn enc_config() -> RagfsConfig {RagfsConfig {@@ -94,18 +98,36 @@ mod tests {}}+ /// Create a plugin config for stack builder tests.+ fn plugin_config(+ name: &str,+ mount_path: &str,+ params: HashMap<String, ConfigValue>,+ ) -> PluginConfig {+ PluginConfig::single_backend(name, mount_path, params)+ }+async fn mount_mem(stack: &RagfsStack) {stack.mountable- .mount(PluginConfig {- name: "memfs".to_string(),- mount_path: "/mem".to_string(),- params: HashMap::new(),- })+ .mount(plugin_config("memfs", "/mem", HashMap::new())).await.unwrap();}+ /// Mount a LocalFS backend backed by a temporary directory.+ async fn mount_local(stack: &RagfsStack, mount_path: &str, local_dir: &str) -> Result<()> {+ let mut params = HashMap::new();+ params.insert(+ "local_dir".to_string(),+ ConfigValue::String(local_dir.to_string()),+ );+ stack+ .mountable+ .mount(plugin_config("localfs", mount_path, params))+ .await+ }+#[tokio::test]async fn encrypted_stack_encrypts_on_disk() {let stack = build_default_stack(enc_config()).await;@@ -122,8 +144,8 @@ mod tests {}).await;- // Underlying mountable holds ciphertext.- let raw = stack.mountable.read("/mem/f", 0, 0).await.unwrap();+ // Read raw bytes from mountable (bypasses encryption layer) to verify ciphertext.+ let raw = stack.mountable.read_raw("/mem/f", 0, 0).await.unwrap();assert!(crypto::is_encrypted(&raw));}@@ -138,7 +160,7 @@ mod tests {.write("/mem/f", b"hello", 0, WriteFlag::Create).await.unwrap();- let raw = stack.mountable.read("/mem/f", 0, 0).await.unwrap();+ let raw = stack.mountable.read_raw("/mem/f", 0, 0).await.unwrap();assert_eq!(raw, b"hello", "plaintext stack stores raw bytes");}@@ -147,11 +169,7 @@ mod tests {let stack = build_default_stack(enc_config()).await;stack.mountable- .mount(PluginConfig {- name: "queuefs".to_string(),- mount_path: "/queue".to_string(),- params: HashMap::new(),- })+ .mount(plugin_config("queuefs", "/queue", HashMap::new())).await.unwrap();@@ -172,4 +190,44 @@ mod tests {}).await;}++ #[tokio::test]+ async fn encrypted_mount_writes_backend_shape_guard() {+ let dir = TempDir::new().unwrap();+ let stack = build_default_stack(enc_config()).await;++ mount_local(&stack, "/local", dir.path().to_str().unwrap())+ .await+ .unwrap();++ let manifest = fs::read(dir.path().join("backend_meta.json")).unwrap();+ assert!(crypto::is_encrypted(&manifest));+ let account_key = crypto::hkdf_sha256(&[4u8; 32], b"_system");+ let (_provider, enc_key, key_iv, data_iv, ciphertext) =+ crypto::parse_envelope(&manifest).unwrap();+ let file_key =+ crypto::aes_gcm_decrypt(&account_key, key_iv.try_into().unwrap(), enc_key).unwrap();+ let file_key: [u8; 32] = file_key.as_slice().try_into().unwrap();+ let plaintext =+ crypto::aes_gcm_decrypt(&file_key, data_iv.try_into().unwrap(), ciphertext).unwrap();+ assert!(plaintext.is_empty());+ }++ #[tokio::test]+ async fn encrypted_mount_rejects_legacy_plaintext_backend_without_manifest() {+ let dir = TempDir::new().unwrap();+ fs::write(dir.path().join("legacy.txt"), b"plaintext").unwrap();++ let stack = build_default_stack(enc_config()).await;+ let err = mount_local(&stack, "/local", dir.path().to_str().unwrap())+ .await+ .unwrap_err();++ match err {+ crate::core::errors::Error::Config(message) => {+ assert!(message.contains("backend storage shape mismatch"));+ }+ other => panic!("unexpected error: {other:?}"),+ }+ }}diff --git a/crates/ragfs/src/core/encryption_wrapper.rs b/crates/ragfs/src/core/encryption_wrapper.rsindex 1ac0b7265..18f02c44d 100644--- a/crates/ragfs/src/core/encryption_wrapper.rs+++ b/crates/ragfs/src/core/encryption_wrapper.rs@@ -14,6 +14,7 @@ use std::sync::{Arc, RwLock};use async_trait::async_trait;use crate::crypto;+use crate::shape::SHAPE_MANIFEST_PATH;use super::context::FsContextView;use super::errors::{Error, Result};@@ -35,6 +36,11 @@ pub struct EncryptionWrappedFS {}impl EncryptionWrappedFS {+ /// Get the wrapped filesystem for specialized delegation.+ pub(crate) fn inner_fs(&self) -> &Arc<dyn FileSystem> {+ &self.inner+ }+/// Construct an encryption layer over `inner`. Built only when a root key is configured.pub fn new(inner: Arc<dyn FileSystem>, root_key: [u8; 32], provider_type: u8) -> Self {Self {@@ -94,6 +100,11 @@ impl EncryptionWrappedFS {|| path.starts_with("/serverinfo/")}+ /// Return true when one path points at the backend-shape manifest.+ fn is_shape_manifest_path(path: &str) -> bool {+ path == SHAPE_MANIFEST_PATH || path.ends_with(SHAPE_MANIFEST_PATH)+ }+/// Derive the encryption account domain from a filesystem path.////// `/local/{account_id}/...` paths are tenant-scoped and use that account id. Control or@@ -256,7 +267,11 @@ impl FileSystem for EncryptionWrappedFS {}async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {- self.inner.read_dir(path).await+ let entries = self.inner.read_dir(path).await?;+ Ok(entries+ .into_iter()+ .filter(|entry| entry.name != SHAPE_MANIFEST_PATH.trim_start_matches('/'))+ .collect())}async fn stat(&self, path: &str) -> Result<FileInfo> {@@ -282,9 +297,14 @@ impl FileSystem for EncryptionWrappedFS {level_limit: Option<usize>,) -> Result<Vec<TreeEntry>> {// Metadata only — no content read, so delegate to preserve plugin-native tree optimizations.- self.inner+ let entries = self+ .inner.tree_directory(path, show_hidden, node_limit, level_limit)- .await+ .await?;+ Ok(entries+ .into_iter()+ .filter(|entry| !Self::is_shape_manifest_path(&entry.path))+ .collect())}async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {@@ -305,30 +325,21 @@ mod tests {use crate::core::PluginConfig;use crate::plugins::MemFSPlugin;+ /// Build a memfs plugin config for encryption wrapper tests.+ fn memfs_config(mount_path: &str) -> PluginConfig {+ PluginConfig::single_backend("memfs", mount_path, HashMap::new())+ }++ /// Build a memfs-backed stack mounted at the default path.async fn memfs_stack() -> Arc<MountableFS> {- let m = Arc::new(MountableFS::new());- m.register_plugin(MemFSPlugin).await;- m.mount(PluginConfig {- name: "memfs".to_string(),- mount_path: "/mem".to_string(),- params: HashMap::new(),- })- .await- .unwrap();- m+ memfs_stack_at("/mem").await}/// Build a memfs-backed stack mounted at the provided path for path-sensitive tests.async fn memfs_stack_at(mount_path: &str) -> Arc<MountableFS> {let m = Arc::new(MountableFS::new());m.register_plugin(MemFSPlugin).await;- m.mount(PluginConfig {- name: "memfs".to_string(),- mount_path: mount_path.to_string(),- params: HashMap::new(),- })- .await- .unwrap();+ m.mount(memfs_config(mount_path)).await.unwrap();m}@@ -483,7 +494,9 @@ mod tests {enc.write("/mem/a.txt", b"first", 0, WriteFlag::Create).await.unwrap();- let appended = enc.write("/mem/a.txt", b"second", 0, WriteFlag::Append).await;+ let appended = enc+ .write("/mem/a.txt", b"second", 0, WriteFlag::Append)+ .await;assert!(appended.is_err());}).await;diff --git a/crates/ragfs/src/core/errors.rs b/crates/ragfs/src/core/errors.rsindex b2f802842..f6ff185f6 100644--- a/crates/ragfs/src/core/errors.rs+++ b/crates/ragfs/src/core/errors.rs@@ -4,12 +4,23 @@//! We use `thiserror` for structured error definitions to ensure type safety//! and clear error messages.-use std::io;use serde_json;+use std::io;/// Result type alias for RAGFS operationspub type Result<T> = std::result::Result<T, Error>;+/// Structured failure detail for one backup target in synchronous multi-write fanout.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct SyncWriteFailureDetail {+ /// Logical backup backend name.+ pub backend: String,+ /// Stable error kind for programmatic inspection.+ pub kind: String,+ /// Human-readable error message including the original error display text.+ pub message: String,+}+/// Main error type for RAGFS operations#[derive(Debug, thiserror::Error)]pub enum Error {@@ -77,6 +88,25 @@ pub enum Error {#[error("operation timed out: {0}")]Timeout(String),+ /// Multi-write synchronous fanout failed to reach the required acknowledgement quorum+ #[error(+ "sync write quorum failed: {succeeded}/{attempted} backups succeeded (required {required}); failures: {failures:?}"+ )]+ SyncWriteQuorum {+ /// Number of backup targets that completed successfully.+ succeeded: usize,+ /// Minimum number of backup acknowledgements required for success.+ required: usize,+ /// Total number of backup targets that were attempted.+ attempted: usize,+ /// Structured failure details for each unsuccessful backup target.+ failures: Vec<SyncWriteFailureDetail>,+ },++ /// Required filesystem context is missing from the current task+ #[error("filesystem context missing: {0}")]+ ContextMissing(String),+/// Internal error#[error("internal error: {0}")]Internal(String),@@ -124,6 +154,41 @@ impl Error {Self::Internal(msg.into())}+ /// Create a ContextMissing error+ pub fn context_missing(msg: impl Into<String>) -> Self {+ Self::ContextMissing(msg.into())+ }++ /// Create a Timeout error+ pub fn timeout(msg: impl Into<String>) -> Self {+ Self::Timeout(msg.into())+ }++ /// Return a stable error kind name for structured reporting.+ pub fn kind_name(&self) -> &'static str {+ match self {+ Self::NotFound(_) => "not_found",+ Self::AlreadyExists(_) => "already_exists",+ Self::PermissionDenied(_) => "permission_denied",+ Self::InvalidPath(_) => "invalid_path",+ Self::NotADirectory(_) => "not_a_directory",+ Self::IsADirectory(_) => "is_a_directory",+ Self::DirectoryNotEmpty(_) => "directory_not_empty",+ Self::InvalidOperation(_) => "invalid_operation",+ Self::Io(_) => "io",+ Self::Plugin(_) => "plugin",+ Self::Config(_) => "config",+ Self::MountPointNotFound(_) => "mount_point_not_found",+ Self::MountPointExists(_) => "mount_point_exists",+ Self::Serialization(_) => "serialization",+ Self::Network(_) => "network",+ Self::Timeout(_) => "timeout",+ Self::SyncWriteQuorum { .. } => "sync_write_quorum",+ Self::ContextMissing(_) => "context_missing",+ Self::Internal(_) => "internal",+ }+ }+/// Create an InvalidOperation errorpub fn invalid_operation(msg: impl Into<String>) -> Self {Self::InvalidOperation(msg.into())diff --git a/crates/ragfs/src/core/filesystem.rs b/crates/ragfs/src/core/filesystem.rsindex be60d2b3e..121315d7c 100644--- a/crates/ragfs/src/core/filesystem.rs+++ b/crates/ragfs/src/core/filesystem.rs@@ -6,6 +6,7 @@use async_trait::async_trait;use regex::Regex;+use std::any::Any;use super::errors::{Error, Result};use super::types::{FileInfo, GrepResult, TreeEntry, WriteFlag};@@ -93,7 +94,7 @@ pub(crate) fn compile_grep_regex(pattern: &str, case_insensitive: bool) -> Resul/// All filesystem plugins must implement this trait to provide file operations./// All methods are async to support I/O-bound operations efficiently.#[async_trait]-pub trait FileSystem: Send + Sync {+pub trait FileSystem: Send + Sync + Any {/// Create an empty file at the specified path////// # Arguments@@ -786,11 +787,38 @@ mod tests {entries.iter().map(|e| e.rel_path.clone()).collect()}+ /// Run the default tree traversal for the common `/root` test case.+ async fn root_tree(+ fs: &TreeFS,+ show_hidden: bool,+ node_limit: Option<usize>,+ level_limit: Option<usize>,+ ) -> Vec<TreeEntry> {+ fs.tree_directory("/root", show_hidden, node_limit, level_limit)+ .await+ .unwrap()+ }++ /// Assert tree entry names in order.+ fn assert_tree_names(entries: &[TreeEntry], expected: &[&str]) {+ assert_eq!(tree_names(entries), strings(expected));+ }++ /// Assert tree entry relative paths in order.+ fn assert_tree_rel_paths(entries: &[TreeEntry], expected: &[&str]) {+ assert_eq!(tree_rel_paths(entries), strings(expected));+ }++ /// Convert string slices to owned strings for assertions.+ fn strings(values: &[&str]) -> Vec<String> {+ values.iter().map(|value| value.to_string()).collect()+ }+#[tokio::test]async fn test_tree_empty_dir() {let fs = TreeFS::default().with_dir_entries("/root", vec![]);- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();+ let entries = root_tree(&fs, false, None, None).await;assert!(entries.is_empty(), "empty dir returns empty vec");}@@ -802,11 +830,11 @@ mod tests {vec![("a.txt", false), ("b.txt", false), ("c.txt", false)],);- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();+ let entries = root_tree(&fs, false, None, None).await;assert_eq!(entries.len(), 3);- assert_eq!(tree_names(&entries), vec!["a.txt", "b.txt", "c.txt"]);- assert_eq!(tree_rel_paths(&entries), vec!["a.txt", "b.txt", "c.txt"]);+ assert_tree_names(&entries, &["a.txt", "b.txt", "c.txt"]);+ assert_tree_rel_paths(&entries, &["a.txt", "b.txt", "c.txt"]);}#[tokio::test]@@ -815,11 +843,11 @@ mod tests {.with_dir_entries("/root", vec![("a.txt", false), ("sub", true)]).with_dir_entries("/root/sub", vec![("b.txt", false)]);- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();+ let entries = root_tree(&fs, false, None, None).await;assert_eq!(entries.len(), 3);- assert_eq!(tree_names(&entries), vec!["a.txt", "sub", "b.txt"]);- assert_eq!(tree_rel_paths(&entries), vec!["a.txt", "sub", "sub/b.txt"]);+ assert_tree_names(&entries, &["a.txt", "sub", "b.txt"]);+ assert_tree_rel_paths(&entries, &["a.txt", "sub", "sub/b.txt"]);assert_eq!(tree_paths(&entries),vec!["/root/a.txt", "/root/sub", "/root/sub/b.txt"]@@ -844,23 +872,17 @@ mod tests {],);- let entries = fs- .tree_directory("/root", false, Some(3), None)- .await- .unwrap();+ let entries = root_tree(&fs, false, Some(3), None).await;assert_eq!(entries.len(), 3);- assert_eq!(tree_names(&entries), vec!["a.txt", "b.txt", "c.txt"]);+ assert_tree_names(&entries, &["a.txt", "b.txt", "c.txt"]);}#[tokio::test]async fn test_tree_node_limit_zero() {let fs = TreeFS::default().with_dir_entries("/root", vec![("a.txt", false)]);- let entries = fs- .tree_directory("/root", false, Some(0), None)- .await- .unwrap();+ let entries = root_tree(&fs, false, Some(0), None).await;assert!(entries.is_empty(), "node_limit=0 returns empty");}@@ -871,10 +893,7 @@ mod tests {.with_dir_entries("/root", vec![("a.txt", false), ("sub", true)]).with_dir_entries("/root/sub", vec![("b.txt", false)]);- let entries = fs- .tree_directory("/root", false, None, Some(0))- .await- .unwrap();+ let entries = root_tree(&fs, false, None, Some(0)).await;assert!(entries.is_empty(),@@ -888,13 +907,10 @@ mod tests {.with_dir_entries("/root", vec![("a.txt", false), ("sub", true)]).with_dir_entries("/root/sub", vec![("b.txt", false)]);- let entries = fs- .tree_directory("/root", false, None, Some(1))- .await- .unwrap();+ let entries = root_tree(&fs, false, None, Some(1)).await;assert_eq!(entries.len(), 2);- assert_eq!(tree_names(&entries), vec!["a.txt", "sub"]);+ assert_tree_names(&entries, &["a.txt", "sub"]);}#[tokio::test]@@ -904,7 +920,7 @@ mod tests {.with_dir_entries("/root/sub", vec![("b.txt", false), ("deep", true)]).with_dir_entries("/root/sub/deep", vec![("c.txt", false)]);- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();+ let entries = root_tree(&fs, false, None, None).await;assert_eq!(entries.len(), 5);assert!(tree_paths(&entries).contains(&"/root/sub/deep/c.txt".to_string()));@@ -917,7 +933,7 @@ mod tests {vec![("a.txt", false), (".hidden", false), ("visible", true)],);- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();+ let entries = root_tree(&fs, false, None, None).await;let names = tree_names(&entries);assert!(names.contains(&"a.txt".to_string()));@@ -931,10 +947,10 @@ mod tests {.with_dir_entries("/root", vec![(".hidden_dir", true)]).with_dir_entries("/root/.hidden_dir", vec![("secret.txt", false)]);- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();+ let entries = root_tree(&fs, false, None, None).await;assert_eq!(entries.len(), 2);- assert_eq!(tree_names(&entries), vec![".hidden_dir", "secret.txt"]);+ assert_tree_names(&entries, &[".hidden_dir", "secret.txt"]);}#[tokio::test]@@ -944,7 +960,7 @@ mod tests {vec![("a.txt", false), (".hidden", false), (".hidden_dir", true)],);- let entries = fs.tree_directory("/root", true, None, None).await.unwrap();+ let entries = root_tree(&fs, true, None, None).await;let names = tree_names(&entries);assert!(names.contains(&"a.txt".to_string()));@@ -1002,7 +1018,7 @@ mod tests {.with_dir_entries("/root/sub", vec![("b.txt", false)]).with_dir_entries("/root/.hidden_dir", vec![("secret.txt", false)]);- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();+ let entries = root_tree(&fs, false, None, None).await;assert_eq!(entries.len(), 5);let names = tree_names(&entries);diff --git a/crates/ragfs/src/core/mod.rs b/crates/ragfs/src/core/mod.rsindex cd51f2c0a..b7469ce8b 100644--- a/crates/ragfs/src/core/mod.rs+++ b/crates/ragfs/src/core/mod.rs@@ -13,24 +13,30 @@ pub mod encryption_wrapper;pub mod errors;pub mod filesystem;pub mod mountable;+pub mod multibackend_wrapper;pub mod plugin;pub mod stats;pub mod stats_wrapper;pub mod types;// Re-export commonly used types+pub use crate::multibackend::{FsContextResolver, MetaStateStore};pub use builder::{- build_default_stack, register_builtin_plugins, EncryptionConfig, RagfsConfig, RagfsStack,+ build_default_stack, register_builtin_plugins, EncryptionConfig as BuilderEncryptionConfig,+ RagfsConfig, RagfsStack,};pub use context::{FsContext, FsContextInner, FsContextView, FS_CTX};pub use encryption_wrapper::EncryptionWrappedFS;pub use errors::{Error, Result};pub use filesystem::FileSystem;pub use mountable::MountableFS;+pub use multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS, SyncMode};pub use plugin::{HealthStatus, PluginRegistry, ServicePlugin};pub use stats::{FilesystemStats, FsOperation, OperationStats, OperationTimer, StatsCollector};pub use stats_wrapper::StatsWrappedFS;pub use types::{- ConfigParameter, ConfigValue, FileInfo, GrepMatch, GrepResult, PluginConfig, TreeEntry,- WriteFlag,+ BackendItemConfig, BackendRole, BackendSyncState, BackendsConfig, ConfigParameter, ConfigValue,+ EncryptionConfig, FileInfo, GrepMatch, GrepResult, OperationItemConfig, PluginConfig,+ RedirectEntry, RedirectMeta, RedirectPolicy, SyncLogEntry, SyncLogMeta, SyncOp, SyncType,+ TreeEntry, WriteFlag,};diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rsindex 1262365f8..a1a922f69 100644--- a/crates/ragfs/src/core/mountable.rs+++ b/crates/ragfs/src/core/mountable.rs@@ -6,16 +6,27 @@use async_trait::async_trait;use radix_trie::{Trie, TrieCommon};+use serde_json::Value;use std::collections::HashMap;use std::sync::Arc;use tokio::sync::RwLock;+use tracing::warn;+use crate::multibackend::factory::build_multi_write_fs;+use crate::multibackend::types::MultiBackendBuildContext;+use crate::shape::validate::ensure_backend_shape;++/// Python-side transaction path-lock marker — hidden in all listing/grep operations.+const PATH_LOCK_FILE: &str = ".path.ovlock";++use super::encryption_wrapper::EncryptionWrappedFS;use super::errors::{Error, Result};use super::filesystem::FileSystem;+use super::multibackend_wrapper::MultiWriteWrappedFS;use super::plugin::ServicePlugin;use super::stats::{FilesystemStats, StatsCollector};use super::stats_wrapper::StatsWrappedFS;-use super::types::{FileInfo, GrepResult, PluginConfig, TreeEntry, WriteFlag};+use super::types::{BackendsConfig, FileInfo, GrepResult, PluginConfig, TreeEntry, WriteFlag};/// Information about a mounted filesystem#[derive(Clone)]@@ -26,6 +37,10 @@ struct MountInfo {/// The filesystem instance (wrapped with statistics)fs: Arc<dyn FileSystem>,+ /// The raw plugin backend before encryption wrapping (for read_raw/write_raw).+ /// None for multi-write mounts.+ raw_fs: Option<Arc<dyn FileSystem>>,+/// The statistics collector for this mountstats: Arc<StatsCollector>,@@ -33,6 +48,8 @@ struct MountInfo {plugin_name: String,}+const DEFAULT_RAW_COPY_CHUNK_SIZE: usize = 8 * 1024 * 1024;+/// MountableFS routes filesystem operations to mounted plugins////// This is the core component that allows multiple filesystem implementations@@ -44,17 +61,92 @@ pub struct MountableFS {/// Plugin registry for creating new filesystem instancesregistry: Arc<RwLock<HashMap<String, Arc<dyn ServicePlugin>>>>,++ /// Global encryption config (root_key + provider_type), shared across all mounts.+ /// When set, single-backend mounts are wrapped with EncryptionWrappedFS;+ /// multi-write mounts use it for per-backend encryption decisions.+ encryption_root_key: RwLock<Option<[u8; 32]>>,+ encryption_provider_type: RwLock<Option<u8>>,}impl MountableFS {+ /// Return the raw backend for one mounted path when raw access is supported.+ fn raw_backend_for_mount<'a>(+ mount_info: &'a MountInfo,+ op_name: &str,+ ) -> Result<&'a Arc<dyn FileSystem>> {+ if Self::as_multiwrite(&mount_info.fs).is_some() {+ return Err(Error::invalid_operation(format!(+ "{} is not supported for multi-write mounts",+ op_name+ )));+ }+ mount_info.raw_fs.as_ref().ok_or_else(|| {+ Error::internal(format!(+ "raw backend is unavailable for mount '{}'",+ mount_info.path+ ))+ })+ }++ /// Try to unwrap a mounted filesystem stack to the underlying multi-write wrapper.+ fn as_multiwrite(fs: &Arc<dyn FileSystem>) -> Option<&MultiWriteWrappedFS> {+ let any = fs.as_ref() as &dyn std::any::Any;+ if let Some(mw) = any.downcast_ref::<MultiWriteWrappedFS>() {+ return Some(mw);+ }+ if let Some(stats) = any.downcast_ref::<StatsWrappedFS>() {+ return Self::as_multiwrite(stats.inner_fs());+ }+ if let Some(enc) = any.downcast_ref::<EncryptionWrappedFS>() {+ return Self::as_multiwrite(enc.inner_fs());+ }+ None+ }++ /// Query multi-write sync status for a mounted path.+ pub async fn system_sync_status(&self, path: &str) -> Result<Value> {+ let (mount_info, rel_path) = self.find_mount(path).await?;+ let multiwrite = Self::as_multiwrite(&mount_info.fs)+ .ok_or_else(|| Error::invalid_operation("mounted filesystem is not multi-write"))?;+ multiwrite.system_sync_status(&rel_path).await+ }++ /// Manually retry pending multi-write sync operations under a mounted path.+ pub async fn system_sync_retry(&self, path: &str) -> Result<Value> {+ let (mount_info, rel_path) = self.find_mount(path).await?;+ let multiwrite = Self::as_multiwrite(&mount_info.fs)+ .ok_or_else(|| Error::invalid_operation("mounted filesystem is not multi-write"))?;+ multiwrite.system_sync_retry(&rel_path).await+ }+/// Create a new MountableFSpub fn new() -> Self {Self {mounts: Arc::new(RwLock::new(Trie::new())),registry: Arc::new(RwLock::new(HashMap::new())),+ encryption_root_key: RwLock::new(None),+ encryption_provider_type: RwLock::new(None),}}+ /// Set the global encryption configuration for per-mount encryption wrapping.+ pub async fn set_encryption_config(+ &self,+ root_key: Option<[u8; 32]>,+ provider_type: Option<u8>,+ ) {+ *self.encryption_root_key.write().await = root_key;+ *self.encryption_provider_type.write().await = provider_type;+ }++ /// Get the encryption root key and provider type.+ async fn get_encryption_config(&self) -> (Option<[u8; 32]>, Option<u8>) {+ let rk = *self.encryption_root_key.read().await;+ let pt = *self.encryption_provider_type.read().await;+ (rk, pt)+ }+/// Register a plugin////// # Arguments@@ -99,17 +191,59 @@ impl MountableFS {// Validate configurationplugin.validate(&config).await?;- // Initialize filesystem- let fs = plugin.initialize(config.clone()).await?;+ // Branch: multi-write vs single backend+ let (inner_fs, raw_fs): (Arc<dyn FileSystem>, Option<Arc<dyn FileSystem>>) = match &config+ .backups+ {+ None => {+ // Single backend: initialize plugin, optionally wrap with encryption.+ // Control plugins (queuefs, serverinfofs) are never encrypted.+ let raw = plugin.initialize(config.clone()).await?;+ let raw_arc: Arc<dyn FileSystem> = Arc::from(raw);+ let is_control_plugin = matches!(config.name.as_str(), "queuefs" | "serverinfofs");+ let (enc_root_key, enc_provider_type) = self.get_encryption_config().await;+ if !is_control_plugin {+ ensure_backend_shape(+ &raw_arc,+ &config.name,+ enc_root_key.is_some() && enc_provider_type.is_some(),+ enc_provider_type,+ enc_root_key,+ )+ .await?;+ }+ let inner: Arc<dyn FileSystem> = if is_control_plugin {+ raw_arc.clone()+ } else {+ match (enc_root_key, enc_provider_type) {+ (Some(rk), Some(pt)) => {+ Arc::new(EncryptionWrappedFS::new(raw_arc.clone(), rk, pt))+ }+ _ => raw_arc.clone(),+ }+ };+ (inner, Some(raw_arc))+ }+ Some(bc) => {+ // Multi-write: build MultiWriteWrappedFS+ warn!(+ "multi-write metadata locking is process-local only; multi-instance shared-primary mounts are not safe"+ );+ let mw = self.build_multi_write_fs(&config, bc).await?;+ let arc: Arc<dyn FileSystem> = Arc::new(mw);+ (arc, None)+ }+ };// Wrap with statistics- let wrapped_fs = StatsWrappedFS::new(fs);+ let wrapped_fs = StatsWrappedFS::with_arc(inner_fs);let stats_collector = wrapped_fs.stats_collector();// Add to mountslet mount_info = MountInfo {path: normalized_path.clone(),fs: Arc::from(Box::new(wrapped_fs) as Box<dyn FileSystem>),+ raw_fs,stats: stats_collector,plugin_name: config.name.clone(),};@@ -120,6 +254,31 @@ impl MountableFS {Ok(())}+ /// Build a MultiWriteWrappedFS from primary + backup configurations.+ ///+ /// Steps:+ /// 1. Initialize primary backend via registry, optionally wrap with EncryptionWrappedFS+ /// 2. Initialize each backup backend via registry, optionally wrap with EncryptionWrappedFS+ /// 3. Validate: unique names, primary has no operations, redirect targets exist+ /// 4. Assemble MultiWriteWrappedFS+ async fn build_multi_write_fs(+ &self,+ config: &PluginConfig,+ bc: &BackendsConfig,+ ) -> Result<MultiWriteWrappedFS> {+ let (enc_root_key, enc_provider_type) = self.get_encryption_config().await;+ build_multi_write_fs(+ &self.registry,+ config,+ bc,+ MultiBackendBuildContext {+ enc_root_key,+ enc_provider_type,+ },+ )+ .await+ }+/// Unmount a filesystem at the specified path////// # Arguments@@ -129,6 +288,17 @@ impl MountableFS {/// * `Error::MountPointNotFound` - If no filesystem is mounted at this pathpub async fn unmount(&self, path: &str) -> Result<()> {let normalized_path = normalize_path(path);+ let mount_info = {+ let mounts = self.mounts.read().await;+ mounts+ .get(&normalized_path)+ .cloned()+ .ok_or_else(|| Error::MountPointNotFound(normalized_path.clone()))?+ };++ if let Some(multiwrite) = Self::as_multiwrite(&mount_info.fs) {+ multiwrite.shutdown().await?;+ }let mut mounts = self.mounts.write().await;if mounts.remove(&normalized_path).is_none() {@@ -184,6 +354,95 @@ impl MountableFS {result}+ /// Read raw bytes from the underlying plugin backend, bypassing the encryption layer.+ ///+ /// Used by tests to verify ciphertext on disk and by cp/persist for verbatim blob copies.+ pub async fn read_raw(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {+ let (mount_info, rel_path) = self.find_mount(path).await?;+ let raw = Self::raw_backend_for_mount(&mount_info, "read_raw")?;+ raw.read(&rel_path, offset, size).await+ }++ /// Write raw bytes to the underlying plugin backend, bypassing the encryption layer.+ ///+ /// Counterpart to `read_raw` for cp/persist verbatim blob copies.+ pub async fn write_raw(&self, path: &str, data: &[u8], flags: WriteFlag) -> Result<u64> {+ let (mount_info, rel_path) = self.find_mount(path).await?;+ let raw = Self::raw_backend_for_mount(&mount_info, "write_raw")?;+ raw.write(&rel_path, data, 0, flags).await+ }++ /// Copy one file within the same mounted filesystem while preserving raw bytes when possible.+ pub async fn copy_within_mount(&self, src_path: &str, dst_path: &str) -> Result<bool> {+ let (src_mount, src_rel_path) = self.find_mount(src_path).await?;+ let (dst_mount, dst_rel_path) = self.find_mount(dst_path).await?;++ if src_mount.path != dst_mount.path {+ return Ok(false);+ }++ if normalize_path(&src_rel_path) == normalize_path(&dst_rel_path) {+ return Ok(true);+ }++ if let Some(multiwrite) = Self::as_multiwrite(&src_mount.fs) {+ return multiwrite+ .copy_within_primary(&src_rel_path, &dst_rel_path)+ .await;+ }++ let Some(raw_backend) = src_mount.raw_fs.clone() else {+ return Ok(false);+ };++ Self::copy_raw_within_mount(raw_backend, &src_rel_path, &dst_rel_path).await?;+ Ok(true)+ }++ /// Copy one file within one raw backend in bounded-size chunks.+ async fn copy_raw_within_mount(+ raw_backend: Arc<dyn FileSystem>,+ src_rel_path: &str,+ dst_rel_path: &str,+ ) -> Result<()> {+ let source_info = raw_backend.stat(src_rel_path).await?;+ if source_info.is_dir {+ return Err(Error::IsADirectory(src_rel_path.to_string()));+ }++ raw_backend.ensure_parent_dirs(dst_rel_path, 0o755).await?;++ if source_info.size == 0 {+ raw_backend+ .write(dst_rel_path, &[], 0, WriteFlag::Create)+ .await?;+ return Ok(());+ }++ let mut offset = 0u64;+ while offset < source_info.size {+ let chunk_len = (source_info.size - offset).min(DEFAULT_RAW_COPY_CHUNK_SIZE as u64);+ let chunk = raw_backend.read(src_rel_path, offset, chunk_len).await?;+ let flag = if offset == 0 {+ WriteFlag::Create+ } else {+ WriteFlag::None+ };+ raw_backend+ .write(dst_rel_path, &chunk, offset, flag)+ .await?;+ offset = offset.saturating_add(chunk.len() as u64);+ if chunk.is_empty() {+ return Err(Error::internal(format!(+ "raw backend returned empty chunk while copying '{}' -> '{}'",+ src_rel_path, dst_rel_path+ )));+ }+ }++ Ok(())+ }+/// Find the mount point for a given path////// # Arguments@@ -292,7 +551,9 @@ impl FileSystem for MountableFS {async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {let (mount_info, rel_path) = self.find_mount(path).await?;- mount_info.fs.read_dir(&rel_path).await+ let mut entries = mount_info.fs.read_dir(&rel_path).await?;+ entries.retain(|e| e.name != PATH_LOCK_FILE);+ Ok(entries)}async fn stat(&self, path: &str) -> Result<FileInfo> {@@ -360,7 +621,7 @@ impl FileSystem for MountableFS {},};- mount_info+ let mut result = mount_info.fs.grep(&rel_path,@@ -371,7 +632,17 @@ impl FileSystem for MountableFS {exclude_rel.as_deref(),level_limit,)- .await+ .await?;++ // Filter out path-lock markers.+ result.matches.retain(|m| {+ m.file+ .rsplit('/')+ .next()+ .map_or(true, |name| name != PATH_LOCK_FILE)+ });+ result.count = result.matches.len();+ Ok(result)}async fn tree_directory(@@ -404,6 +675,14 @@ impl FileSystem for MountableFS {}}+ // Filter out path-lock markers.+ entries.retain(|e| {+ e.path+ .rsplit('/')+ .next()+ .map_or(true, |name| name != PATH_LOCK_FILE)+ });+Ok(entries)}}@@ -411,8 +690,51 @@ impl FileSystem for MountableFS {#[cfg(test)]mod tests {use super::*;+ use crate::core::BackendItemConfig;+ use crate::core::RedirectPolicy;+ use crate::shape::SHAPE_MANIFEST_PATH;+ use serde_json::Value;use std::collections::HashMap;+ /// Helper to create a PluginConfig for tests with new fields defaulted.+ fn test_config(name: &str, mount_path: &str) -> PluginConfig {+ PluginConfig::single_backend(name, mount_path, HashMap::new())+ }++ /// Helper to create a simple multi-write PluginConfig for tests.+ fn multiwrite_test_config(+ primary_backend: &str,+ backup_backend: &str,+ mount_path: &str,+ ) -> PluginConfig {+ PluginConfig {+ name: primary_backend.to_string(),+ mount_path: mount_path.to_string(),+ params: HashMap::new(),+ backups: Some(BackendsConfig {+ sync_type: "async".to_string(),+ write_ack_count: None,+ write_ack_timeout_ms: None,+ write_concurrency: None,+ retry_interval_ms: None,+ retry_backoff_base_ms: None,+ retry_max_retries_per_round: None,+ retry_quarantine_after_failures: None,+ read_probe_cache_ttl_ms: None,+ items: vec![BackendItemConfig {+ name: "backup1".to_string(),+ backend: backup_backend.to_string(),+ params: Value::Null,+ timeout: None,+ encryption: None,+ operations: None,+ excludes: None,+ }],+ }),+ ..PluginConfig::default()+ }+ }+// Mock filesystem for testingstruct MockFS {name: String,@@ -453,7 +775,13 @@ mod tests {Ok(())}- async fn read(&self, _path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {+ async fn read(&self, path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {+ if path == SHAPE_MANIFEST_PATH+ || path.ends_with(".redirect.json")+ || path.ends_with(".sync_log.json")+ {+ return Err(Error::not_found(path));+ }Ok(self.name.as_bytes().to_vec())}@@ -562,6 +890,24 @@ mod tests {}}+ /// Create a MountableFS with one registered and mounted mock plugin.+ async fn mounted_mock(name: &str, mount_path: &str) -> MountableFS {+ let mfs = MountableFS::new();+ mfs.register_plugin(MockPlugin::new(name)).await;+ mfs.mount(test_config(name, mount_path)).await.unwrap();+ mfs+ }++ /// Create a MountableFS with two mounted mock plugins.+ async fn mounted_two_mocks() -> MountableFS {+ let mfs = MountableFS::new();+ for (name, mount_path) in [("mock1", "/fs1"), ("mock2", "/fs2")] {+ mfs.register_plugin(MockPlugin::new(name)).await;+ mfs.mount(test_config(name, mount_path)).await.unwrap();+ }+ mfs+ }+#[test]fn test_normalize_path() {assert_eq!(normalize_path("/test"), "/test");@@ -586,11 +932,7 @@ mod tests {mfs.register_plugin(MockPlugin::new("mock")).await;// Mount filesystem- let config = PluginConfig {- name: "mock".to_string(),- mount_path: "/mock".to_string(),- params: HashMap::new(),- };+ let config = test_config("mock", "/mock");assert!(mfs.mount(config).await.is_ok());@@ -613,11 +955,7 @@ mod tests {let mfs = MountableFS::new();mfs.register_plugin(MockPlugin::new("mock")).await;- let config = PluginConfig {- name: "mock".to_string(),- mount_path: "/mock".to_string(),- params: HashMap::new(),- };+ let config = test_config("mock", "/mock");// First mount should succeedassert!(mfs.mount(config.clone()).await.is_ok());@@ -639,16 +977,7 @@ mod tests {#[tokio::test]async fn test_filesystem_operations() {- let mfs = MountableFS::new();- mfs.register_plugin(MockPlugin::new("mock")).await;-- let config = PluginConfig {- name: "mock".to_string(),- mount_path: "/mock".to_string(),- params: HashMap::new(),- };-- mfs.mount(config).await.unwrap();+ let mfs = mounted_mock("mock", "/mock").await;// Test read operationlet data = mfs.read("/mock/test.txt", 0, 0).await.unwrap();@@ -668,25 +997,7 @@ mod tests {#[tokio::test]async fn test_path_routing() {- let mfs = MountableFS::new();- mfs.register_plugin(MockPlugin::new("mock1")).await;- mfs.register_plugin(MockPlugin::new("mock2")).await;-- // Mount two filesystems- let config1 = PluginConfig {- name: "mock1".to_string(),- mount_path: "/fs1".to_string(),- params: HashMap::new(),- };-- let config2 = PluginConfig {- name: "mock2".to_string(),- mount_path: "/fs2".to_string(),- params: HashMap::new(),- };-- mfs.mount(config1).await.unwrap();- mfs.mount(config2).await.unwrap();+ let mfs = mounted_two_mocks().await;// Test routing to different filesystemslet data1 = mfs.read("/fs1/file.txt", 0, 0).await.unwrap();@@ -698,24 +1009,7 @@ mod tests {#[tokio::test]async fn test_rename_across_mounts_error() {- let mfs = MountableFS::new();- mfs.register_plugin(MockPlugin::new("mock1")).await;- mfs.register_plugin(MockPlugin::new("mock2")).await;-- let config1 = PluginConfig {- name: "mock1".to_string(),- mount_path: "/fs1".to_string(),- params: HashMap::new(),- };-- let config2 = PluginConfig {- name: "mock2".to_string(),- mount_path: "/fs2".to_string(),- params: HashMap::new(),- };-- mfs.mount(config1).await.unwrap();- mfs.mount(config2).await.unwrap();+ let mfs = mounted_two_mocks().await;// Try to rename across different mounts - should faillet result = mfs.rename("/fs1/file.txt", "/fs2/file.txt").await;@@ -727,16 +1021,7 @@ mod tests {async fn test_concurrent_operations() {use tokio::task;- let mfs = Arc::new(MountableFS::new());- mfs.register_plugin(MockPlugin::new("mock")).await;-- let config = PluginConfig {- name: "mock".to_string(),- mount_path: "/mock".to_string(),- params: HashMap::new(),- };-- mfs.mount(config).await.unwrap();+ let mfs = Arc::new(mounted_mock("mock", "/mock").await);// Spawn multiple concurrent read operationslet mut handles = vec![];@@ -774,11 +1059,7 @@ mod tests {for i in 0..5 {let mfs_clone = Arc::clone(&mfs);let handle = task::spawn(async move {- let config = PluginConfig {- name: format!("mock{}", i),- mount_path: format!("/mock{}", i),- params: HashMap::new(),- };+ let config = test_config(&format!("mock{}", i), &format!("/mock{}", i));mfs_clone.mount(config).await});handles.push(handle);@@ -816,15 +1097,7 @@ mod tests {#[tokio::test]async fn test_grep_routes_to_plugin() {- let mfs = MountableFS::new();- mfs.register_plugin(MockPlugin::new("mock")).await;-- let config = PluginConfig {- name: "mock".to_string(),- mount_path: "/mock".to_string(),- params: HashMap::new(),- };- mfs.mount(config).await.unwrap();+ let mfs = mounted_mock("mock", "/mock").await;let result = mfs.grep("/mock/a.txt", "foo", false, false, None, None, None)@@ -842,12 +1115,10 @@ mod tests {TreeEntry {path: path.to_string(),rel_path: rel_path.to_string(),- info: FileInfo {- name: name.to_string(),- size: if is_dir { 0 } else { 100 },- mode: if is_dir { 0o755 } else { 0o644 },- mod_time: std::time::UNIX_EPOCH,- is_dir,+ info: if is_dir {+ FileInfo::new_dir(name.to_string(), 0o755)+ } else {+ FileInfo::new_file(name.to_string(), 100, 0o644)},extra: HashMap::new(),}@@ -875,13 +1146,7 @@ mod tests {)],);mfs.register_plugin(plugin).await;- mfs.mount(PluginConfig {- name: "rewrite".to_string(),- mount_path: "/rewrite".to_string(),- params: HashMap::new(),- })- .await- .unwrap();+ mfs.mount(test_config("rewrite", "/rewrite")).await.unwrap();let result = mfs.tree_directory("/rewrite/sub", false, None, None)@@ -900,13 +1165,9 @@ mod tests {vec![make_tree_entry("/a/b/c.txt", "a/b/c.txt", "c.txt", false)],);mfs.register_plugin(plugin).await;- mfs.mount(PluginConfig {- name: "relpath".to_string(),- mount_path: "/local/test_account".to_string(),- params: HashMap::new(),- })- .await- .unwrap();+ mfs.mount(test_config("relpath", "/local/test_account"))+ .await+ .unwrap();let result = mfs.tree_directory("/local/test_account/a", false, None, None)@@ -916,4 +1177,226 @@ mod tests {assert_eq!(result[0].path, "/local/test_account/a/b/c.txt");assert_eq!(result[0].rel_path, "a/b/c.txt");}++ #[tokio::test]+ async fn test_build_multi_write_fs_rejects_primary_encryption_disable() {+ let mfs = MountableFS::new();+ mfs.register_plugin(MockPlugin::new("primary")).await;+ mfs.register_plugin(MockPlugin::new("backupfs")).await;+ mfs.set_encryption_config(Some([7u8; 32]), Some(1)).await;++ let backups = BackendsConfig {+ sync_type: "async".to_string(),+ write_ack_count: None,+ write_ack_timeout_ms: None,+ write_concurrency: None,+ retry_interval_ms: None,+ retry_backoff_base_ms: None,+ retry_max_retries_per_round: None,+ retry_quarantine_after_failures: None,+ read_probe_cache_ttl_ms: None,+ items: vec![BackendItemConfig {+ name: "backup1".to_string(),+ backend: "backupfs".to_string(),+ params: serde_json::Value::Null,+ timeout: None,+ encryption: None,+ operations: None,+ excludes: None,+ }],+ };+ let config = PluginConfig {+ name: "primary".to_string(),+ mount_path: "/local".to_string(),+ params: HashMap::new(),+ backups: Some(backups.clone()),+ server_encryption_enabled: true,+ ..PluginConfig::default()+ };++ let result = mfs.build_multi_write_fs(&config, &backups).await;+ assert!(result.is_err());+ assert!(matches!(result.err(), Some(Error::Config(_))));+ }++ #[tokio::test]+ async fn test_build_multi_write_fs_rejects_exclude_target_field() {+ let mfs = MountableFS::new();+ mfs.register_plugin(MockPlugin::new("primary")).await;+ mfs.register_plugin(MockPlugin::new("backupfs")).await;++ let backups = BackendsConfig {+ sync_type: "async".to_string(),+ write_ack_count: None,+ write_ack_timeout_ms: None,+ write_concurrency: None,+ retry_interval_ms: None,+ retry_backoff_base_ms: None,+ retry_max_retries_per_round: None,+ retry_quarantine_after_failures: None,+ read_probe_cache_ttl_ms: None,+ items: vec![BackendItemConfig {+ name: "backup1".to_string(),+ backend: "backupfs".to_string(),+ params: serde_json::Value::Null,+ timeout: None,+ encryption: None,+ operations: None,+ excludes: Some(vec![RedirectPolicy::FileExtensionPolicy {+ extensions: vec!["\\.tmp$".to_string()],+ target: Some(vec!["should-not-exist".to_string()]),+ }]),+ }],+ };+ let config = PluginConfig {+ name: "primary".to_string(),+ mount_path: "/local".to_string(),+ params: HashMap::new(),+ backups: Some(backups.clone()),+ ..PluginConfig::default()+ };++ let result = mfs.build_multi_write_fs(&config, &backups).await;+ assert!(result.is_err());+ assert!(matches!(result.err(), Some(Error::Config(_))));+ }++ #[tokio::test]+ async fn test_build_multi_write_fs_rejects_reserved_primary_backup_name() {+ let mfs = MountableFS::new();+ mfs.register_plugin(MockPlugin::new("primary")).await;+ mfs.register_plugin(MockPlugin::new("backupfs")).await;++ let backups = BackendsConfig {+ sync_type: "async".to_string(),+ write_ack_count: None,+ write_ack_timeout_ms: None,+ write_concurrency: None,+ retry_interval_ms: None,+ retry_backoff_base_ms: None,+ retry_max_retries_per_round: None,+ retry_quarantine_after_failures: None,+ read_probe_cache_ttl_ms: None,+ items: vec![BackendItemConfig {+ name: "primary".to_string(),+ backend: "backupfs".to_string(),+ params: serde_json::Value::Null,+ timeout: None,+ encryption: None,+ operations: None,+ excludes: None,+ }],+ };+ let config = PluginConfig {+ name: "primary".to_string(),+ mount_path: "/local".to_string(),+ params: HashMap::new(),+ backups: Some(backups.clone()),+ ..PluginConfig::default()+ };++ let result = mfs.build_multi_write_fs(&config, &backups).await;+ assert!(result.is_err());+ assert!(matches!(result.err(), Some(Error::Config(_))));+ }++ #[tokio::test]+ async fn test_single_backend_mount_wraps_stats_outside_encryption() {+ let mfs = MountableFS::new();+ mfs.register_plugin(MockPlugin::new("mock")).await;+ mfs.set_encryption_config(Some([9u8; 32]), Some(1)).await;++ mfs.mount(test_config("mock", "/mock")).await.unwrap();++ let mounts = mfs.mounts.read().await;+ let mount_info = mounts.get("/mock").expect("mounted entry should exist");+ let wrapped = (mount_info.fs.as_ref() as &dyn std::any::Any)+ .downcast_ref::<StatsWrappedFS>()+ .expect("stats wrapper should be outermost");+ assert!(+ (wrapped.inner_fs().as_ref() as &dyn std::any::Any)+ .downcast_ref::<EncryptionWrappedFS>()+ .is_some(),+ "single-backend encrypted mount should place encryption under stats"+ );+ }++ #[tokio::test]+ async fn test_multiwrite_mount_wraps_stats_outside_multiwrite() {+ let mfs = MountableFS::new();+ mfs.register_plugin(MockPlugin::new("primary")).await;+ mfs.register_plugin(MockPlugin::new("backupfs")).await;++ mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))+ .await+ .unwrap();++ let mounts = mfs.mounts.read().await;+ let mount_info = mounts.get("/local").expect("mounted entry should exist");+ let wrapped = (mount_info.fs.as_ref() as &dyn std::any::Any)+ .downcast_ref::<StatsWrappedFS>()+ .expect("stats wrapper should be outermost");+ assert!(+ (wrapped.inner_fs().as_ref() as &dyn std::any::Any)+ .downcast_ref::<MultiWriteWrappedFS>()+ .is_some(),+ "multi-write mount should place MultiWriteWrappedFS directly under stats"+ );+ }++ #[tokio::test]+ async fn test_multiwrite_raw_access_is_rejected() {+ let mfs = MountableFS::new();+ mfs.register_plugin(MockPlugin::new("primary")).await;+ mfs.register_plugin(MockPlugin::new("backupfs")).await;++ mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))+ .await+ .unwrap();++ let read_err = mfs.read_raw("/local/data.txt", 0, 0).await.unwrap_err();+ assert!(matches!(read_err, Error::InvalidOperation(_)));++ let write_err = mfs+ .write_raw("/local/data.txt", b"raw", WriteFlag::Create)+ .await+ .unwrap_err();+ assert!(matches!(write_err, Error::InvalidOperation(_)));+ }++ #[tokio::test]+ async fn test_mountable_multiwrite_admin_smoke() {+ let mfs = MountableFS::new();+ mfs.register_plugin(MockPlugin::new("primary")).await;+ mfs.register_plugin(MockPlugin::new("backupfs")).await;++ mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))+ .await+ .unwrap();++ let status = crate::core::FS_CTX+ .scope(+ Arc::new(crate::core::context::FsContextInner::new(+ "acct".to_string(),+ )),+ async { mfs.system_sync_status("/local").await.unwrap() },+ )+ .await;+ assert_eq!(status["path"], "/");+ assert_eq!(status["entry_count"], 0);+ assert_eq!(status["pending_target_count"], 0);+ assert_eq!(status["capabilities"]["multi_instance_safe"], false);++ let retry = crate::core::FS_CTX+ .scope(+ Arc::new(crate::core::context::FsContextInner::new(+ "acct".to_string(),+ )),+ async { mfs.system_sync_retry("/local").await.unwrap() },+ )+ .await;+ assert_eq!(retry["path"], "/");+ assert_eq!(retry["retried"], 0);+ assert_eq!(retry["failed"], 0);+ }}diff --git a/crates/ragfs/src/core/multibackend_wrapper.rs b/crates/ragfs/src/core/multibackend_wrapper.rsnew file mode 100644index 000000000..932d24fa8--- /dev/null+++ b/crates/ragfs/src/core/multibackend_wrapper.rs@@ -0,0 +1,1673 @@+//! Multi-write wrapper — routes operations across primary and backup backends.+//!+//! Implements `MultiWriteWrappedFS` which handles:+//! - Write fanout to primary + backup backends (sync/async)+//! - Read routing with priority-based fallback chain+//! - Redirect policy evaluation+//! - Exclude policy filtering+//! - `.redirect.json` / `.sync_log.json` metadata management++use std::collections::{HashMap, HashSet};+use std::future::Future;+use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};+use std::sync::Arc;+use std::time::Duration;++use async_trait::async_trait;+use regex::Regex;+use serde_json::Value;+use tokio::sync::{Mutex, Notify};++use super::context::{FsContext, FS_CTX};+use super::errors::{Error, Result};+use super::filesystem::{normalize_prefix_path, relative_match_file, FileSystem};+use super::types::{+ BackendRole, BackendSyncState, FileInfo, GrepResult, OperationItemConfig, RedirectEntry,+ RedirectPolicy, SyncLogEntry, SyncOp, SyncType, TreeEntry, WriteFlag,+};+use crate::multibackend::meta::{+ current_required_ctx, file_name, parent_dir, DefaultFsContextResolver, FsContextResolver,+ MetaStateStore, PathSerializer, MULTIWRITE_INTERNAL_NAMES,+};++mod retry;+mod routing;+#[cfg(test)]+mod tests;++/// Default chunk size used when copying file state from primary to backup.+const DEFAULT_COPY_CHUNK_SIZE: usize = 8 * 1024 * 1024;+/// Default retry loop interval.+const DEFAULT_RETRY_INTERVAL: Duration = Duration::from_secs(30);+/// Default retry backoff base in milliseconds.+const DEFAULT_RETRY_BACKOFF_BASE_MS: u64 = 1000;+/// Default maximum retries per file per retry round.+const DEFAULT_MAX_RETRIES_PER_ROUND: usize = 3;+/// Default failure threshold before a target is quarantined.+const DEFAULT_QUARANTINE_AFTER_FAILURES: u32 = 9;+/// Default wait timeout when shutting down background tasks.+const DEFAULT_SHUTDOWN_WAIT: Duration = Duration::from_secs(5);++#[derive(Debug)]+struct SyncFanoutTaskResult {+ backend: String,+ result: Result<()>,+}++#[derive(Clone)]+enum BackupWriteOp {+ Replay(SyncOp),+ WriteFile {+ data: Arc<Vec<u8>>,+ offset: u64,+ flags: WriteFlag,+ },+ EnsureParentDirs {+ mode: u32,+ },+}++impl BackupWriteOp {+ /// Apply this backup-side operation under the provided request context.+ async fn apply(+ &self,+ inner: &Inner,+ backup: Arc<dyn FileSystem>,+ path: &str,+ ctx: &FsContext,+ ) -> Result<()> {+ match self {+ Self::Replay(op) => {+ op.replay(inner.primary().backend.clone(), backup, path, ctx)+ .await+ }+ Self::WriteFile {+ data,+ offset,+ flags,+ } => {+ let data = Arc::clone(data);+ let offset = *offset;+ let flags = *flags;+ FS_CTX+ .scope(ctx.clone(), async move {+ backup.ensure_parent_dirs(path, 0o755).await?;+ backup.write(path, data.as_slice(), offset, flags).await?;+ Ok(())+ })+ .await+ }+ Self::EnsureParentDirs { mode } => {+ let mode = *mode;+ FS_CTX+ .scope(ctx.clone(), async move {+ backup.ensure_parent_dirs(path, mode).await+ })+ .await+ }+ }+ }+}++/// Cloned target data passed to fanout strategies without borrowing `Inner`.+#[derive(Clone)]+struct FanoutTarget {+ name: String,+ backend: Arc<dyn FileSystem>,+}++/// Effective sync work item with resolved target backend names.+pub(crate) struct SyncWorkEntry {+ pub(crate) file_path: String,+ pub(crate) entry: SyncLogEntry,+ pub(crate) targets: Vec<String>,+}++#[derive(Clone, Copy)]+enum ReadRouteSource {+ Backup,+ Primary,+ Redirect,+ Miss,+}++/// Builder-style sync mode configuration.+pub enum SyncMode {+ /// Synchronous fanout requiring backup acknowledgement.+ Sync {+ /// Minimum backup acknowledgements required for a successful write.+ ack_count: usize,+ /// Maximum time to wait for backup acknowledgements, in milliseconds.+ timeout_ms: u64,+ },+ /// Asynchronous fanout with background retry.+ Async,+}++/// A backend entry within the multi-write wrapper.+pub struct BackendEntry {+ /// Logical name (globally unique)+ pub name: String,+ /// Role: Primary or Backup+ pub role: BackendRole,+ /// The backend filesystem handle (may be encrypted)+ pub backend: Arc<dyn FileSystem>,+ /// Optional raw backend handle used by primary verbatim copy fast-paths.+ pub raw_backend: Option<Arc<dyn FileSystem>>,+ /// Operations this backend participates in (only for Backup)+ pub operations: Vec<OperationItemConfig>,+ /// Exclude policies (only for Backup)+ pub excludes: Vec<RedirectPolicy>,+}++impl BackendEntry {+ /// Check if this backend participates in read operations.+ fn participates_in_read(&self) -> bool {+ self.operations.iter().any(|op| op.operation == "read")+ }++ /// Check if this backend participates in write operations.+ /// Backups default to write-enabled when operations is empty.+ fn participates_in_write(&self) -> bool {+ if self.operations.is_empty() {+ true+ } else {+ self.operations.iter().any(|op| op.operation == "write")+ }+ }++ /// Get read priority (lower = higher priority). Returns None if not read-enabled.+ fn read_priority(&self) -> Option<u32> {+ self.operations+ .iter()+ .find(|op| op.operation == "read")+ .map(|op| op.priority)+ }++ /// Convert this backend entry into a fanout target.+ fn fanout_target(&self) -> FanoutTarget {+ FanoutTarget {+ name: self.name.clone(),+ backend: self.backend.clone(),+ }+ }+}++/// File policy trait — shared by redirects and excludes.+pub trait FilePolicy {+ /// Check if this policy matches the given file.+ fn matches(&self, path: &str, size: u64) -> bool;+}++impl FilePolicy for RedirectPolicy {+ fn matches(&self, path: &str, size: u64) -> bool {+ match self {+ RedirectPolicy::FileOverSizePolicy { max_size_mb, .. } => {+ let max_bytes = max_size_mb * 1024 * 1024;+ size > max_bytes+ }+ RedirectPolicy::FileExtensionPolicy { extensions, .. } => {+ let name = file_name(path);+ extensions.iter().any(|ext_pattern| {+ if let Ok(re) = Regex::new(ext_pattern) {+ re.is_match(name)+ } else {+ name.ends_with(ext_pattern.as_str())+ }+ })+ }+ }+ }+}++impl SyncOp {+ /// Replay this operation on one backup backend using the original request semantics.+ async fn replay(+ &self,+ primary: Arc<dyn FileSystem>,+ backup: Arc<dyn FileSystem>,+ file_path: &str,+ ctx: &FsContext,+ ) -> Result<()> {+ match self {+ SyncOp::SyncFile { size } => {+ let size = *size;+ copy_current_primary_state(primary, backup, file_path, size, ctx).await+ }+ SyncOp::Create => {+ FS_CTX+ .scope(ctx.clone(), async { backup.create(file_path).await })+ .await+ }+ SyncOp::Mkdir { mode } => {+ let mode = *mode;+ FS_CTX+ .scope(ctx.clone(), async { backup.mkdir(file_path, mode).await })+ .await+ }+ SyncOp::Remove => {+ match FS_CTX+ .scope(ctx.clone(), async { backup.remove(file_path).await })+ .await+ {+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),+ Err(e) => Err(e),+ }+ }+ SyncOp::RemoveAll => {+ match FS_CTX+ .scope(ctx.clone(), async { backup.remove_all(file_path).await })+ .await+ {+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),+ Err(e) => Err(e),+ }+ }+ SyncOp::Rename { to } => {+ let to = to.clone();+ FS_CTX+ .scope(ctx.clone(), async {+ backup.ensure_parent_dirs(&to, 0o755).await?;+ backup.rename(file_path, &to).await+ })+ .await+ }+ SyncOp::Chmod { mode } => {+ let mode = *mode;+ FS_CTX+ .scope(ctx.clone(), async { backup.chmod(file_path, mode).await })+ .await+ }+ }+ }+}++/// Copy one file between two filesystem handles in bounded-size chunks.+async fn copy_file_state(+ source: Arc<dyn FileSystem>,+ source_path: &str,+ destination: Arc<dyn FileSystem>,+ destination_path: &str,+ size: u64,+ ctx: &FsContext,+) -> Result<()> {+ FS_CTX+ .scope(ctx.clone(), async {+ destination+ .ensure_parent_dirs(destination_path, 0o755)+ .await?;+ if size == 0 {+ if destination.exists(destination_path).await {+ return destination.truncate(destination_path, 0).await;+ }+ return destination.create(destination_path).await;+ }++ let mut offset = 0u64;+ while offset < size {+ let chunk_len = (size - offset).min(DEFAULT_COPY_CHUNK_SIZE as u64);+ let chunk = source.read(source_path, offset, chunk_len).await?;+ let flag = if offset == 0 {+ WriteFlag::Create+ } else {+ WriteFlag::None+ };+ destination+ .write(destination_path, &chunk, offset, flag)+ .await?;+ offset = offset.saturating_add(chunk.len() as u64);+ if chunk.is_empty() {+ return Err(Error::internal(format!(+ "source returned empty chunk while copying '{}' -> '{}'",+ source_path, destination_path+ )));+ }+ }+ Ok(())+ })+ .await+}++/// Copy the current file state from primary to backup in bounded-size chunks.+async fn copy_current_primary_state(+ primary: Arc<dyn FileSystem>,+ backup: Arc<dyn FileSystem>,+ file_path: &str,+ size: u64,+ ctx: &FsContext,+) -> Result<()> {+ copy_file_state(primary, file_path, backup, file_path, size, ctx).await+}++/// Copy one file within the primary raw backend without decrypting and re-encrypting bytes.+async fn copy_raw_primary_state(+ primary_raw: Arc<dyn FileSystem>,+ source_path: &str,+ destination_path: &str,+ ctx: &FsContext,+) -> Result<()> {+ let raw_size = FS_CTX+ .scope(ctx.clone(), async {+ let source_info = primary_raw.stat(source_path).await?;+ if source_info.is_dir {+ return Err(Error::IsADirectory(source_path.to_string()));+ }+ Ok::<u64, Error>(source_info.size)+ })+ .await?;++ copy_file_state(+ primary_raw.clone(),+ source_path,+ primary_raw,+ destination_path,+ raw_size,+ ctx,+ )+ .await+}++/// Inner state shared via Arc for async spawn and retry_loop.+pub(crate) struct Inner {+ /// All backend entries (primary at index 0)+ backends: Vec<BackendEntry>,+ /// Index of the primary backend+ primary_idx: usize,+ /// Sync type: Async or Sync.+ sync_type: SyncType,+ /// Minimum backup ack count for sync mode+ write_ack_count: usize,+ /// Timeout for waiting backup ack in sync mode (ms)+ write_ack_timeout_ms: u64,+ /// Semaphore for async write concurrency control+ write_sem: Option<Arc<tokio::sync::Semaphore>>,+ /// Primary redirect policies+ redirects: Vec<RedirectPolicy>,+ /// Metadata store (encrypted via primary_backend)+ pub(crate) meta_store: MetaStateStore,+ /// Per-path serialization queues+ path_queues: PathSerializer,+ /// Directories that currently have outstanding retry work.+ pending_dirs: Mutex<HashSet<String>>,+ /// Retry loop interval.+ retry_interval: Duration,+ /// Base retry backoff in milliseconds.+ pub(crate) retry_backoff_base_ms: u64,+ /// Maximum retry attempts for one target in one round.+ pub(crate) max_retry_per_round: usize,+ /// Failure threshold before quarantining one target.+ quarantine_after_failures: u32,+ /// Number of background tasks currently in flight.+ background_tasks: AtomicUsize,+ /// Notifier fired when background task count reaches zero.+ idle_notify: Notify,+ /// Read route hit metrics.+ read_backup_hits: AtomicU64,+ read_primary_hits: AtomicU64,+ read_redirect_hits: AtomicU64,+ read_misses: AtomicU64,+ /// Cancellation flag for the background retry loop.+ retry_cancelled: AtomicBool,+ /// Wake-up signal used to stop retry_loop promptly on drop.+ retry_shutdown: Notify,+}++/// Multi-write wrapped filesystem.+pub struct MultiWriteWrappedFS {+ pub(crate) inner: Arc<Inner>,+}++/// Builder for `MultiWriteWrappedFS`.+pub struct MultiWriteWrappedFSBuilder {+ primary_backend: Arc<dyn FileSystem>,+ primary_raw_backend: Option<Arc<dyn FileSystem>>,+ backup_entries: Vec<BackendEntry>,+ redirects: Vec<RedirectPolicy>,+ sync_mode: SyncMode,+ write_concurrency: Option<usize>,+ retry_interval: Duration,+ retry_backoff_base_ms: u64,+ max_retry_per_round: usize,+ quarantine_after_failures: u32,+ ctx_resolver: Arc<dyn FsContextResolver>,+}++impl MultiWriteWrappedFSBuilder {+ /// Attach the raw primary backend so the wrapper can do verbatim primary copies.+ pub fn with_primary_raw_backend(mut self, primary_raw_backend: Arc<dyn FileSystem>) -> Self {+ self.primary_raw_backend = Some(primary_raw_backend);+ self+ }++ /// Set backup backend entries on the builder.+ pub fn with_backups(mut self, backup_entries: Vec<BackendEntry>) -> Self {+ self.backup_entries = backup_entries;+ self+ }++ /// Set redirect policies for the primary backend.+ pub fn with_redirects(mut self, redirects: Vec<RedirectPolicy>) -> Self {+ self.redirects = redirects;+ self+ }++ /// Select the sync mode used by write fanout.+ pub fn sync_mode(mut self, sync_mode: SyncMode) -> Self {+ self.sync_mode = sync_mode;+ self+ }++ /// Set the maximum number of concurrent async backup writes.+ pub fn write_concurrency(mut self, write_concurrency: Option<usize>) -> Self {+ self.write_concurrency = write_concurrency;+ self+ }++ /// Configure retry loop interval.+ pub fn retry_interval(mut self, retry_interval: Duration) -> Self {+ self.retry_interval = retry_interval;+ self+ }++ /// Configure retry backoff base duration in milliseconds.+ pub fn retry_backoff_base_ms(mut self, retry_backoff_base_ms: u64) -> Self {+ self.retry_backoff_base_ms = retry_backoff_base_ms;+ self+ }++ /// Configure the maximum number of retries per round.+ pub fn max_retry_per_round(mut self, max_retry_per_round: usize) -> Self {+ self.max_retry_per_round = max_retry_per_round.max(1);+ self+ }++ /// Configure quarantine threshold for one path/backup pair.+ pub fn quarantine_after_failures(mut self, quarantine_after_failures: u32) -> Self {+ self.quarantine_after_failures = quarantine_after_failures.max(1);+ self+ }++ /// Accept the legacy read-route cache TTL option as a no-op.+ pub fn read_route_cache_ttl(self, _read_route_cache_ttl: Duration) -> Self {+ self+ }++ /// Configure the context resolver used by retry and admin background paths.+ pub fn ctx_resolver(mut self, ctx_resolver: Arc<dyn FsContextResolver>) -> Self {+ self.ctx_resolver = ctx_resolver;+ self+ }++ /// Build the multi-write wrapper and start the retry loop when needed.+ pub fn build(self) -> Result<MultiWriteWrappedFS> {+ let mut backends = Vec::new();+ backends.push(BackendEntry {+ name: "primary".to_string(),+ role: BackendRole::Primary,+ backend: self.primary_backend.clone(),+ raw_backend: self.primary_raw_backend.clone(),+ operations: Vec::new(),+ excludes: Vec::new(),+ });+ backends.extend(self.backup_entries);++ let (sync_type, write_ack_count, write_ack_timeout_ms) = match self.sync_mode {+ SyncMode::Sync {+ ack_count,+ timeout_ms,+ } => (SyncType::Sync, ack_count, timeout_ms),+ SyncMode::Async => (SyncType::Async, usize::MAX, 0),+ };++ let write_sem = self+ .write_concurrency+ .filter(|&n| n > 0)+ .map(|n| Arc::new(tokio::sync::Semaphore::new(n)));++ let meta_store = MetaStateStore::new(self.primary_backend, self.ctx_resolver);++ let inner = Arc::new(Inner {+ backends,+ primary_idx: 0,+ sync_type,+ write_ack_count,+ write_ack_timeout_ms,+ write_sem,+ redirects: self.redirects,+ meta_store,+ path_queues: PathSerializer::new(),+ pending_dirs: Mutex::new(HashSet::new()),+ retry_interval: self.retry_interval,+ retry_backoff_base_ms: self.retry_backoff_base_ms,+ max_retry_per_round: self.max_retry_per_round,+ quarantine_after_failures: self.quarantine_after_failures,+ background_tasks: AtomicUsize::new(0),+ idle_notify: Notify::new(),+ read_backup_hits: AtomicU64::new(0),+ read_primary_hits: AtomicU64::new(0),+ read_redirect_hits: AtomicU64::new(0),+ read_misses: AtomicU64::new(0),+ retry_cancelled: AtomicBool::new(false),+ retry_shutdown: Notify::new(),+ });++ // Start retry_loop if there are write-enabled backups.+ if inner.write_backups().next().is_some() {+ inner.background_task_started();+ tokio::spawn(Inner::retry_loop(Arc::clone(&inner)));+ }++ Ok(MultiWriteWrappedFS { inner })+ }+}++impl MultiWriteWrappedFS {+ /// Start building a multi-write wrapper from a primary backend.+ pub fn builder(primary_backend: Arc<dyn FileSystem>) -> MultiWriteWrappedFSBuilder {+ MultiWriteWrappedFSBuilder {+ primary_backend,+ primary_raw_backend: None,+ backup_entries: Vec::new(),+ redirects: Vec::new(),+ sync_mode: SyncMode::Async,+ write_concurrency: None,+ retry_interval: DEFAULT_RETRY_INTERVAL,+ retry_backoff_base_ms: DEFAULT_RETRY_BACKOFF_BASE_MS,+ max_retry_per_round: DEFAULT_MAX_RETRIES_PER_ROUND,+ quarantine_after_failures: DEFAULT_QUARANTINE_AFTER_FAILURES,+ ctx_resolver: Arc::new(DefaultFsContextResolver),+ }+ }+}++impl Inner {+ /// Build the per-path/per-backend queue key used by both fanout and retry.+ fn backup_queue_key(path: &str, backup_name: &str) -> String {+ format!("{}\0{}", path, backup_name)+ }++ /// Iterate over write-enabled backup entries.+ fn write_backups(&self) -> impl Iterator<Item = &BackendEntry> {+ self.backends[self.primary_idx + 1..]+ .iter()+ .filter(|be| be.participates_in_write())+ }++ /// Resolve write-enabled backup targets after applying exclude policies.+ fn write_targets(&self, path: &str, size: u64) -> Vec<FanoutTarget> {+ self.write_backups()+ .filter(|be| !self.is_excluded(be, path, size))+ .map(BackendEntry::fanout_target)+ .collect()+ }++ /// Resolve explicitly named backup targets.+ fn named_targets(&self, target_names: &[String]) -> Vec<FanoutTarget> {+ target_names+ .iter()+ .filter_map(|name| self.backup_by_name(name))+ .map(BackendEntry::fanout_target)+ .collect()+ }++ /// Execute one backup write synchronously so redirect visibility only appears+ /// after at least one target has durably materialized the file contents.+ async fn write_first_target(+ inner: &Arc<Inner>,+ path: &str,+ target: &FanoutTarget,+ ctx: &FsContext,+ op: BackupWriteOp,+ ) -> Result<()> {+ let queue_key = Self::backup_queue_key(path, &target.name);+ let fs = target.backend.clone();+ let ack_ctx = ctx.clone();++ inner+ .path_queues+ .with_path_lock(&queue_key, || async move {+ op.apply(inner, fs, path, ctx).await+ })+ .await?;++ inner+ .update_backup_acked_seq(path, &target.name, &ack_ctx)+ .await+ }++ /// Resolve effective target backend names for sync/retry work.+ pub(crate) fn target_backend_names(+ &self,+ redirect_meta: &super::types::RedirectMeta,+ file_name: &str,+ file_path: &str,+ sync_entry: &SyncLogEntry,+ ) -> Vec<String> {+ if let Some(redir) = redirect_meta.entries.get(file_name) {+ return redir.targets.clone();+ }+ let policy_size = self.retry_policy_size(sync_entry);+ self.write_backups()+ .filter(|be| !self.is_excluded(be, file_path, policy_size))+ .map(|be| be.name.clone())+ .collect()+ }++ /// Iterate over read-enabled backup entries sorted by priority.+ fn read_backups_sorted(&self) -> Vec<&BackendEntry> {+ let mut read_backups: Vec<&BackendEntry> = self.backends[self.primary_idx + 1..]+ .iter()+ .filter(|be| be.participates_in_read())+ .collect();+ read_backups.sort_by_key(|be| be.read_priority().unwrap_or(u32::MAX));+ read_backups+ }++ /// Get the primary backend entry.+ pub(crate) fn primary(&self) -> &BackendEntry {+ &self.backends[self.primary_idx]+ }++ /// Get a backup entry by name.+ fn backup_by_name(&self, name: &str) -> Option<&BackendEntry> {+ self.backends.iter().find(|be| be.name == name)+ }++ /// Check if a file should be excluded from a backup.+ fn is_excluded(&self, backup: &BackendEntry, path: &str, size: u64) -> bool {+ backup+ .excludes+ .iter()+ .any(|policy| policy.matches(path, size))+ }++ /// Check if a file matches any redirect policy.+ fn check_redirect(&self, path: &str, size: u64) -> Option<Vec<String>> {+ for policy in &self.redirects {+ if policy.matches(path, size) {+ let targets = match policy {+ RedirectPolicy::FileOverSizePolicy { target, .. } => target.clone(),+ RedirectPolicy::FileExtensionPolicy { target, .. } => target.clone(),+ };+ return targets;+ }+ }+ None+ }++ /// Resolve persisted redirect targets for one already-written file path.+ async fn redirect_targets_for_path(+ &self,+ path: &str,+ ctx: &FsContext,+ ) -> Result<Option<Vec<String>>> {+ let dir = parent_dir(path);+ let name = file_name(path);+ let redirect_meta = self.meta_store.get_redirect_meta(&dir, ctx).await?;+ Ok(redirect_meta+ .entries+ .get(name)+ .map(|entry| entry.targets.clone()))+ }++ /// Generate and persist the next sequence number.+ async fn next_seq(&self) -> Result<u64> {+ self.meta_store.next_seq().await+ }++ /// Resolve a file size for retry-time policy decisions.+ fn retry_policy_size(&self, sync_entry: &SyncLogEntry) -> u64 {+ match &sync_entry.op {+ SyncOp::SyncFile { size } => *size,+ _ => 0,+ }+ }++ /// Execute the primary-write, sync-log and backup-fanout pipeline.+ async fn execute_write<R, P, PFut>(+ inner: &Arc<Self>,+ path: String,+ size: u64,+ sync_op: Option<SyncOp>,+ backup_op: Option<BackupWriteOp>,+ primary_fn: P,+ ) -> Result<R>+ where+ P: FnOnce(Arc<dyn FileSystem>) -> PFut + Send,+ PFut: Future<Output = Result<R>> + Send,+ {+ let ctx = current_required_ctx()?;+ inner.invalidate_read_route(&path).await;++ let prepared_entry = if let Some(entry) = sync_op {+ let dir = parent_dir(&path);+ let name = file_name(&path).to_string();+ let seq = inner.next_seq().await?;+ inner+ .meta_store+ .update_dir_meta(&dir, &ctx, {+ let name = name.clone();+ move |_redirect, sync_log| {+ sync_log.entries.insert(name, SyncLogEntry::new(seq, entry));+ Ok(())+ }+ })+ .await?;+ Some((dir, name, seq))+ } else {+ None+ };++ let result = match FS_CTX+ .scope(ctx.clone(), primary_fn(inner.primary().backend.clone()))+ .await+ {+ Ok(result) => result,+ Err(err) => {+ if let Some((dir, name, seq)) = prepared_entry.as_ref() {+ let _ = inner+ .meta_store+ .update_dir_meta(dir, &ctx, {+ let name = name.clone();+ let seq = *seq;+ move |_redirect, sync_log| {+ let should_remove =+ sync_log.entries.get(&name).is_some_and(|entry| {+ entry.latest_seq == seq && !entry.is_primary_committed()+ });+ if should_remove {+ sync_log.entries.remove(&name);+ }+ Ok(())+ }+ })+ .await;+ }+ return Err(err);+ }+ };++ if let Some((dir, name, seq)) = prepared_entry {+ inner+ .meta_store+ .update_dir_meta(&dir, &ctx, move |_redirect, sync_log| {+ let entry = sync_log.entries.get_mut(&name).ok_or_else(|| {+ Error::internal(format!(+ "prepared sync log entry missing while committing '{}'",+ name+ ))+ })?;+ if entry.latest_seq != seq {+ return Err(Error::internal(format!(+ "prepared sync log entry seq mismatch while committing '{}'",+ name+ )));+ }+ entry.mark_primary_committed();+ Ok(())+ })+ .await?;+ inner.mark_pending_dir(&dir).await;+ }++ let dir = parent_dir(&path);+ let fanout_result = if let Some(backup_op) = backup_op {+ Inner::fanout_write(inner, &path, size, ctx.clone(), backup_op).await+ } else {+ Ok(())+ };+ inner.refresh_pending_dir(&dir, &ctx).await?;+ fanout_result?;+ Ok(result)+ }++ /// Execute a write that may be redirected away from the primary backend.+ async fn execute_write_with_redirect<P, PFut>(+ inner: &Arc<Self>,+ path: String,+ size: u64,+ sync_op: SyncOp,+ backup_op: BackupWriteOp,+ primary_fn: P,+ ) -> Result<u64>+ where+ P: FnOnce(Arc<dyn FileSystem>) -> PFut + Send,+ PFut: Future<Output = Result<u64>> + Send,+ {+ let ctx = current_required_ctx()?;+ inner.invalidate_read_route(&path).await;++ if let Some(targets) = inner.check_redirect(&path, size) {+ let dir = parent_dir(&path);+ let name = file_name(&path).to_string();+ let seq = inner.next_seq().await?;+ let resolved_targets = inner.named_targets(&targets);+ let first_target = resolved_targets.first().cloned().ok_or_else(|| {+ Error::internal(format!(+ "redirect path '{}' resolved no writable targets",+ path+ ))+ })?;+ Inner::write_first_target(inner, &path, &first_target, &ctx, backup_op.clone()).await?;+ let targets_clone = targets.clone();+ let first_target_name = first_target.name.clone();+ inner+ .meta_store+ .update_dir_meta(&dir, &ctx, move |redirect, sync_log| {+ redirect.entries.insert(+ name.clone(),+ RedirectEntry {+ targets: targets_clone.clone(),+ },+ );+ let mut committed = SyncLogEntry::committed(seq, sync_op);+ committed+ .backends+ .insert(first_target_name.clone(), BackendSyncState::acked(seq));+ sync_log.entries.insert(name, committed);+ Ok(())+ })+ .await?;++ let remaining_targets: Vec<FanoutTarget> =+ resolved_targets.into_iter().skip(1).collect();+ if !remaining_targets.is_empty() {+ inner.mark_pending_dir(&dir).await;+ Inner::fanout_async(inner, &path, remaining_targets, &ctx, backup_op).await;+ }+ inner.refresh_pending_dir(&dir, &ctx).await?;+ return Ok(size);+ }++ Inner::execute_write(+ inner,+ path,+ size,+ Some(sync_op),+ Some(backup_op),+ primary_fn,+ )+ .await+ }++ /// Fanout a write operation to all write-enabled backups.+ /// Takes `&Arc<Inner>` so spawned tasks can clone the Arc for acked_seq updates.+ /// `ctx` is required for encrypted backup backends and acked_seq updates.+ async fn fanout_write(+ inner: &Arc<Inner>,+ path: &str,+ size: u64,+ ctx: FsContext,+ op: BackupWriteOp,+ ) -> Result<()> {+ Inner::fanout_targets(inner, path, inner.write_targets(path, size), ctx, op).await+ }++ /// Fanout a write operation to explicitly named backup targets (used by redirect path).+ /// Resolves names to BackendEntry references, then delegates to sync/async state machine.+ async fn fanout_write_to_targets(+ inner: &Arc<Inner>,+ path: &str,+ target_names: &[String],+ ctx: FsContext,+ op: BackupWriteOp,+ ) -> Result<()> {+ Inner::fanout_targets(inner, path, inner.named_targets(target_names), ctx, op).await+ }++ /// Fanout to already resolved targets using the configured sync mode.+ async fn fanout_targets(+ inner: &Arc<Inner>,+ path: &str,+ targets: Vec<FanoutTarget>,+ ctx: FsContext,+ op: BackupWriteOp,+ ) -> Result<()> {+ if targets.is_empty() {+ if matches!(inner.sync_type, SyncType::Sync)+ && inner.write_ack_count > 0+ && inner.write_backups().next().is_some()+ {+ return Err(Error::SyncWriteQuorum {+ succeeded: 0,+ required: inner.write_ack_count,+ attempted: 0,+ failures: Vec::new(),+ });+ }+ return Ok(());+ }++ match inner.sync_type {+ SyncType::Sync => Inner::fanout_sync(inner, path, &targets, &ctx, op).await,+ SyncType::Async => {+ Inner::fanout_async(inner, path, targets, &ctx, op).await;+ Ok(())+ }+ }+ }++ /// Synchronous fanout: execute writes in parallel, wait for quorum.+ async fn fanout_sync(+ inner: &Arc<Inner>,+ path: &str,+ targets: &[FanoutTarget],+ ctx: &FsContext,+ op: BackupWriteOp,+ ) -> Result<()> {+ let ack_count = inner.write_ack_count.min(targets.len());+ let timeout = if inner.write_ack_timeout_ms > 0 {+ Some(Duration::from_millis(inner.write_ack_timeout_ms))+ } else {+ None+ };++ let path_owned = path.to_string();+ let ctx = Some(ctx.clone());++ // Launch parallel tasks for all backup writes.+ let mut handles = Vec::new();+ for target in targets {+ let fs = target.backend.clone();+ let name = target.name.clone();+ let path = path_owned.clone();+ let inner = Arc::clone(inner);+ let ctx = ctx.clone();+ let op_clone = op.clone();+ let queue_key = Self::backup_queue_key(&path, &name);++ handles.push(tokio::spawn(async move {+ // Wrap in FS_CTX.scope so encrypted backends can access account_id.+ let exec = inner.path_queues.with_path_lock(&queue_key, || async {+ op_clone+ .apply(&inner, fs, &path, ctx.as_ref().unwrap())+ .await+ });++ let result = if let Some(timeout) = timeout {+ match tokio::time::timeout(timeout, exec).await {+ Ok(Ok(())) => Ok(()),+ Ok(Err(e)) => Err(e),+ Err(_) => Err(Error::timeout(format!(+ "backup '{}' timed out waiting for sync acknowledgement",+ name+ ))),+ }+ } else {+ exec.await+ };++ // Update acked_seq on success.+ if result.is_ok() {+ if let Some(ref ctx) = ctx {+ let _ = inner.update_backup_acked_seq(&path, &name, ctx).await;+ }+ }++ SyncFanoutTaskResult {+ backend: name,+ result,+ }+ }));+ }++ let results = futures::future::join_all(handles).await;++ let mut successes = 0usize;+ let mut failures = Vec::new();++ for result in results {+ match result {+ Ok(SyncFanoutTaskResult {+ backend: _,+ result: Ok(()),+ }) => {+ successes += 1;+ }+ Ok(SyncFanoutTaskResult {+ backend,+ result: Err(e),+ }) => {+ failures.push(super::errors::SyncWriteFailureDetail {+ backend,+ kind: e.kind_name().to_string(),+ message: e.to_string(),+ });+ }+ Err(e) => {+ failures.push(super::errors::SyncWriteFailureDetail {+ backend: "<spawn>".to_string(),+ kind: "join_error".to_string(),+ message: e.to_string(),+ });+ }+ }+ }++ if successes >= ack_count {+ Ok(())+ } else {+ Err(Error::SyncWriteQuorum {+ succeeded: successes,+ required: ack_count,+ attempted: targets.len(),+ failures,+ })+ }+ }++ /// Asynchronous fanout: spawn background tasks that update acked_seq on completion.+ /// Uses per-path serialization to prevent out-of-order application on backup backends.+ async fn fanout_async(+ inner: &Arc<Inner>,+ path: &str,+ targets: Vec<FanoutTarget>,+ ctx: &FsContext,+ op: BackupWriteOp,+ ) {+ let path_owned = path.to_string();+ let sem = inner.write_sem.clone();++ for target in targets {+ let fs = target.backend.clone();+ let name = target.name.clone();+ let path = path_owned.clone();+ let ctx = ctx.clone();+ let sem = sem.clone();+ let inner = Arc::clone(inner);+ let op_clone = op.clone();+ let queue_key = Self::backup_queue_key(&path, &name);+ inner.background_task_started();++ tokio::spawn(async move {+ {+ // Per (path, backup) serialization preserves FIFO without blocking other backups.+ let result = inner+ .path_queues+ .with_path_lock(&queue_key, || async {+ let _permit = if let Some(ref sem) = sem {+ sem.acquire().await.ok()+ } else {+ None+ };++ op_clone.apply(&inner, fs, &path, &ctx).await+ })+ .await;++ // Update acked_seq on successful write.+ if result.is_ok() {+ let _ = inner.update_backup_acked_seq(&path, &name, &ctx).await;+ }+ }+ inner.background_task_finished();+ });+ }+ }++ /// Record rename metadata and migrate redirect state when needed.+ async fn record_rename_meta(+ &self,+ old_path: &str,+ new_path: &str,+ ctx: &FsContext,+ ) -> Result<()> {+ let source_dir = parent_dir(old_path);+ let target_dir = parent_dir(new_path);+ let old_name = file_name(old_path).to_string();+ let new_name = file_name(new_path).to_string();+ let seq = self.next_seq().await?;+ let rename_op = SyncOp::Rename {+ to: new_path.to_string(),+ };++ if source_dir == target_dir {+ self.meta_store+ .update_dir_meta(&source_dir, ctx, move |redirect, sync_log| {+ sync_log+ .entries+ .insert(old_name.clone(), SyncLogEntry::committed(seq, rename_op));+ if let Some(redirect_entry) = redirect.entries.remove(&old_name) {+ redirect.entries.insert(new_name, redirect_entry);+ }+ Ok(())+ })+ .await+ } else {+ self.meta_store+ .update_dual_dir_meta(+ &source_dir,+ &target_dir,+ ctx,+ move |src_redirect, src_sync_log, tgt_redirect, _tgt_sync_log| {+ src_sync_log+ .entries+ .insert(old_name.clone(), SyncLogEntry::committed(seq, rename_op));+ if let Some(redirect_entry) = src_redirect.entries.remove(&old_name) {+ tgt_redirect.entries.insert(new_name, redirect_entry);+ }+ Ok(())+ },+ )+ .await+ }+ }+}++/// Return true when `path` falls under `exclude_path` (including itself).+fn is_excluded_grep_path(path: &str, exclude_path: Option<&str>) -> bool {+ let Some(exclude_path) = exclude_path.map(normalize_prefix_path) else {+ return false;+ };+ let normalized_path = normalize_prefix_path(path);+ normalized_path == exclude_path+ || normalized_path+ .strip_prefix(&exclude_path)+ .is_some_and(|suffix| suffix.starts_with('/'))+}++// ── FileSystem trait implementation ──++impl Drop for MultiWriteWrappedFS {+ /// Signal retry_loop to exit when the wrapper is unmounted or dropped.+ fn drop(&mut self) {+ self.inner.retry_cancelled.store(true, Ordering::SeqCst);+ self.inner.retry_shutdown.notify_waiters();+ }+}++impl MultiWriteWrappedFS {+ /// Stop background retry work and wait for in-flight async fanout to drain.+ pub async fn shutdown(&self) -> Result<()> {+ self.inner.retry_cancelled.store(true, Ordering::SeqCst);+ self.inner.retry_shutdown.notify_waiters();+ self.inner.wait_idle(DEFAULT_SHUTDOWN_WAIT).await+ }++ /// Copy one primary-resident file to a new path while preserving physical primary bytes.+ pub async fn copy_within_primary(&self, src_path: &str, dst_path: &str) -> Result<bool> {+ if normalize_prefix_path(src_path) == normalize_prefix_path(dst_path) {+ return Ok(true);+ }++ let ctx = current_required_ctx()?;+ let inner = &self.inner;+ let source_path = src_path.to_string();+ let destination_path = dst_path.to_string();++ let source_exists_on_primary = FS_CTX+ .scope(ctx.clone(), async {+ inner.primary().backend.exists(&source_path).await+ })+ .await;+ if !source_exists_on_primary {+ return Ok(false);+ }++ let source_size = FS_CTX+ .scope(ctx.clone(), async {+ let source_info = inner.primary().backend.stat(&source_path).await?;+ if source_info.is_dir {+ return Err(Error::IsADirectory(source_path.clone()));+ }+ Ok::<u64, Error>(source_info.size)+ })+ .await?;++ if inner+ .check_redirect(&destination_path, source_size)+ .is_some()+ {+ return Ok(false);+ }++ let Some(primary_raw_backend) = inner.primary().raw_backend.clone() else {+ return Ok(false);+ };++ let primary_destination = destination_path.clone();+ Inner::execute_write(+ inner,+ destination_path,+ source_size,+ Some(SyncOp::SyncFile { size: source_size }),+ Some(BackupWriteOp::Replay(SyncOp::SyncFile {+ size: source_size,+ })),+ move |_ignored_fs| {+ let primary_raw_backend = primary_raw_backend.clone();+ let primary_source = source_path.clone();+ let primary_destination = primary_destination.clone();+ let primary_ctx = ctx.clone();+ async move {+ copy_raw_primary_state(+ primary_raw_backend,+ &primary_source,+ &primary_destination,+ &primary_ctx,+ )+ .await?;+ Ok(())+ }+ },+ )+ .await?;++ Ok(true)+ }++ /// Execute one non-redirecting write-like operation through the shared multi-write pipeline.+ async fn execute_simple_write<R, P, PFut>(+ &self,+ path: &str,+ size: u64,+ sync_op: Option<SyncOp>,+ primary_fn: P,+ ) -> Result<R>+ where+ R: Send + 'static,+ P: FnOnce(Arc<dyn FileSystem>, String) -> PFut + Send,+ PFut: Future<Output = Result<R>> + Send,+ {+ let path_owned = path.to_string();+ let primary_path = path_owned.clone();+ let backup_op = sync_op.clone().map(BackupWriteOp::Replay);+ Inner::execute_write(+ &self.inner,+ path_owned,+ size,+ sync_op,+ backup_op,+ move |fs| primary_fn(fs, primary_path),+ )+ .await+ }+}++#[async_trait]+impl FileSystem for MultiWriteWrappedFS {+ async fn create(&self, path: &str) -> Result<()> {+ self.execute_simple_write(path, 0, Some(SyncOp::Create), |fs, path| async move {+ fs.create(&path).await+ })+ .await+ }++ async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {+ self.execute_simple_write(+ path,+ 0,+ Some(SyncOp::Mkdir { mode }),+ move |fs, path| async move { fs.mkdir(&path, mode).await },+ )+ .await+ }++ async fn remove(&self, path: &str) -> Result<()> {+ self.execute_simple_write(path, 0, Some(SyncOp::Remove), |fs, path| async move {+ fs.remove(&path).await+ })+ .await+ }++ async fn remove_all(&self, path: &str) -> Result<()> {+ self.execute_simple_write(path, 0, Some(SyncOp::RemoveAll), |fs, path| async move {+ fs.remove_all(&path).await+ })+ .await+ }++ async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {+ if let Some(fs) = self.inner.resolve_read_backend(path).await {+ return fs.read(path, offset, size).await;+ }+ Err(Error::not_found(path))+ }++ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {+ let inner = &self.inner;+ let data_len = data.len() as u64;+ let path_owned = path.to_string();+ let backup_op = BackupWriteOp::WriteFile {+ data: Arc::new(data.to_vec()),+ offset,+ flags,+ };+ let d = data.to_vec();+ let primary_path = path_owned.clone();+ Inner::execute_write_with_redirect(+ inner,+ path_owned,+ data_len,+ SyncOp::SyncFile { size: data_len },+ backup_op,+ move |fs: Arc<dyn FileSystem>| async move {+ fs.write(&primary_path, &d, offset, flags).await+ },+ )+ .await+ }++ async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {+ let inner = &self.inner;+ let mut entries = inner.primary().backend.read_dir(path).await?;++ // Filter multi-write internal names.+ entries.retain(|e| !MULTIWRITE_INTERNAL_NAMES.contains(&e.name.as_str()));++ // Merge redirect entries so users can see redirected files in listings.+ let ctx =+ current_required_ctx().or_else(|_| inner.meta_store.ctx_resolver().resolve(path))?;++ if let Ok(redirect_meta) = inner.meta_store.get_redirect_meta(path, &ctx).await {+ for (name, redirect_entry) in &redirect_meta.entries {+ if !entries.iter().any(|e| &e.name == name) {+ let virtual_path = if path == "/" {+ format!("/{}", name)+ } else {+ format!("{}/{}", path.trim_end_matches('/'), name)+ };+ entries.push(+ inner+ .redirect_file_info(&virtual_path, name, redirect_entry)+ .await,+ );+ }+ }+ }++ Ok(entries)+ }++ async fn stat(&self, path: &str) -> Result<FileInfo> {+ if let Some(fs) = self.inner.resolve_read_backend(path).await {+ return fs.stat(path).await;+ }+ Err(Error::not_found(path))+ }++ async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {+ let ctx = current_required_ctx()?;+ let inner = &self.inner;+ let old_owned = old_path.to_string();+ let new_owned = new_path.to_string();+ let source_dir = parent_dir(&old_owned);+ let redirect_targets = inner.redirect_targets_for_path(&old_owned, &ctx).await?;+ inner.invalidate_read_route(&old_owned).await;+ inner.invalidate_read_route(&new_owned).await;++ let primary_has_old = inner.primary().backend.exists(&old_owned).await;+ if primary_has_old {+ FS_CTX+ .scope(ctx.clone(), async {+ inner.primary().backend.rename(&old_owned, &new_owned).await+ })+ .await?;+ } else if redirect_targets.is_none() {+ return Err(Error::NotFound(old_owned));+ }++ inner+ .record_rename_meta(&old_owned, &new_owned, &ctx)+ .await?;+ inner.mark_pending_dir(&source_dir).await;++ let fanout_path = old_owned.clone();+ let backup_op = BackupWriteOp::Replay(SyncOp::Rename {+ to: new_owned.clone(),+ });+ if let Some(targets) = redirect_targets {+ Inner::fanout_write_to_targets(inner, &fanout_path, &targets, ctx.clone(), backup_op)+ .await?;+ } else {+ Inner::fanout_write(inner, &fanout_path, 0, ctx.clone(), backup_op).await?;+ }+ inner.refresh_pending_dir(&source_dir, &ctx).await?;++ Ok(())+ }++ async fn chmod(&self, path: &str, mode: u32) -> Result<()> {+ self.execute_simple_write(+ path,+ 0,+ Some(SyncOp::Chmod { mode }),+ move |fs, path| async move { fs.chmod(&path, mode).await },+ )+ .await+ }++ async fn truncate(&self, path: &str, size: u64) -> Result<()> {+ self.execute_simple_write(+ path,+ size,+ Some(SyncOp::SyncFile { size }),+ move |fs, path| async move { fs.truncate(&path, size).await },+ )+ .await+ }++ async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {+ let path_owned = path.to_string();+ let primary_path = path_owned.clone();+ Inner::execute_write(+ &self.inner,+ path_owned,+ 0,+ None,+ Some(BackupWriteOp::EnsureParentDirs { mode }),+ move |fs| async move { fs.ensure_parent_dirs(&primary_path, mode).await },+ )+ .await+ }++ async fn grep(+ &self,+ path: &str,+ pattern: &str,+ recursive: bool,+ case_insensitive: bool,+ node_limit: Option<usize>,+ exclude_path: Option<&str>,+ level_limit: Option<usize>,+ ) -> Result<GrepResult> {+ let inner = &self.inner;+ let path_owned = path.to_string();+ let pattern_owned = pattern.to_string();+ let exclude_owned = exclude_path.map(|s| s.to_string());++ let mut result = inner+ .primary()+ .backend+ .grep(+ &path_owned,+ &pattern_owned,+ recursive,+ case_insensitive,+ node_limit,+ exclude_owned.as_deref(),+ level_limit,+ )+ .await?;++ // Filter out multi-write internal metadata files from grep results.+ result.matches.retain(|m| {+ let file_name = m.file.rsplit('/').next().unwrap_or(&m.file);+ !MULTIWRITE_INTERNAL_NAMES.contains(&file_name)+ });+ result.count = result.matches.len();++ // For redirect files, also grep in target backends.+ let ctx = current_required_ctx()+ .or_else(|_| inner.meta_store.ctx_resolver().resolve(&path_owned))?;++ let search_dir = if inner+ .primary()+ .backend+ .stat(&path_owned)+ .await+ .map(|s| s.is_dir)+ .unwrap_or(false)+ {+ path_owned.clone()+ } else {+ parent_dir(&path_owned)+ };++ if recursive && search_dir == path_owned {+ let redirect_entries = self+ .tree_directory(&path_owned, true, None, level_limit)+ .await?;+ for entry in redirect_entries {+ if node_limit.is_some_and(|limit| result.count >= limit) {+ break;+ }+ if entry.info.is_dir+ || entry.extra.get("redirect").and_then(Value::as_bool) != Some(true)+ {+ continue;+ }+ if is_excluded_grep_path(&entry.path, exclude_owned.as_deref()) {+ continue;+ }+ let Some(read_backend) = inner.resolve_read_backend(&entry.path).await else {+ continue;+ };+ let rel_path = relative_match_file(&path_owned, &entry.path);+ let target_result = match read_backend+ .grep(+ &entry.path,+ &pattern_owned,+ false,+ case_insensitive,+ node_limit.map(|limit| limit.saturating_sub(result.count)),+ None,+ None,+ )+ .await+ {+ Ok(found) => found,+ Err(_) => continue,+ };+ for m in target_result.matches {+ if node_limit.is_some_and(|limit| result.count >= limit) {+ break;+ }+ result.add_match(rel_path.clone(), m.line, m.content);+ }+ }+ return Ok(result);+ }++ if let Ok(redirect_meta) = inner.meta_store.get_redirect_meta(&search_dir, &ctx).await {+ for (name, redirect_entry) in &redirect_meta.entries {+ for target_name in &redirect_entry.targets {+ if let Some(be) = inner.backup_by_name(target_name) {+ let redirect_path = if search_dir == "/" {+ format!("/{}", name)+ } else {+ format!("{}/{}", search_dir, name)+ };+ if let Ok(target_result) = be+ .backend+ .grep(+ &redirect_path,+ &pattern_owned,+ false,+ case_insensitive,+ node_limit,+ None,+ None,+ )+ .await+ {+ let rel_path = relative_match_file(&path_owned, &redirect_path);+ for m in target_result.matches {+ if node_limit.is_some_and(|limit| result.count >= limit) {+ break;+ }+ result.add_match(rel_path.clone(), m.line, m.content);+ }+ }+ }+ }+ }+ }++ Ok(result)+ }++ async fn tree_directory(+ &self,+ path: &str,+ show_hidden: bool,+ node_limit: Option<usize>,+ level_limit: Option<usize>,+ ) -> Result<Vec<TreeEntry>> {+ let base = normalize_prefix_path(path);+ let mut entries = self+ .inner+ .primary()+ .backend+ .tree_directory(path, show_hidden, node_limit, level_limit)+ .await?;++ entries.retain(|e| {+ let name = file_name(&e.path);+ !MULTIWRITE_INTERNAL_NAMES.contains(&name)+ });++ let ctx = current_required_ctx()+ .or_else(|_| self.inner.meta_store.ctx_resolver().resolve(&base))?;+ let mut seen_paths: HashSet<String> = entries.iter().map(|e| e.path.clone()).collect();+ let mut dir_paths = vec![base.clone()];+ for entry in &entries {+ if entry.info.is_dir {+ let dir = normalize_prefix_path(&entry.path);+ if !dir_paths.iter().any(|p| p == &dir) {+ dir_paths.push(dir);+ }+ }+ }++ for dir in dir_paths {+ let redirect_meta = match self.inner.meta_store.get_redirect_meta(&dir, &ctx).await {+ Ok(meta) => meta,+ Err(_) => continue,+ };+ for (name, redirect_entry) in redirect_meta.entries {+ let virtual_path = if dir == "/" {+ format!("/{}", name)+ } else {+ format!("{}/{}", dir, name)+ };+ if seen_paths.contains(&virtual_path) {+ continue;+ }+ let rel_path = if base == "/" {+ virtual_path.trim_start_matches('/').to_string()+ } else {+ virtual_path+ .strip_prefix(&base)+ .unwrap_or(&virtual_path)+ .trim_start_matches('/')+ .to_string()+ };+ let mut extra = HashMap::new();+ extra.insert("redirect".to_string(), Value::Bool(true));+ entries.push(TreeEntry {+ path: virtual_path.clone(),+ rel_path,+ info: self+ .inner+ .redirect_file_info(&virtual_path, &name, &redirect_entry)+ .await,+ extra,+ });+ seen_paths.insert(virtual_path);+ }+ }++ Ok(entries)+ }+}diff --git a/crates/ragfs/src/core/multibackend_wrapper/retry.rs b/crates/ragfs/src/core/multibackend_wrapper/retry.rsnew file mode 100644index 000000000..c4581bf22--- /dev/null+++ b/crates/ragfs/src/core/multibackend_wrapper/retry.rs@@ -0,0 +1,270 @@+use std::time::{Duration, Instant};++use rand::Rng;+use tracing::warn;++use super::*;++impl Inner {+ /// Mark one directory as pending retry work.+ pub(super) async fn mark_pending_dir(&self, dir: &str) {+ self.pending_dirs.lock().await.insert(dir.to_string());+ }++ /// Remove one directory from the retry set.+ async fn clear_pending_dir(&self, dir: &str) {+ self.pending_dirs.lock().await.remove(dir);+ }++ /// Snapshot the current set of pending retry directories.+ pub(crate) async fn pending_dirs_snapshot(&self) -> Vec<String> {+ self.pending_dirs.lock().await.iter().cloned().collect()+ }++ /// Recompute whether one directory still has pending retry work.+ pub(super) async fn refresh_pending_dir(&self, dir: &str, ctx: &FsContext) -> Result<()> {+ if self.sync_work_in_dir(dir, ctx).await?.iter().any(|work| {+ work.entry.is_primary_committed()+ && work.targets.iter().any(|target| {+ !work.entry.is_in_sync(target) && !work.entry.is_quarantined(target)+ })+ }) {+ self.mark_pending_dir(dir).await;+ } else {+ self.clear_pending_dir(dir).await;+ }+ Ok(())+ }++ /// Collect sync work entries for one metadata directory.+ pub(crate) async fn sync_work_in_dir(+ &self,+ dir: &str,+ ctx: &FsContext,+ ) -> Result<Vec<SyncWorkEntry>> {+ let sync_log = self.meta_store.get_sync_log_meta(dir, ctx).await?;+ if sync_log.entries.is_empty() {+ return Ok(Vec::new());+ }++ let redirect_meta = self.meta_store.get_redirect_meta(dir, ctx).await?;+ Ok(sync_log+ .entries+ .into_iter()+ .map(|(file_name, entry)| {+ let file_path = if dir == "/" {+ format!("/{}", file_name)+ } else {+ format!("{}/{}", dir, file_name)+ };+ let targets =+ self.target_backend_names(&redirect_meta, &file_name, &file_path, &entry);+ SyncWorkEntry {+ file_path,+ entry,+ targets,+ }+ })+ .collect())+ }++ /// Record a started background task.+ pub(super) fn background_task_started(&self) {+ self.background_tasks.fetch_add(1, Ordering::SeqCst);+ }++ /// Record a finished background task and wake idle waiters if needed.+ pub(super) fn background_task_finished(&self) {+ if self.background_tasks.fetch_sub(1, Ordering::SeqCst) == 1 {+ self.idle_notify.notify_waiters();+ }+ }++ /// Wait for all background tasks to drain.+ pub(super) async fn wait_idle(&self, timeout: Duration) -> Result<()> {+ let deadline = Instant::now() + timeout;+ loop {+ if self.background_tasks.load(Ordering::SeqCst) == 0 {+ return Ok(());+ }+ let now = Instant::now();+ if now >= deadline {+ return Err(Error::timeout(+ "timed out while waiting multi-write background tasks to drain",+ ));+ }+ let wait = deadline.saturating_duration_since(now);+ tokio::time::timeout(wait, self.idle_notify.notified())+ .await+ .map_err(|_| {+ Error::timeout("timed out while waiting multi-write background tasks to drain")+ })?;+ }+ }++ /// Update the acked_seq for a backup in the sync log.+ pub(super) async fn update_backup_acked_seq(+ &self,+ path: &str,+ backup_name: &str,+ ctx: &FsContext,+ ) -> Result<()> {+ self.update_backend_state(path, backup_name, ctx, |state, latest_seq| {+ state.mark_acked(latest_seq);+ })+ .await?;+ self.refresh_pending_dir(&parent_dir(path), ctx).await+ }++ /// Record a replay failure and quarantine the target after repeated failures.+ pub(crate) async fn record_backup_retry_failure(+ &self,+ path: &str,+ backup_name: &str,+ ctx: &FsContext,+ ) -> Result<()> {+ self.update_backend_state(path, backup_name, ctx, |state, _latest_seq| {+ state.mark_retry_failed(self.quarantine_after_failures);+ })+ .await?;+ self.refresh_pending_dir(&parent_dir(path), ctx).await+ }++ /// Update per-backend sync state for a file entry.+ async fn update_backend_state<F>(+ &self,+ path: &str,+ backup_name: &str,+ ctx: &FsContext,+ update: F,+ ) -> Result<()>+ where+ F: FnOnce(&mut BackendSyncState, u64) + Send,+ {+ let dir = parent_dir(path);+ let backup_name = backup_name.to_string();+ let name = file_name(path).to_string();+ self.meta_store+ .update_dir_meta(&dir, ctx, move |_redirect, sync_log| {+ if let Some(entry) = sync_log.entries.get_mut(&name) {+ let state = entry.backends.entry(backup_name).or_default();+ update(state, entry.latest_seq);+ }+ Ok(())+ })+ .await+ }++ /// Replay a single operation on a lagging backup.+ pub(crate) async fn replay_operation(+ &self,+ file_path: &str,+ backup_name: &str,+ ctx: &FsContext,+ ) -> Result<()> {+ let queue_key = Self::backup_queue_key(file_path, backup_name);+ self.path_queues+ .with_path_lock(&queue_key, || async {+ let dir = parent_dir(file_path);+ let name = file_name(file_path).to_string();+ let sync_log = self.meta_store.get_sync_log_meta(&dir, ctx).await?;+ let entry = sync_log+ .entries+ .get(&name)+ .ok_or_else(|| Error::not_found(file_path))?+ .clone();+ let backup = match self.backup_by_name(backup_name) {+ Some(b) => b,+ None => {+ return Err(Error::internal(format!(+ "backup '{}' not found",+ backup_name+ )))+ }+ };++ entry+ .op+ .replay(+ self.primary().backend.clone(),+ backup.backend.clone(),+ file_path,+ ctx,+ )+ .await?;++ self.update_backup_acked_seq(file_path, backup_name, ctx)+ .await?;++ Ok(())+ })+ .await+ }++ /// Retry all pending backup work for one directory and keep metadata errors visible.+ pub(crate) async fn retry_pending_dir(&self, dir: &str) -> Result<()> {+ let ctx = self.meta_store.ctx_resolver().resolve(dir)?;++ for work in self.sync_work_in_dir(dir, &ctx).await? {+ if !work.entry.is_primary_committed() {+ continue;+ }++ for backup_name in &work.targets {+ if work.entry.is_in_sync(backup_name) || work.entry.is_quarantined(backup_name) {+ continue;+ }++ let mut success = false;+ for attempt in 0..self.max_retry_per_round {+ if self.retry_cancelled.load(Ordering::SeqCst) {+ return Ok(());+ }+ if self+ .replay_operation(&work.file_path, backup_name, &ctx)+ .await+ .is_ok()+ {+ success = true;+ break;+ }+ let base_ms = self.retry_backoff_base_ms.saturating_mul(1u64 << attempt);+ let jitter_ms = rand::thread_rng().gen_range(0..=50);+ tokio::time::sleep(Duration::from_millis(base_ms + jitter_ms)).await;+ }++ if !success {+ self.record_backup_retry_failure(&work.file_path, backup_name, &ctx)+ .await?;+ }+ }+ }++ self.refresh_pending_dir(dir, &ctx).await+ }++ /// Background retry loop: periodically scans sync_log for lagging backups and replays.+ pub(super) async fn retry_loop(inner: Arc<Inner>) {+ loop {+ if inner.retry_cancelled.load(Ordering::SeqCst) {+ break;+ }++ tokio::select! {+ _ = tokio::time::sleep(inner.retry_interval) => {}+ _ = inner.retry_shutdown.notified() => break,+ }+ let dirs = inner.pending_dirs_snapshot().await;+ for dir in dirs {+ if let Err(err) = inner.retry_pending_dir(&dir).await {+ warn!(+ dir = %dir,+ error = %err,+ "multi-write retry skipped one pending directory due to metadata or replay error"+ );+ }+ }+ }+ inner.background_task_finished();+ }+}diff --git a/crates/ragfs/src/core/multibackend_wrapper/routing.rs b/crates/ragfs/src/core/multibackend_wrapper/routing.rsnew file mode 100644index 000000000..8de9babf2--- /dev/null+++ b/crates/ragfs/src/core/multibackend_wrapper/routing.rs@@ -0,0 +1,120 @@+use std::sync::Arc;++use serde_json::{json, Value};++use super::*;+use crate::multibackend::meta::{current_required_ctx, file_name, parent_dir};++impl Inner {+ /// Keep write-side call sites explicit even though read routing is uncached.+ pub(super) async fn invalidate_read_route(&self, _path: &str) {}++ /// Record read-route counters in one place so hot paths stay explicit.+ fn record_read_route(&self, source: ReadRouteSource) {+ match source {+ ReadRouteSource::Backup => {+ self.read_backup_hits.fetch_add(1, Ordering::Relaxed);+ }+ ReadRouteSource::Primary => {+ self.read_primary_hits.fetch_add(1, Ordering::Relaxed);+ }+ ReadRouteSource::Redirect => {+ self.read_redirect_hits.fetch_add(1, Ordering::Relaxed);+ }+ ReadRouteSource::Miss => {+ self.read_misses.fetch_add(1, Ordering::Relaxed);+ }+ }+ }++ /// Export read-route metrics for operational introspection.+ pub(crate) fn read_route_metrics(&self) -> Value {+ json!({+ "cache_hits": 0,+ "backup_hits": self.read_backup_hits.load(Ordering::Relaxed),+ "primary_hits": self.read_primary_hits.load(Ordering::Relaxed),+ "redirect_hits": self.read_redirect_hits.load(Ordering::Relaxed),+ "misses": self.read_misses.load(Ordering::Relaxed),+ })+ }++ /// Stat the first reachable redirect target and return user-visible metadata.+ pub(super) async fn redirect_file_info(+ &self,+ path: &str,+ name: &str,+ redirect_entry: &RedirectEntry,+ ) -> FileInfo {+ for target_name in &redirect_entry.targets {+ if let Some(be) = self.backup_by_name(target_name) {+ if let Ok(mut info) = be.backend.stat(path).await {+ info.name = name.to_string();+ return info;+ }+ }+ }+ FileInfo::new_file(name.to_string(), 0, 0o644)+ }++ /// Resolve the read backend for a path using the fallback chain.+ pub(super) async fn resolve_read_backend(&self, path: &str) -> Option<Arc<dyn FileSystem>> {+ let normalized = normalize_prefix_path(path);+ let read_backups = self.read_backups_sorted();+ let backup_exists = futures::future::join_all(read_backups.iter().map(|backup| async {+ (+ backup.name.clone(),+ backup.backend.clone(),+ backup.backend.exists(&normalized).await,+ )+ }))+ .await;+ for (_name, backend, exists) in backup_exists {+ if exists {+ self.record_read_route(ReadRouteSource::Backup);+ return Some(backend);+ }+ }++ if self.primary().backend.exists(&normalized).await {+ self.record_read_route(ReadRouteSource::Primary);+ return Some(self.primary().backend.clone());+ }++ let dir = parent_dir(&normalized);+ let name = file_name(&normalized).to_string();+ let ctx = current_required_ctx()+ .or_else(|_| self.meta_store.ctx_resolver().resolve(&dir))+ .ok()?;+ if let Ok(redirect_meta) = self.meta_store.get_redirect_meta(&dir, &ctx).await {+ if let Some(entry) = redirect_meta.entries.get(&name) {+ let redirect_targets: Vec<(String, Arc<dyn FileSystem>)> = entry+ .targets+ .iter()+ .filter_map(|target_name| {+ self.backup_by_name(target_name)+ .map(|be| (be.name.clone(), be.backend.clone()))+ })+ .collect();+ let redirect_exists = futures::future::join_all(redirect_targets.iter().map(+ |(target_name, backend)| async {+ (+ target_name.clone(),+ backend.clone(),+ backend.exists(&normalized).await,+ )+ },+ ))+ .await;+ for (_target_name, backend, exists) in redirect_exists {+ if exists {+ self.record_read_route(ReadRouteSource::Redirect);+ return Some(backend);+ }+ }+ }+ }++ self.record_read_route(ReadRouteSource::Miss);+ None+ }+}diff --git a/crates/ragfs/src/core/multibackend_wrapper/tests.rs b/crates/ragfs/src/core/multibackend_wrapper/tests.rsnew file mode 100644index 000000000..357c56fcf--- /dev/null+++ b/crates/ragfs/src/core/multibackend_wrapper/tests.rs@@ -0,0 +1,774 @@+use std::sync::Arc;++use serde_json::Value;++use super::*;+use crate::core::context::FsContextInner;+use crate::plugins::memfs::MemFileSystem;++/// Create the shared test context used by multi-write tests.+fn test_ctx() -> FsContext {+ Arc::new(FsContextInner::new("acct".to_string()))+}++/// Create a sync multi-write filesystem with one memfs backup.+fn test_multiwrite_fs(redirects: Vec<RedirectPolicy>) -> MultiWriteWrappedFS {+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let builder = MultiWriteWrappedFS::builder(primary.clone())+ .with_primary_raw_backend(primary)+ .with_backups(vec![BackendEntry {+ name: "backup1".to_string(),+ role: BackendRole::Backup,+ backend: backup,+ raw_backend: None,+ operations: Vec::new(),+ excludes: Vec::new(),+ }])+ .sync_mode(SyncMode::Sync {+ ack_count: 1,+ timeout_ms: 0,+ });++ if redirects.is_empty() {+ builder+ } else {+ builder.with_redirects(redirects)+ }+ .build()+ .unwrap()+}++#[test]+fn test_file_policy_over_size() {+ let policy = RedirectPolicy::FileOverSizePolicy {+ max_size_mb: 1,+ target: Some(vec!["backup1".to_string()]),+ };+ assert!(policy.matches("/a/big.bin", 2 * 1024 * 1024));+ assert!(!policy.matches("/a/small.txt", 512));+}++#[test]+fn test_file_policy_extension() {+ let policy = RedirectPolicy::FileExtensionPolicy {+ extensions: vec!["(pdf|ppt)".to_string()],+ target: Some(vec!["backup1".to_string()]),+ };+ assert!(policy.matches("/a/doc.pdf", 0));+ assert!(policy.matches("/a/slides.ppt", 0));+ assert!(!policy.matches("/a/text.txt", 0));+}++#[tokio::test]+async fn test_read_dir_redirect_entries_use_target_stat() {+ let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {+ extensions: vec!["\\.pdf$".to_string()],+ target: Some(vec!["backup1".to_string()]),+ }]);+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx, async {+ fs.ensure_parent_dirs("/local/acct/docs/report.pdf", 0o755)+ .await?;+ fs.write(+ "/local/acct/docs/report.pdf",+ b"pdf body",+ 0,+ WriteFlag::Create,+ )+ .await?;++ let entries = fs.read_dir("/local/acct/docs").await?;+ let report = entries+ .iter()+ .find(|entry| entry.name == "report.pdf")+ .expect("redirected file should be visible in read_dir");+ assert_eq!(report.size, 8);+ assert_eq!(report.mode, 0o644);+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_recursive_grep_finds_nested_redirected_files() {+ let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {+ extensions: vec!["\\.md$".to_string()],+ target: Some(vec!["backup1".to_string()]),+ }]);+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx, async {+ fs.ensure_parent_dirs("/local/acct/resources/doc-1/page.md", 0o755)+ .await?;+ fs.write(+ "/local/acct/resources/doc-1/page.md",+ b"MULTIWRITE_TEST_MARKER\n",+ 0,+ WriteFlag::Create,+ )+ .await?;++ let result = fs+ .grep(+ "/local/acct/resources",+ "MULTIWRITE_TEST_MARKER",+ true,+ false,+ None,+ None,+ Some(5),+ )+ .await?;+ assert_eq!(result.count, 1);+ assert_eq!(result.matches.len(), 1);+ assert_eq!(result.matches[0].file, "doc-1/page.md");+ assert_eq!(result.matches[0].line, 1);+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_redirect_write_marks_first_target_acked_before_returning() {+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup1: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup2: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup1_handle = backup1.clone();+ let fs = MultiWriteWrappedFS::builder(primary.clone())+ .with_primary_raw_backend(primary)+ .with_backups(vec![+ BackendEntry {+ name: "backup1".to_string(),+ role: BackendRole::Backup,+ backend: backup1,+ raw_backend: None,+ operations: Vec::new(),+ excludes: Vec::new(),+ },+ BackendEntry {+ name: "backup2".to_string(),+ role: BackendRole::Backup,+ backend: backup2,+ raw_backend: None,+ operations: Vec::new(),+ excludes: Vec::new(),+ },+ ])+ .with_redirects(vec![RedirectPolicy::FileExtensionPolicy {+ extensions: vec!["\\.pdf$".to_string()],+ target: Some(vec!["backup1".to_string(), "backup2".to_string()]),+ }])+ .sync_mode(SyncMode::Async)+ .build()+ .unwrap();+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx.clone(), async {+ fs.ensure_parent_dirs("/local/acct/docs/report.pdf", 0o755)+ .await?;+ fs.write(+ "/local/acct/docs/report.pdf",+ b"pdf body",+ 0,+ WriteFlag::Create,+ )+ .await?;++ let redirect_meta = fs+ .inner+ .meta_store+ .get_redirect_meta("/local/acct/docs", &ctx)+ .await?;+ let entry = redirect_meta+ .entries+ .get("report.pdf")+ .expect("redirect metadata should only appear after one target succeeds");+ assert_eq!(+ entry.targets,+ vec!["backup1".to_string(), "backup2".to_string()]+ );++ let sync_log = fs+ .inner+ .meta_store+ .get_sync_log_meta("/local/acct/docs", &ctx)+ .await?;+ let log_entry = sync_log+ .entries+ .get("report.pdf")+ .expect("redirected file should be tracked in sync log");+ assert!(log_entry.is_primary_committed());+ assert!(+ log_entry.is_in_sync("backup1"),+ "the first redirect target must be acknowledged before returning"+ );+ assert!(+ backup1_handle.exists("/local/acct/docs/report.pdf").await,+ "the first redirect target must already contain the file"+ );+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_redirect_write_does_not_publish_metadata_when_first_target_fails() {+ let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {+ extensions: vec!["\\.pdf$".to_string()],+ target: Some(vec!["missing-backup".to_string()]),+ }]);+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx.clone(), async {+ let err = fs+ .write(+ "/local/acct/docs/fail.pdf",+ b"content",+ 0,+ WriteFlag::Create,+ )+ .await+ .expect_err("redirect write should fail when the first target is not durable");++ assert!(+ err.to_string().contains("resolved no writable targets"),+ "unexpected error: {err}"+ );++ let redirect_meta = fs+ .inner+ .meta_store+ .get_redirect_meta("/local/acct/docs", &ctx)+ .await?;+ assert!(+ !redirect_meta.entries.contains_key("fail.pdf"),+ "redirect metadata must not be published before the first target succeeds"+ );++ let sync_log = fs+ .inner+ .meta_store+ .get_sync_log_meta("/local/acct/docs", &ctx)+ .await?;+ assert!(+ !sync_log.entries.contains_key("fail.pdf"),+ "sync log must not record a committed redirect write before durability"+ );+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_cross_dir_rename_does_not_create_target_write_log() {+ let fs = test_multiwrite_fs(Vec::new());+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx.clone(), async {+ fs.ensure_parent_dirs("/local/acct/src/file.txt", 0o755)+ .await?;+ fs.ensure_parent_dirs("/local/acct/dst/file.txt", 0o755)+ .await?;+ fs.write(+ "/local/acct/src/file.txt",+ b"rename me",+ 0,+ WriteFlag::Create,+ )+ .await?;+ fs.rename("/local/acct/src/file.txt", "/local/acct/dst/file.txt")+ .await?;++ let target_log = fs+ .inner+ .meta_store+ .get_sync_log_meta("/local/acct/dst", &ctx)+ .await?;+ assert!(+ !target_log.entries.contains_key("file.txt"),+ "target directory must not record a fake write for cross-dir rename"+ );+ let source_log = fs+ .inner+ .meta_store+ .get_sync_log_meta("/local/acct/src", &ctx)+ .await?;+ let rename_entry = source_log+ .entries+ .get("file.txt")+ .expect("source directory should record rename command");+ assert!(matches!(+ &rename_entry.op,+ SyncOp::Rename { to } if to == "/local/acct/dst/file.txt"+ ));+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_rename_redirected_file_renames_original_redirect_target() {+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup_handle = backup.clone();+ let fs = MultiWriteWrappedFS::builder(primary.clone())+ .with_primary_raw_backend(primary)+ .with_backups(vec![BackendEntry {+ name: "backup1".to_string(),+ role: BackendRole::Backup,+ backend: backup,+ raw_backend: None,+ operations: vec![OperationItemConfig {+ operation: "read".to_string(),+ priority: 1,+ }],+ excludes: Vec::new(),+ }])+ .with_redirects(vec![RedirectPolicy::FileExtensionPolicy {+ extensions: vec!["\\.pdf$".to_string()],+ target: Some(vec!["backup1".to_string()]),+ }])+ .sync_mode(SyncMode::Sync {+ ack_count: 1,+ timeout_ms: 0,+ })+ .build()+ .unwrap();+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx, async {+ fs.ensure_parent_dirs("/local/acct/src/a.pdf", 0o755)+ .await?;+ fs.ensure_parent_dirs("/local/acct/dst/b.pdf", 0o755)+ .await?;+ fs.write(+ "/local/acct/src/a.pdf",+ b"pdf payload",+ 0,+ WriteFlag::Create,+ )+ .await?;+ assert!(backup_handle.exists("/local/acct/src/a.pdf").await);++ fs.rename("/local/acct/src/a.pdf", "/local/acct/dst/b.pdf")+ .await?;++ assert!(+ backup_handle.exists("/local/acct/dst/b.pdf").await,+ "redirect target backend should receive rename fanout"+ );+ assert!(+ !backup_handle.exists("/local/acct/src/a.pdf").await,+ "redirect target backend should not retain the old path"+ );+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_copy_within_primary_preserves_primary_bytes_and_fanouts_backups() {+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup_handle = backup.clone();+ let fs = MultiWriteWrappedFS::builder(primary.clone())+ .with_primary_raw_backend(primary)+ .with_backups(vec![BackendEntry {+ name: "backup1".to_string(),+ role: BackendRole::Backup,+ backend: backup,+ raw_backend: None,+ operations: Vec::new(),+ excludes: Vec::new(),+ }])+ .sync_mode(SyncMode::Sync {+ ack_count: 1,+ timeout_ms: 0,+ })+ .build()+ .unwrap();+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx.clone(), async {+ fs.ensure_parent_dirs("/local/acct/docs/src.bin", 0o755)+ .await?;+ fs.write("/local/acct/docs/src.bin", b"payload", 0, WriteFlag::Create)+ .await?;++ let copied = fs+ .copy_within_primary("/local/acct/docs/src.bin", "/local/acct/docs/dst.bin")+ .await?;+ assert!(+ copied,+ "copy_within_primary should use the primary fast-path"+ );++ assert_eq!(+ fs.inner+ .primary()+ .backend+ .read("/local/acct/docs/dst.bin", 0, 0)+ .await?,+ b"payload"+ );+ assert_eq!(+ backup_handle.read("/local/acct/docs/dst.bin", 0, 0).await?,+ b"payload"+ );++ let sync_log = fs+ .inner+ .meta_store+ .get_sync_log_meta("/local/acct/docs", &ctx)+ .await?;+ let entry = sync_log+ .entries+ .get("dst.bin")+ .expect("copied file should be tracked for backup fanout");+ assert!(entry.is_primary_committed());+ assert!(matches!(entry.op, SyncOp::SyncFile { size } if size == 7));+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_sync_log_write_entry_does_not_embed_large_payload() {+ let fs = test_multiwrite_fs(Vec::new());+ let ctx = test_ctx();+ let payload = vec![b'x'; 256 * 1024];++ FS_CTX+ .scope(ctx.clone(), async {+ fs.ensure_parent_dirs("/local/acct/docs/large.bin", 0o755)+ .await?;+ fs.write("/local/acct/docs/large.bin", &payload, 0, WriteFlag::Create)+ .await?;++ let sync_log = fs+ .inner+ .meta_store+ .get_sync_log_meta("/local/acct/docs", &ctx)+ .await?;+ let encoded = serde_json::to_vec(&sync_log)?;+ assert!(+ encoded.len() < 16 * 1024,+ "sync log should stay metadata-sized, got {} bytes",+ encoded.len()+ );+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_sync_log_truncate_entry_does_not_embed_snapshot() {+ let fs = test_multiwrite_fs(Vec::new());+ let ctx = test_ctx();+ let payload = vec![b'y'; 256 * 1024];++ FS_CTX+ .scope(ctx.clone(), async {+ fs.ensure_parent_dirs("/local/acct/docs/large.bin", 0o755)+ .await?;+ fs.write("/local/acct/docs/large.bin", &payload, 0, WriteFlag::Create)+ .await?;+ fs.truncate("/local/acct/docs/large.bin", 128).await?;++ let sync_log = fs+ .inner+ .meta_store+ .get_sync_log_meta("/local/acct/docs", &ctx)+ .await?;+ let encoded = serde_json::to_vec(&sync_log)?;+ assert!(+ encoded.len() < 16 * 1024,+ "truncate sync log should stay metadata-sized, got {} bytes",+ encoded.len()+ );+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_fanout_sync_returns_structured_quorum_error() {+ let fs = test_multiwrite_fs(Vec::new());+ let ctx = test_ctx();+ let targets = fs.inner.write_targets("/local/acct/docs/fail.txt", 0);++ let err = Inner::fanout_sync(+ &fs.inner,+ "/local/acct/docs/fail.txt",+ &targets,+ &ctx,+ BackupWriteOp::Replay(SyncOp::SyncFile { size: 1 }),+ )+ .await+ .expect_err("sync fanout should surface structured quorum failure");++ match err {+ Error::SyncWriteQuorum {+ succeeded,+ required,+ attempted,+ failures,+ } => {+ assert_eq!(succeeded, 0);+ assert_eq!(required, 1);+ assert_eq!(attempted, 1);+ assert_eq!(failures.len(), 1);+ assert_eq!(failures[0].backend, "backup1");+ assert_eq!(failures[0].kind, "not_found");+ assert_eq!(failures[0].message, "not found: /local/acct/docs/fail.txt");+ }+ other => panic!("unexpected error: {other}"),+ }+}++#[tokio::test]+async fn test_sync_write_errors_when_required_ack_has_no_targets() {+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let fs = MultiWriteWrappedFS::builder(primary.clone())+ .with_primary_raw_backend(primary)+ .with_backups(vec![BackendEntry {+ name: "backup1".to_string(),+ role: BackendRole::Backup,+ backend: backup,+ raw_backend: None,+ operations: Vec::new(),+ excludes: vec![RedirectPolicy::FileExtensionPolicy {+ extensions: vec!["\\.txt$".to_string()],+ target: None,+ }],+ }])+ .sync_mode(SyncMode::Sync {+ ack_count: 1,+ timeout_ms: 0,+ })+ .build()+ .unwrap();+ let ctx = test_ctx();++ let err = FS_CTX+ .scope(ctx, async {+ fs.ensure_parent_dirs("/local/acct/docs/no-target.txt", 0o755)+ .await?;+ fs.write(+ "/local/acct/docs/no-target.txt",+ b"payload",+ 0,+ WriteFlag::Create,+ )+ .await+ })+ .await+ .expect_err("sync mode must not silently pass when all backups are excluded");++ match err {+ Error::SyncWriteQuorum {+ succeeded,+ required,+ attempted,+ failures,+ } => {+ assert_eq!(succeeded, 0);+ assert_eq!(required, 1);+ assert_eq!(attempted, 0);+ assert!(failures.is_empty());+ }+ other => panic!("unexpected error: {other}"),+ }+}++#[tokio::test]+async fn test_read_route_metrics_capture_backup_fallbacks() {+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let backup_handle = backup.clone();+ let fs = MultiWriteWrappedFS::builder(primary)+ .with_backups(vec![BackendEntry {+ name: "backup1".to_string(),+ role: BackendRole::Backup,+ backend: backup,+ raw_backend: None,+ operations: vec![OperationItemConfig {+ operation: "read".to_string(),+ priority: 1,+ }],+ excludes: Vec::new(),+ }])+ .build()+ .unwrap();+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx, async {+ backup_handle+ .ensure_parent_dirs("/local/acct/hot/cache.txt", 0o755)+ .await?;+ backup_handle+ .write("/local/acct/hot/cache.txt", b"hot", 0, WriteFlag::Create)+ .await?;++ assert_eq!(fs.read("/local/acct/hot/cache.txt", 0, 0).await?, b"hot");+ assert_eq!(fs.read("/local/acct/hot/cache.txt", 0, 0).await?, b"hot");++ let metrics = fs.inner.read_route_metrics();+ assert_eq!(metrics.get("backup_hits").and_then(Value::as_u64), Some(2));+ assert_eq!(metrics.get("cache_hits").and_then(Value::as_u64), Some(0));+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_shutdown_drains_background_retry_loop() {+ let fs = test_multiwrite_fs(Vec::new());+ fs.shutdown().await.unwrap();+}++#[tokio::test]+async fn test_system_sync_status_reports_corrupt_redirect_meta() {+ let fs = test_multiwrite_fs(Vec::new());+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx.clone(), async {+ fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)+ .await?;+ fs.write("/local/acct/docs/a.txt", b"hello", 0, WriteFlag::Create)+ .await?;+ fs.inner+ .primary()+ .backend+ .write(+ "/local/acct/docs/.redirect.json",+ b"{not-json",+ 0,+ WriteFlag::Create,+ )+ .await?;++ let err = fs+ .system_sync_status("/local/acct/docs/a.txt")+ .await+ .expect_err("corrupt redirect metadata must be reported");+ assert!(+ err.to_string().contains("redirect")+ || err.to_string().contains("json")+ || err.to_string().contains("serialization"),+ "unexpected error: {err}"+ );+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_retry_pending_dir_reports_corrupt_redirect_meta() {+ let fs = test_multiwrite_fs(Vec::new());+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx.clone(), async {+ fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)+ .await?;+ fs.write("/local/acct/docs/a.txt", b"hello", 0, WriteFlag::Create)+ .await?;+ fs.inner+ .meta_store+ .update_dir_meta("/local/acct/docs", &ctx, |_redirect, sync_log| {+ sync_log+ .entries+ .insert("a.txt".to_string(), SyncLogEntry::new(1, SyncOp::Create));+ Ok(())+ })+ .await?;+ fs.inner+ .primary()+ .backend+ .write(+ "/local/acct/docs/.redirect.json",+ b"{not-json",+ 0,+ WriteFlag::Create,+ )+ .await?;++ let err = fs+ .inner+ .retry_pending_dir("/local/acct/docs")+ .await+ .expect_err("corrupt redirect metadata must be reported to retry logic");+ assert!(+ err.to_string().contains("redirect")+ || err.to_string().contains("json")+ || err.to_string().contains("serialization"),+ "unexpected error: {err}"+ );+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}++#[tokio::test]+async fn test_uncommitted_sync_log_entry_is_visible_but_not_retried() {+ let fs = test_multiwrite_fs(Vec::new());+ let ctx = test_ctx();++ FS_CTX+ .scope(ctx.clone(), async {+ fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)+ .await?;+ fs.inner+ .meta_store+ .update_dir_meta("/local/acct/docs", &ctx, |_redirect, sync_log| {+ sync_log+ .entries+ .insert("a.txt".to_string(), SyncLogEntry::new(1, SyncOp::Create));+ Ok(())+ })+ .await?;++ let status = fs.system_sync_status("/local/acct/docs").await?;+ assert_eq!(status["entry_count"], 1);+ assert_eq!(status["pending_target_count"], 0);+ assert_eq!(status["entries"][0]["primary_committed"], false);+ assert_eq!(+ status["entries"][0]["targets"][0]["primary_committed"],+ false+ );++ let retry = fs.system_sync_retry("/local/acct/docs").await?;+ assert_eq!(retry["retried"], 0);+ assert_eq!(retry["failed"], 0);+ assert_eq!(retry["skipped"], 1);+ assert_eq!(retry["results"][0]["status"], "awaiting_primary_commit");+ Ok::<(), Error>(())+ })+ .await+ .unwrap();+}diff --git a/crates/ragfs/src/core/plugin.rs b/crates/ragfs/src/core/plugin.rsindex 2bbcaf1cc..07326e902 100644--- a/crates/ragfs/src/core/plugin.rs+++ b/crates/ragfs/src/core/plugin.rs@@ -211,7 +211,13 @@ mod tests {async fn read(&self, _path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {Ok(vec![])}- async fn write(&self, _path: &str, _data: &[u8], _offset: u64, _flags: WriteFlag) -> Result<u64> {+ async fn write(+ &self,+ _path: &str,+ _data: &[u8],+ _offset: u64,+ _flags: WriteFlag,+ ) -> Result<u64> {Ok(_data.len() as u64)}async fn read_dir(&self, _path: &str) -> Result<Vec<FileInfo>> {@@ -254,11 +260,7 @@ mod tests {async fn test_plugin_lifecycle() {let plugin = MockPlugin;- let config = PluginConfig {- name: "mock".to_string(),- mount_path: "/mock".to_string(),- params: HashMap::new(),- };+ let config = PluginConfig::single_backend("mock", "/mock", HashMap::new());assert!(plugin.validate(&config).await.is_ok());assert!(plugin.initialize(config).await.is_ok());diff --git a/crates/ragfs/src/core/stats_wrapper.rs b/crates/ragfs/src/core/stats_wrapper.rsindex bab6c60f2..0a32380ea 100644--- a/crates/ragfs/src/core/stats_wrapper.rs+++ b/crates/ragfs/src/core/stats_wrapper.rs@@ -18,6 +18,11 @@ pub struct StatsWrappedFS {}impl StatsWrappedFS {+ /// Get the wrapped filesystem for specialized delegation.+ pub(crate) fn inner_fs(&self) -> &Arc<dyn FileSystem> {+ &self.inner+ }+/// Create a new statistics-wrapped filesystempub fn new(inner: Box<dyn FileSystem>) -> Self {Self {diff --git a/crates/ragfs/src/core/types.rs b/crates/ragfs/src/core/types.rsindex 1a502c084..eb98a5be9 100644--- a/crates/ragfs/src/core/types.rs+++ b/crates/ragfs/src/core/types.rs@@ -3,6 +3,7 @@//! This module defines the fundamental data structures used throughout RAGFS,//! including file metadata, write flags, and configuration types.+use serde::de::DeserializeOwned;use serde::{Deserialize, Serialize};use std::collections::HashMap;use std::time::SystemTime;@@ -74,7 +75,11 @@ impl GrepResult {/// Add a match to the resultpub fn add_match(&mut self, file: String, line: u64, content: String) {- self.matches.push(GrepMatch { file, line, content });+ self.matches.push(GrepMatch {+ file,+ line,+ content,+ });self.count += 1;}@@ -149,7 +154,7 @@ impl FileInfo {}/// Write operation flags-#[derive(Debug, Clone, Copy, PartialEq, Eq)]+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]pub enum WriteFlag {/// Create new file or truncate existingCreate,@@ -231,6 +236,136 @@ pub struct PluginConfig {/// Configuration parameterspub params: HashMap<String, ConfigValue>,++ /// Multi-write backups config (None = single backend mode)+ #[serde(default)]+ pub backups: Option<BackendsConfig>,++ /// Global encryption enabled (server.encryption.enabled)+ #[serde(default)]+ pub server_encryption_enabled: bool,++ /// Primary encryption enabled (follows global, not independently configurable)+ #[serde(default)]+ pub primary_encryption_enabled: bool,++ /// Primary redirect policies+ #[serde(default)]+ pub primary_redirects: Vec<RedirectPolicy>,+}++impl Default for PluginConfig {+ /// Build an empty plugin config with all optional multi-write fields disabled.+ fn default() -> Self {+ Self {+ name: String::new(),+ mount_path: String::new(),+ params: HashMap::new(),+ backups: None,+ server_encryption_enabled: false,+ primary_encryption_enabled: false,+ primary_redirects: Vec::new(),+ }+ }+}++impl PluginConfig {+ /// Build one single-backend plugin config with defaulted optional fields.+ pub fn single_backend(+ name: impl Into<String>,+ mount_path: impl Into<String>,+ params: HashMap<String, ConfigValue>,+ ) -> Self {+ Self {+ name: name.into(),+ mount_path: mount_path.into(),+ params,+ ..Self::default()+ }+ }++ /// Build one plugin config from raw params, extracting multi-write fields in one place.+ pub fn from_raw_parts(+ name: impl Into<String>,+ mount_path: impl Into<String>,+ mut params: HashMap<String, ConfigValue>,+ ) -> crate::core::Result<Self> {+ let backups = Self::take_optional_json("backups", &mut params)?;+ let server_encryption_enabled =+ Self::take_bool_with_default("server_encryption_enabled", &mut params, false)?;+ let primary_encryption_enabled =+ Self::take_bool_with_default("primary_encryption_enabled", &mut params, false)?;+ let primary_redirects =+ Self::take_json_with_default("primary_redirects", &mut params, Vec::new())?;++ Ok(Self {+ name: name.into(),+ mount_path: mount_path.into(),+ params,+ backups,+ server_encryption_enabled,+ primary_encryption_enabled,+ primary_redirects,+ })+ }++ /// Remove one optional bool config value and type-check it.+ fn take_bool_with_default(+ field_name: &str,+ params: &mut HashMap<String, ConfigValue>,+ default: bool,+ ) -> crate::core::Result<bool> {+ match params.remove(field_name) {+ None => Ok(default),+ Some(ConfigValue::Bool(value)) => Ok(value),+ Some(other) => Err(crate::core::Error::config(format!(+ "'{}' must be a boolean, got {}",+ field_name,+ Self::config_value_kind(&other)+ ))),+ }+ }++ /// Remove one optional JSON config value and deserialize it when present.+ fn take_optional_json<T: DeserializeOwned>(+ field_name: &str,+ params: &mut HashMap<String, ConfigValue>,+ ) -> crate::core::Result<Option<T>> {+ match params.remove(field_name) {+ None => Ok(None),+ Some(ConfigValue::Json(value)) => {+ let parsed = serde_json::from_value(value).map_err(|err| {+ crate::core::Error::config(format!("invalid '{}': {}", field_name, err))+ })?;+ Ok(Some(parsed))+ }+ Some(other) => Err(crate::core::Error::config(format!(+ "'{}' must be a JSON object/array, got {}",+ field_name,+ Self::config_value_kind(&other)+ ))),+ }+ }++ /// Remove one optional JSON config value and fall back to the provided default.+ fn take_json_with_default<T: DeserializeOwned>(+ field_name: &str,+ params: &mut HashMap<String, ConfigValue>,+ default: T,+ ) -> crate::core::Result<T> {+ Ok(Self::take_optional_json(field_name, params)?.unwrap_or(default))+ }++ /// Return a human-readable config value kind for error messages.+ fn config_value_kind(value: &ConfigValue) -> &'static str {+ match value {+ ConfigValue::String(_) => "string",+ ConfigValue::Int(_) => "integer",+ ConfigValue::Bool(_) => "boolean",+ ConfigValue::StringList(_) => "string_list",+ ConfigValue::Json(_) => "json",+ }+ }}/// Configuration value types@@ -248,6 +383,9 @@ pub enum ConfigValue {/// List of stringsStringList(Vec<String>),++ /// Nested JSON value (for complex config like backups)+ Json(serde_json::Value),}impl ConfigValue {@@ -284,6 +422,298 @@ impl ConfigValue {}}+// ── Multi-write configuration types ──++/// Multi-write backends container configuration+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct BackendsConfig {+ /// Sync type: "sync" or "async", default async+ #[serde(default = "default_sync_type")]+ pub sync_type: String,+ /// Minimum backup ack count for sync mode+ pub write_ack_count: Option<usize>,+ /// Timeout for waiting backup ack in sync mode (ms)+ pub write_ack_timeout_ms: Option<u64>,+ /// Max concurrent async writes+ pub write_concurrency: Option<usize>,+ /// Retry loop interval in milliseconds+ pub retry_interval_ms: Option<u64>,+ /// Base backoff in milliseconds for retry attempts+ pub retry_backoff_base_ms: Option<u64>,+ /// Maximum retry attempts per file/target in one round+ pub retry_max_retries_per_round: Option<usize>,+ /// Failure threshold before quarantining one file/target pair+ pub retry_quarantine_after_failures: Option<u32>,+ /// Deprecated no-op retained for config compatibility.+ #[serde(default)]+ pub read_probe_cache_ttl_ms: Option<u64>,+ /// Backup items+ pub items: Vec<BackendItemConfig>,+}++fn default_sync_type() -> String {+ "async".to_string()+}++/// Single backup backend item configuration+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct BackendItemConfig {+ /// Logical name, globally unique+ pub name: String,+ /// Plugin type (local/s3/memfs/kvfs/...)+ pub backend: String,+ /// Plugin-specific params (nested JSON)+ #[serde(default)]+ pub params: serde_json::Value,+ /// Timeout in seconds+ pub timeout: Option<u64>,+ /// Encryption config for this backup+ pub encryption: Option<EncryptionConfig>,+ /// Operations this backup participates in+ pub operations: Option<Vec<OperationItemConfig>>,+ /// Exclude policies+ pub excludes: Option<Vec<RedirectPolicy>>,+}++/// Per-operation priority config+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct OperationItemConfig {+ /// Operation type: "read" | "write"+ pub operation: String,+ /// Priority (smaller = higher priority)+ pub priority: u32,+}++/// Encryption on/off config for a backend+#[derive(Debug, Clone, Serialize, Deserialize)]+pub struct EncryptionConfig {+ /// Whether encryption is enabled for this backend+ pub enabled: bool,+}++/// Redirect / exclude policy (shared trait)+#[derive(Debug, Clone, Serialize, Deserialize)]+#[serde(tag = "type")]+pub enum RedirectPolicy {+ /// Redirect/exclude files exceeding a size threshold+ #[serde(rename = "FileOverSizePolicy")]+ FileOverSizePolicy {+ /// Max file size in MB+ max_size_mb: u64,+ /// Target backend names (redirect only)+ target: Option<Vec<String>>,+ },+ /// Redirect/exclude files matching extension patterns+ #[serde(rename = "FileExtensionPolicy")]+ FileExtensionPolicy {+ /// Regex patterns for file extensions+ extensions: Vec<String>,+ /// Target backend names (redirect only)+ target: Option<Vec<String>>,+ },+}++/// Strongly typed multi-write operation stored in `.sync_log.json`.+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+#[serde(tag = "type")]+pub enum SyncOp {+ /// Create an empty file.+ Create,+ /// Synchronize file content by copying the current primary state to the backup.+ SyncFile {+ /// File size used by retry/exclude policy decisions.+ size: u64,+ },+ /// Create a directory.+ Mkdir {+ /// Directory mode.+ mode: u32,+ },+ /// Change file mode.+ Chmod {+ /// File mode.+ mode: u32,+ },+ /// Remove one file.+ Remove,+ /// Remove a file tree.+ RemoveAll,+ /// Rename to another path.+ Rename {+ /// Rename target path.+ to: String,+ },+}++/// Per-backend sync state+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]+pub struct BackendSyncState {+ /// Acknowledged sequence number+ pub acked_seq: u64,+ /// Consecutive retry failures for the current operation.+ #[serde(default)]+ pub retry_failures: u32,+ /// Whether this backend/path pair is quarantined for manual intervention.+ #[serde(default)]+ pub quarantined: bool,+}++impl BackendSyncState {+ /// Create an acknowledged backend sync state.+ pub fn acked(acked_seq: u64) -> Self {+ Self {+ acked_seq,+ retry_failures: 0,+ quarantined: false,+ }+ }++ /// Mark this backend as acknowledged and clear retry state.+ pub fn mark_acked(&mut self, acked_seq: u64) {+ self.acked_seq = acked_seq;+ self.retry_failures = 0;+ self.quarantined = false;+ }++ /// Record one retry failure and quarantine after the configured threshold.+ pub fn mark_retry_failed(&mut self, quarantine_after_failures: u32) {+ self.retry_failures = self.retry_failures.saturating_add(1);+ if self.retry_failures >= quarantine_after_failures {+ self.quarantined = true;+ }+ }+}++/// Sync log entry for a single file path.+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct SyncLogEntry {+ /// Latest monotonic sequence number.+ pub latest_seq: u64,+ /// Whether the primary backend has durably completed this operation.+ #[serde(default = "default_primary_committed")]+ pub primary_committed: bool,+ /// Strongly typed operation payload.+ pub op: SyncOp,+ /// Per-backend acked sequence numbers.+ pub backends: std::collections::HashMap<String, BackendSyncState>,+}++fn default_primary_committed() -> bool {+ true+}++impl SyncLogEntry {+ /// Create a new prepared sync log entry for a sequenced operation.+ pub fn new(latest_seq: u64, op: SyncOp) -> Self {+ Self {+ latest_seq,+ primary_committed: false,+ op,+ backends: std::collections::HashMap::new(),+ }+ }++ /// Create a committed sync log entry for an operation already applied on primary.+ pub fn committed(latest_seq: u64, op: SyncOp) -> Self {+ Self {+ latest_seq,+ primary_committed: true,+ op,+ backends: std::collections::HashMap::new(),+ }+ }++ /// Mark this entry as committed on primary.+ pub fn mark_primary_committed(&mut self) {+ self.primary_committed = true;+ }++ /// Return whether this entry is committed on primary.+ pub fn is_primary_committed(&self) -> bool {+ self.primary_committed+ }++ /// Return the backend sync state if present.+ pub fn backend_state(&self, backend_name: &str) -> Option<&BackendSyncState> {+ self.backends.get(backend_name)+ }++ /// Return the acknowledged sequence for a backend, or 0 if unknown.+ pub fn acked_seq(&self, backend_name: &str) -> u64 {+ self.backend_state(backend_name)+ .map(|state| state.acked_seq)+ .unwrap_or(0)+ }++ /// Return whether the backend is quarantined.+ pub fn is_quarantined(&self, backend_name: &str) -> bool {+ self.backend_state(backend_name)+ .map(|state| state.quarantined)+ .unwrap_or(false)+ }++ /// Return whether the backend has acknowledged the latest operation.+ pub fn is_in_sync(&self, backend_name: &str) -> bool {+ self.acked_seq(backend_name) >= self.latest_seq+ }+}++/// Sync log file content+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]+pub struct SyncLogMeta {+ /// File entries keyed by file name (current directory)+ pub entries: std::collections::HashMap<String, SyncLogEntry>,+}++/// Redirect metadata file content+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct RedirectMeta {+ /// Schema version+ #[serde(default = "default_redirect_version")]+ pub version: u32,+ /// Redirect entries keyed by file name+ #[serde(default)]+ pub entries: std::collections::HashMap<String, RedirectEntry>,+}++fn default_redirect_version() -> u32 {+ 1+}++impl Default for RedirectMeta {+ fn default() -> Self {+ Self {+ version: 1,+ entries: HashMap::new(),+ }+ }+}++/// Single redirect entry+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+pub struct RedirectEntry {+ /// Target backend names+ pub targets: Vec<String>,+}++/// Backend role+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum BackendRole {+ /// Primary backend (authoritative source)+ Primary,+ /// Backup backend (replica)+ Backup,+}++/// Sync type enum+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum SyncType {+ /// Synchronous: wait for backup ack before returning+ Sync,+ /// Asynchronous: return after primary write, sync in background+ Async,+}+/// Custom serde module for SystemTimemod systemtime_serde {use serde::{Deserialize, Deserializer, Serialize, Serializer};@@ -342,4 +772,87 @@ mod tests {assert!(param.required);assert_eq!(param.param_type, "string");}++ #[test]+ fn test_sync_log_entry_legacy_payload_defaults_primary_committed_true() {+ let entry: SyncLogEntry = serde_json::from_value(serde_json::json!({+ "latest_seq": 7,+ "op": {+ "type": "Create"+ },+ "backends": {}+ }))+ .unwrap();++ assert!(entry.is_primary_committed());+ assert_eq!(entry.latest_seq, 7);+ }++ #[test]+ fn test_plugin_config_from_raw_parts_handles_multiwrite_shapes() {+ let mut valid_params = HashMap::new();+ valid_params.insert(+ "root_path".to_string(),+ ConfigValue::String("/tmp/data".to_string()),+ );+ valid_params.insert(+ "backups".to_string(),+ ConfigValue::Json(serde_json::json!({+ "items": [{"name": "backup1", "backend": "memory"}]+ })),+ );+ valid_params.insert(+ "server_encryption_enabled".to_string(),+ ConfigValue::Bool(true),+ );+ valid_params.insert(+ "primary_encryption_enabled".to_string(),+ ConfigValue::Bool(true),+ );+ valid_params.insert(+ "primary_redirects".to_string(),+ ConfigValue::Json(serde_json::json!([+ {+ "type": "FileExtensionPolicy",+ "extensions": [".bin"],+ "target": ["backup1"]+ }+ ])),+ );++ let config = PluginConfig::from_raw_parts("localfs", "/local", valid_params).unwrap();+ assert!(config.backups.is_some());+ assert!(config.server_encryption_enabled);+ assert!(config.primary_encryption_enabled);+ assert_eq!(config.primary_redirects.len(), 1);+ assert_eq!(+ config.params.get("root_path"),+ Some(&ConfigValue::String("/tmp/data".to_string()))+ );+ assert!(!config.params.contains_key("backups"));+ assert!(!config.params.contains_key("server_encryption_enabled"));+ assert!(!config.params.contains_key("primary_encryption_enabled"));+ assert!(!config.params.contains_key("primary_redirects"));++ for (params, expected_message) in [+ (+ HashMap::from([(+ "backups".to_string(),+ ConfigValue::Json(serde_json::json!({"items": "invalid"})),+ )]),+ "invalid 'backups'",+ ),+ (+ HashMap::from([(+ "server_encryption_enabled".to_string(),+ ConfigValue::String("true".to_string()),+ )]),+ "'server_encryption_enabled' must be a boolean",+ ),+ ] {+ let err = PluginConfig::from_raw_parts("localfs", "/local", params).unwrap_err();+ assert!(matches!(err, crate::core::Error::Config(_)));+ assert!(err.to_string().contains(expected_message));+ }+ }}diff --git a/crates/ragfs/src/crypto/mod.rs b/crates/ragfs/src/crypto/mod.rsindex 63e17f401..a9d3c5afd 100644--- a/crates/ragfs/src/crypto/mod.rs+++ b/crates/ragfs/src/crypto/mod.rs@@ -226,7 +226,8 @@ mod tests {assert!(is_encrypted(&env));let (_p, enc_key2, key_iv2, data_iv2, ct2) = parse_envelope(&env).unwrap();- let fk_bytes = aes_gcm_decrypt(&account_key, key_iv2.try_into().unwrap(), enc_key2).unwrap();+ let fk_bytes =+ aes_gcm_decrypt(&account_key, key_iv2.try_into().unwrap(), enc_key2).unwrap();let fk: [u8; 32] = fk_bytes.try_into().unwrap();let back = aes_gcm_decrypt(&fk, data_iv2.try_into().unwrap(), ct2).unwrap();assert_eq!(back, plaintext);@@ -268,7 +269,11 @@ mod tests {let got = hkdf_sha256(&root, b"tenant-7");let expected =hex_to_vec("a5447823b691e80727c6bf913bd3ac051f32c36ad387f39bf062f81c68307ae4");- assert_eq!(&got[..], &expected[..], "HKDF must match Python byte-for-byte");+ assert_eq!(+ &got[..],+ &expected[..],+ "HKDF must match Python byte-for-byte"+ );}#[test]@@ -283,9 +288,8 @@ mod tests {c42d408a1d21e8707cbd617acc62cb9b75dffea1ef5e7889163d970d13a8622cd4d31cddc163f4736556\c0180cbfb5",);- let expected_plaintext = hex_to_vec(- "63726f73732d6c616e6720696e7465726f7020636865636b20e4bda0e5a5bd",- );+ let expected_plaintext =+ hex_to_vec("63726f73732d6c616e6720696e7465726f7020636865636b20e4bda0e5a5bd");// Replicate EncryptionWrappedFS::read decryption path.let account_key = hkdf_sha256(&root, b"tenant-7");diff --git a/crates/ragfs/src/lib.rs b/crates/ragfs/src/lib.rsindex 20775715a..9d3079cc5 100644--- a/crates/ragfs/src/lib.rs+++ b/crates/ragfs/src/lib.rs@@ -31,7 +31,9 @@pub mod core;pub mod crypto;+pub mod multibackend;pub mod plugins;+pub mod shape;// Re-export core types for conveniencepub use core::{diff --git a/crates/ragfs/src/multibackend/admin.rs b/crates/ragfs/src/multibackend/admin.rsnew file mode 100644index 000000000..f9d12331b--- /dev/null+++ b/crates/ragfs/src/multibackend/admin.rs@@ -0,0 +1,234 @@+use std::collections::HashSet;+use std::time::Duration;++use serde_json::{json, Value};++use crate::core::errors::Result;+use crate::core::filesystem::{normalize_prefix_path, FileSystem};+use crate::core::multibackend_wrapper::{MultiWriteWrappedFS, SyncWorkEntry};+use crate::multibackend::meta::{current_required_ctx, parent_dir};++impl MultiWriteWrappedFS {+ /// Collect effective sync work entries under a path using the current request context.+ async fn collect_sync_work(&self, path: &str) -> Result<Vec<SyncWorkEntry>> {+ let ctx = current_required_ctx()?;+ let inner = &self.inner;+ let normalized = normalize_prefix_path(path);+ let path_info = <Self as FileSystem>::stat(self, &normalized).await?;+ let mut dirs = Vec::new();+ let mut seen_dirs = HashSet::new();++ let add_dir = |dirs: &mut Vec<String>, seen_dirs: &mut HashSet<String>, dir: String| {+ if seen_dirs.insert(dir.clone()) {+ dirs.push(dir);+ }+ };++ if path_info.is_dir {+ add_dir(&mut dirs, &mut seen_dirs, normalized.clone());+ for dir in inner.pending_dirs_snapshot().await {+ if dir == normalized+ || dir+ .strip_prefix(&normalized)+ .is_some_and(|suffix| suffix.starts_with('/'))+ {+ add_dir(&mut dirs, &mut seen_dirs, dir);+ }+ }+ if dirs.len() == 1 {+ for entry in inner+ .primary()+ .backend+ .tree_directory(&normalized, true, None, None)+ .await?+ {+ if entry.info.is_dir {+ add_dir(+ &mut dirs,+ &mut seen_dirs,+ normalize_prefix_path(&entry.path),+ );+ }+ }+ }+ } else {+ add_dir(+ &mut dirs,+ &mut seen_dirs,+ normalize_prefix_path(&parent_dir(&normalized)),+ );+ }++ let mut work = Vec::new();+ for dir in dirs {+ work.extend(+ inner+ .sync_work_in_dir(&dir, &ctx)+ .await?+ .into_iter()+ .filter(|work| path_info.is_dir || work.file_path == normalized),+ );+ }++ Ok(work)+ }++ /// Query effective multi-write sync status under a file or directory path.+ pub async fn system_sync_status(&self, path: &str) -> Result<Value> {+ let work = self.collect_sync_work(path).await?;+ let mut entries = Vec::new();+ let mut pending_target_count = 0usize;++ for work in work {+ let mut targets = Vec::new();+ let primary_committed = work.entry.is_primary_committed();+ let mut all_synced = primary_committed;++ for backend_name in work.targets {+ let acked_seq = if primary_committed {+ work.entry.acked_seq(&backend_name)+ } else {+ 0+ };+ let in_sync = primary_committed && work.entry.is_in_sync(&backend_name);+ if primary_committed && !in_sync {+ pending_target_count += 1;+ all_synced = false;+ }+ let state = work.entry.backend_state(&backend_name);+ targets.push(json!({+ "name": backend_name,+ "acked_seq": acked_seq,+ "retry_failures": state.map(|state| state.retry_failures).unwrap_or(0),+ "quarantined": state.map(|state| state.quarantined).unwrap_or(false),+ "primary_committed": primary_committed,+ "in_sync": in_sync,+ }));+ }++ entries.push(json!({+ "path": work.file_path,+ "latest_seq": work.entry.latest_seq,+ "primary_committed": primary_committed,+ "op": serde_json::to_value(&work.entry.op)?,+ "all_synced": all_synced,+ "targets": targets,+ }));+ }++ entries.sort_by(|a, b| {+ let ap = a.get("path").and_then(Value::as_str).unwrap_or_default();+ let bp = b.get("path").and_then(Value::as_str).unwrap_or_default();+ ap.cmp(bp)+ });++ Ok(json!({+ "path": normalize_prefix_path(path),+ "entry_count": entries.len(),+ "pending_target_count": pending_target_count,+ "capabilities": {+ "multi_instance_safe": false+ },+ "read_route_metrics": self.inner.read_route_metrics(),+ "entries": entries,+ }))+ }++ /// Manually retry lagging multi-write targets under a file or directory path.+ pub async fn system_sync_retry(&self, path: &str) -> Result<Value> {+ let ctx = current_required_ctx()?;+ let work = self.collect_sync_work(path).await?;+ let mut results = Vec::new();+ let mut retried = 0usize;+ let mut failed = 0usize;+ let mut skipped = 0usize;++ for work in work {+ if !work.entry.is_primary_committed() {+ skipped += work.targets.len();+ for backend_name in work.targets {+ results.push(json!({+ "path": work.file_path,+ "target": backend_name,+ "status": "awaiting_primary_commit",+ "latest_seq": work.entry.latest_seq,+ "primary_committed": false,+ "acked_seq": 0,+ }));+ }+ continue;+ }++ for backend_name in work.targets {+ let acked_seq = work.entry.acked_seq(&backend_name);+ let was_quarantined = work.entry.is_quarantined(&backend_name);+ if work.entry.is_in_sync(&backend_name) {+ skipped += 1;+ results.push(json!({+ "path": work.file_path,+ "target": backend_name,+ "status": "skipped",+ "latest_seq": work.entry.latest_seq,+ "acked_seq": acked_seq,+ }));+ continue;+ }++ let mut last_error = None;+ let mut success = false;+ for _attempt in 0..self.inner.max_retry_per_round {+ match self+ .inner+ .replay_operation(&work.file_path, &backend_name, &ctx)+ .await+ {+ Ok(()) => {+ success = true;+ break;+ }+ Err(err) => {+ last_error = Some(err.to_string());+ tokio::time::sleep(Duration::from_millis(+ self.inner.retry_backoff_base_ms,+ ))+ .await;+ }+ }+ }++ if success {+ retried += 1;+ results.push(json!({+ "path": work.file_path,+ "target": backend_name,+ "status": "retried",+ "latest_seq": work.entry.latest_seq,+ "acked_seq": work.entry.latest_seq,+ }));+ } else {+ self.inner+ .record_backup_retry_failure(&work.file_path, &backend_name, &ctx)+ .await?;+ failed += 1;+ results.push(json!({+ "path": work.file_path,+ "target": backend_name,+ "status": "failed",+ "latest_seq": work.entry.latest_seq,+ "acked_seq": acked_seq,+ "was_quarantined": was_quarantined,+ "error": last_error.unwrap_or_else(|| "unknown replay error".to_string()),+ }));+ }+ }+ }++ Ok(json!({+ "path": normalize_prefix_path(path),+ "retried": retried,+ "failed": failed,+ "skipped": skipped,+ "results": results,+ }))+ }+}diff --git a/crates/ragfs/src/multibackend/config.rs b/crates/ragfs/src/multibackend/config.rsnew file mode 100644index 000000000..be305f73c--- /dev/null+++ b/crates/ragfs/src/multibackend/config.rs@@ -0,0 +1,127 @@+//! Multi-backend config validation and normalization helpers.++use std::collections::HashMap;++use serde_json::Value;++use crate::core::errors::{Error, Result};+use crate::core::multibackend_wrapper::{BackendEntry, SyncMode};+use crate::core::types::{BackendsConfig, ConfigValue, PluginConfig, RedirectPolicy};++/// Convert one nested backup `params` object into plugin config values.+pub fn item_params_to_config_values(value: &Value) -> Result<HashMap<String, ConfigValue>> {+ if value.is_null() {+ return Ok(HashMap::new());+ }+ let obj = value.as_object().ok_or_else(|| {+ Error::config(format!(+ "backup item params must be a JSON object, got: {:?}",+ value+ ))+ })?;+ let mut params = HashMap::new();+ for (k, v) in obj {+ let cv = match v {+ Value::String(s) => ConfigValue::String(s.clone()),+ Value::Number(n) => {+ if let Some(i) = n.as_i64() {+ ConfigValue::Int(i)+ } else {+ ConfigValue::String(n.to_string())+ }+ }+ Value::Bool(b) => ConfigValue::Bool(*b),+ Value::Array(arr) => ConfigValue::StringList(+ arr.iter()+ .map(|item| match item {+ Value::String(s) => s.clone(),+ other => other.to_string(),+ })+ .collect(),+ ),+ Value::Object(_) => ConfigValue::Json(v.clone()),+ _ => ConfigValue::String(v.to_string()),+ };+ params.insert(k.clone(), cv);+ }+ Ok(params)+}++/// Validate encryption flags on the primary mount config.+pub fn validate_primary_encryption_flags(+ config: &PluginConfig,+ global_encryption_enabled: bool,+) -> Result<()> {+ if global_encryption_enabled && !config.server_encryption_enabled {+ return Err(Error::config(+ "server_encryption_enabled must be true when global encryption is configured"+ .to_string(),+ ));+ }+ if global_encryption_enabled && !config.primary_encryption_enabled {+ return Err(Error::config(+ "primary_encryption_enabled cannot be false when global encryption is enabled",+ ));+ }+ Ok(())+}++/// Validate redirect targets against the assembled backup entries.+pub fn validate_redirect_targets(+ redirects: &[RedirectPolicy],+ backup_entries: &[BackendEntry],+) -> Result<()> {+ for policy in redirects {+ let targets = match policy {+ RedirectPolicy::FileOverSizePolicy { target, .. } => target.as_ref(),+ RedirectPolicy::FileExtensionPolicy { target, .. } => target.as_ref(),+ };+ let target_names = targets+ .ok_or_else(|| Error::config("redirect policy target must not be empty".to_string()))?;+ if target_names.is_empty() {+ return Err(Error::config(+ "redirect policy target must not be empty".to_string(),+ ));+ }+ for name in target_names {+ if !backup_entries.iter().any(|be| &be.name == name) {+ return Err(Error::config(format!(+ "redirect target '{}' not found in backup entries",+ name+ )));+ }+ }+ }+ Ok(())+}++/// Validate that exclude policies do not silently carry redirect targets.+pub fn validate_backup_excludes(bc: &BackendsConfig) -> Result<()> {+ for item in &bc.items {+ for policy in item.excludes.as_deref().unwrap_or(&[]) {+ match policy {+ RedirectPolicy::FileOverSizePolicy { target, .. }+ | RedirectPolicy::FileExtensionPolicy { target, .. } => {+ if target.is_some() {+ return Err(Error::config(format!(+ "exclude policy for backup '{}' must not contain target",+ item.name+ )));+ }+ }+ }+ }+ }+ Ok(())+}++/// Convert config sync settings into the runtime sync mode.+pub fn sync_mode_from_config(bc: &BackendsConfig) -> SyncMode {+ match bc.sync_type.as_str() {+ "sync" => SyncMode::Sync {+ ack_count: bc.write_ack_count.unwrap_or(usize::MAX),+ timeout_ms: bc.write_ack_timeout_ms.unwrap_or(0),+ },+ _ => SyncMode::Async,+ }+}diff --git a/crates/ragfs/src/multibackend/factory.rs b/crates/ragfs/src/multibackend/factory.rsnew file mode 100644index 000000000..efaec6824--- /dev/null+++ b/crates/ragfs/src/multibackend/factory.rs@@ -0,0 +1,161 @@+//! Multi-backend runtime assembly from validated mount configuration.++use std::collections::{HashMap, HashSet};+use std::sync::Arc;+use std::time::Duration;++use tokio::sync::RwLock;++use crate::core::encryption_wrapper::EncryptionWrappedFS;+use crate::core::errors::{Error, Result};+use crate::core::filesystem::FileSystem;+use crate::core::multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS};+use crate::core::plugin::ServicePlugin;+use crate::core::types::{BackendRole, BackendsConfig, ConfigValue, PluginConfig};+use crate::multibackend::config::{+ item_params_to_config_values, sync_mode_from_config, validate_backup_excludes,+ validate_primary_encryption_flags, validate_redirect_targets,+};+use crate::multibackend::meta::MountRootFsContextResolver;+use crate::multibackend::types::MultiBackendBuildContext;+use crate::shape::validate::ensure_backend_shape;++/// Initialize one backend plugin instance from config params.+pub async fn init_backend_plugin(+ registry: &Arc<RwLock<HashMap<String, Arc<dyn ServicePlugin>>>>,+ plugin_name: &str,+ params: &HashMap<String, ConfigValue>,+) -> Result<Arc<dyn FileSystem>> {+ let plugin = {+ let registry = registry.read().await;+ registry+ .get(plugin_name)+ .cloned()+ .ok_or_else(|| Error::plugin(format!("Plugin '{}' not registered", plugin_name)))?+ };++ let plugin_config = PluginConfig::single_backend(plugin_name, String::new(), params.clone());++ plugin.validate(&plugin_config).await?;+ let fs = plugin.initialize(plugin_config).await?;+ Ok(Arc::from(fs))+}++/// Build the multi-backend wrapper from primary and backup configuration.+pub async fn build_multi_write_fs(+ registry: &Arc<RwLock<HashMap<String, Arc<dyn ServicePlugin>>>>,+ config: &PluginConfig,+ bc: &BackendsConfig,+ build_ctx: MultiBackendBuildContext,+) -> Result<MultiWriteWrappedFS> {+ let global_encryption_enabled = build_ctx.global_encryption_enabled();+ validate_primary_encryption_flags(config, global_encryption_enabled)?;++ let primary_raw = init_backend_plugin(registry, &config.name, &config.params).await?;+ ensure_backend_shape(+ &primary_raw,+ &config.name,+ global_encryption_enabled,+ build_ctx.enc_provider_type,+ build_ctx.enc_root_key,+ )+ .await?;+ let primary_backend: Arc<dyn FileSystem> = if global_encryption_enabled {+ Arc::new(EncryptionWrappedFS::new(+ primary_raw.clone(),+ build_ctx+ .enc_root_key+ .expect("global encryption validated before building primary backend"),+ build_ctx+ .enc_provider_type+ .expect("global encryption validated before building primary backend"),+ ))+ } else {+ primary_raw.clone()+ };++ let mut backup_entries: Vec<BackendEntry> = Vec::new();+ let mut seen_names: HashSet<String> = HashSet::new();++ for item in &bc.items {+ if item.name == "primary" {+ return Err(Error::config(+ "backup backend name 'primary' is reserved".to_string(),+ ));+ }+ if !seen_names.insert(item.name.clone()) {+ return Err(Error::config(format!(+ "duplicate backup name '{}'",+ item.name+ )));+ }++ let backup_params = item_params_to_config_values(&item.params)?;+ let backup_raw = init_backend_plugin(registry, &item.backend, &backup_params).await?;+ let backup_encrypted = global_encryption_enabled+ && item+ .encryption+ .as_ref()+ .map(|encryption| encryption.enabled)+ .unwrap_or(true);+ ensure_backend_shape(+ &backup_raw,+ &item.backend,+ backup_encrypted,+ if backup_encrypted {+ build_ctx.enc_provider_type+ } else {+ None+ },+ if backup_encrypted {+ build_ctx.enc_root_key+ } else {+ None+ },+ )+ .await?;++ let backup_backend: Arc<dyn FileSystem> = if backup_encrypted {+ Arc::new(EncryptionWrappedFS::new(+ backup_raw,+ build_ctx+ .enc_root_key+ .expect("global encryption validated before building backup backend"),+ build_ctx+ .enc_provider_type+ .expect("global encryption validated before building backup backend"),+ ))+ } else {+ backup_raw+ };++ backup_entries.push(BackendEntry {+ name: item.name.clone(),+ role: BackendRole::Backup,+ backend: backup_backend,+ raw_backend: None,+ operations: item.operations.clone().unwrap_or_default(),+ excludes: item.excludes.clone().unwrap_or_default(),+ });+ }++ validate_redirect_targets(&config.primary_redirects, &backup_entries)?;+ validate_backup_excludes(bc)?;++ MultiWriteWrappedFS::builder(primary_backend)+ .with_primary_raw_backend(primary_raw)+ .with_backups(backup_entries)+ .with_redirects(config.primary_redirects.clone())+ .sync_mode(sync_mode_from_config(bc))+ .write_concurrency(bc.write_concurrency)+ .retry_interval(Duration::from_millis(+ bc.retry_interval_ms.unwrap_or(30_000),+ ))+ .retry_backoff_base_ms(bc.retry_backoff_base_ms.unwrap_or(1_000))+ .max_retry_per_round(bc.retry_max_retries_per_round.unwrap_or(3))+ .quarantine_after_failures(bc.retry_quarantine_after_failures.unwrap_or(9))+ .ctx_resolver(Arc::new(MountRootFsContextResolver::new(+ &config.mount_path,+ )))+ .build()+}diff --git a/crates/ragfs/src/multibackend/meta.rs b/crates/ragfs/src/multibackend/meta.rsnew file mode 100644index 000000000..10666ddaf--- /dev/null+++ b/crates/ragfs/src/multibackend/meta.rs@@ -0,0 +1,607 @@+//! Multi-write metadata management.+//!+//! Provides `MetaStateStore` for serialized read-modify-write of `.redirect.json` and+//! `.sync_log.json` through `primary_backend`, and `FsContextResolver` for recovering+//! `FsContext` from paths in background tasks.++use std::collections::HashMap;+use std::sync::Arc;+use std::time::Duration;+use tokio::sync::Mutex;++use serde::{de::DeserializeOwned, Deserialize, Serialize};++use crate::core::context::{FsContext, FsContextInner, FS_CTX};+use crate::core::errors::{Error, Result};+use crate::core::filesystem::FileSystem;+use crate::core::types::{RedirectMeta, SyncLogMeta, WriteFlag};++/// Trait for resolving `FsContext` from a filesystem path.+///+/// Used by background tasks (retry_loop, backfill, system_sync_retry) that lack a+/// foreground request context. Implementations extract `account_id` from the path+/// (e.g. `/local/{account_id}/...`).+pub trait FsContextResolver: Send + Sync {+ /// Recover `FsContext` from a normalized path.+ /// Returns an error if the path cannot be resolved to a valid context.+ fn resolve(&self, path: &str) -> Result<FsContext>;+}++/// Default resolver that extracts `account_id` from `/local/{account_id}/...` paths.+pub struct DefaultFsContextResolver;++impl FsContextResolver for DefaultFsContextResolver {+ fn resolve(&self, path: &str) -> Result<FsContext> {+ let parts: Vec<&str> = path.trim_start_matches('/').split('/').collect();+ // Path format: /local/{account_id}/...+ if parts.len() >= 2 && parts[0] == "local" && !parts[1].is_empty() {+ Ok(Arc::new(FsContextInner::new(parts[1].to_string())))+ } else {+ Err(Error::internal(format!(+ "cannot resolve FsContext from path: {}",+ path+ )))+ }+ }+}++/// Resolver that extracts `account_id` from paths under one concrete mount root.+pub struct MountRootFsContextResolver {+ mount_root: String,+}++impl MountRootFsContextResolver {+ /// Create a resolver for one normalized mount root such as `/local` or `/s3`.+ pub fn new(mount_root: &str) -> Self {+ let mut normalized = mount_root.trim_end_matches('/').to_string();+ if normalized.is_empty() {+ normalized = "/".to_string();+ } else if !normalized.starts_with('/') {+ normalized = format!("/{normalized}");+ }+ Self {+ mount_root: normalized,+ }+ }+}++impl FsContextResolver for MountRootFsContextResolver {+ fn resolve(&self, path: &str) -> Result<FsContext> {+ let normalized = if path.starts_with('/') {+ path.to_string()+ } else {+ format!("/{path}")+ };+ let suffix = if self.mount_root == "/" {+ normalized.trim_start_matches('/')+ } else {+ normalized+ .strip_prefix(&self.mount_root)+ .and_then(|suffix| suffix.strip_prefix('/'))+ .ok_or_else(|| {+ Error::internal(format!("cannot resolve FsContext from path: {path}"))+ })?+ };+ let account_id = suffix.split('/').next().unwrap_or_default();+ if account_id.is_empty() {+ return Err(Error::internal(format!(+ "cannot resolve FsContext from path: {path}"+ )));+ }+ Ok(Arc::new(FsContextInner::new(account_id.to_string())))+ }+}++/// Internal file name for redirect metadata.+pub(crate) const REDIRECT_FILE: &str = ".redirect.json";+/// Internal file name for sync-log metadata.+pub(crate) const SYNC_LOG_FILE: &str = ".sync_log.json";+/// Hidden multi-write internal file names (redirect / sync-log metadata).+pub(crate) const MULTIWRITE_INTERNAL_NAMES: &[&str] = &[SYNC_LOG_FILE, REDIRECT_FILE];+const GLOBAL_STATE_FILE: &str = "/local/_system/.multiwrite.global.json";+const GLOBAL_STATE_VERSION: u32 = 1;++type TrackedLock = Mutex<()>;++#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]+struct GlobalMultiWriteState {+ version: u32,+ next_seq: u64,+}++impl Default for GlobalMultiWriteState {+ /// Create the default persisted global state for multi-write sequencing.+ fn default() -> Self {+ Self {+ version: GLOBAL_STATE_VERSION,+ next_seq: 1,+ }+ }+}++/// Unified metadata store for `.redirect.json` and `.sync_log.json`.+///+/// All reads and writes go through `primary_backend`, inheriting its encryption+/// configuration. Directory-level locks ensure serialized access to both metadata+/// files within the same directory.+pub struct MetaStateStore {+ /// Primary backend (may be encrypted)+ primary_backend: Arc<dyn FileSystem>,+ /// Per-directory locks for serialized read-modify-write+ dir_locks: Mutex<HashMap<String, Arc<TrackedLock>>>,+ /// Dedicated global state lock for next_seq persistence.+ global_state_lock: Mutex<()>,+ /// Context resolver for background tasks+ ctx_resolver: Arc<dyn FsContextResolver>,+}++impl MetaStateStore {+ /// Create a new MetaStateStore.+ pub fn new(+ primary_backend: Arc<dyn FileSystem>,+ ctx_resolver: Arc<dyn FsContextResolver>,+ ) -> Self {+ Self::with_cache_config(primary_backend, ctx_resolver, 0, Duration::ZERO)+ }++ /// Create a new MetaStateStore; cache parameters are retained as no-op compatibility knobs.+ pub fn with_cache_config(+ primary_backend: Arc<dyn FileSystem>,+ ctx_resolver: Arc<dyn FsContextResolver>,+ _meta_cache_capacity: usize,+ _meta_cache_ttl: Duration,+ ) -> Self {+ Self {+ primary_backend,+ dir_locks: Mutex::new(HashMap::new()),+ global_state_lock: Mutex::new(()),+ ctx_resolver,+ }+ }++ /// Get or create a per-directory lock.+ async fn get_dir_lock(&self, dir: &str) -> Arc<TrackedLock> {+ let mut locks = self.dir_locks.lock().await;+ if let Some(lock) = locks.get(dir).cloned() {+ return lock;+ }+ let lock = Arc::new(TrackedLock::new(()));+ locks.insert(dir.to_string(), lock.clone());+ lock+ }++ /// Return the dedicated `_system` context used by global metadata.+ fn system_ctx() -> FsContext {+ Arc::new(FsContextInner::new("_system".to_string()))+ }++ /// Build the full path for a metadata file in a directory.+ fn meta_path(dir: &str, filename: &str) -> String {+ if dir == "/" {+ format!("/{}", filename)+ } else {+ format!("{}/{}", dir, filename)+ }+ }++ /// Read one JSON metadata file, returning default for missing or empty files.+ async fn read_meta<T>(&self, dir: &str, filename: &str, ctx: &FsContext) -> Result<T>+ where+ T: DeserializeOwned + Default,+ {+ let path = Self::meta_path(dir, filename);+ match FS_CTX+ .scope(ctx.clone(), async {+ self.primary_backend.read(&path, 0, 0).await+ })+ .await+ {+ Ok(data) => {+ if data.is_empty() {+ Ok(T::default())+ } else {+ serde_json::from_slice(&data).map_err(Error::from)+ }+ }+ Err(Error::NotFound(_)) => Ok(T::default()),+ Err(e) => Err(e),+ }+ }++ /// Read redirect metadata from a directory (returns default if not found).+ async fn read_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result<RedirectMeta> {+ self.read_meta(dir, REDIRECT_FILE, ctx).await+ }++ /// Read sync log metadata from a directory (returns default if not found).+ async fn read_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result<SyncLogMeta> {+ self.read_meta(dir, SYNC_LOG_FILE, ctx).await+ }++ /// Read both metadata files from the primary backend.+ async fn read_dir_meta_pair(+ &self,+ dir: &str,+ ctx: &FsContext,+ ) -> Result<(RedirectMeta, SyncLogMeta)> {+ let redirect = self.read_redirect_meta(dir, ctx).await?;+ let sync_log = self.read_sync_log_meta(dir, ctx).await?;+ Ok((redirect, sync_log))+ }++ /// Write one JSON metadata file to a directory.+ async fn write_meta<T>(+ &self,+ dir: &str,+ filename: &str,+ meta: &T,+ ctx: &FsContext,+ ) -> Result<()>+ where+ T: Serialize,+ {+ let path = Self::meta_path(dir, filename);+ let data = serde_json::to_vec(meta)?;+ FS_CTX+ .scope(ctx.clone(), async {+ self.primary_backend+ .write(&path, &data, 0, WriteFlag::Create)+ .await+ .map(|_| ())+ })+ .await+ }++ /// Write redirect metadata to a directory.+ async fn write_redirect_meta(+ &self,+ dir: &str,+ meta: &RedirectMeta,+ ctx: &FsContext,+ ) -> Result<()> {+ self.write_meta(dir, REDIRECT_FILE, meta, ctx).await+ }++ /// Write sync log metadata to a directory.+ async fn write_sync_log_meta(+ &self,+ dir: &str,+ meta: &SyncLogMeta,+ ctx: &FsContext,+ ) -> Result<()> {+ self.write_meta(dir, SYNC_LOG_FILE, meta, ctx).await+ }++ /// Serialized read-modify-write of both `.redirect.json` and `.sync_log.json` in a directory.+ ///+ /// Acquires the directory lock, reads both metadata files, applies `op`, and writes both back.+ /// This prevents concurrent updates from losing entries.+ pub async fn update_dir_meta<F>(&self, dir: &str, ctx: &FsContext, op: F) -> Result<()>+ where+ F: FnOnce(&mut RedirectMeta, &mut SyncLogMeta) -> Result<()>,+ {+ let lock = self.get_dir_lock(dir).await;+ let _guard = lock.lock().await;++ let (mut redirect_meta, mut sync_log_meta) = self.read_dir_meta_pair(dir, ctx).await?;+ let original_redirect = redirect_meta.clone();+ let original_sync_log = sync_log_meta.clone();++ op(&mut redirect_meta, &mut sync_log_meta)?;++ if redirect_meta != original_redirect {+ self.write_redirect_meta(dir, &redirect_meta, ctx).await?;+ }+ if sync_log_meta != original_sync_log {+ self.write_sync_log_meta(dir, &sync_log_meta, ctx).await?;+ }+ Ok(())+ }++ /// Serialized read-modify-write of two directories' metadata (for cross-directory rename).+ ///+ /// Acquires both directory locks in lexicographic order to prevent deadlock,+ /// then reads and updates all four metadata files within the same critical section.+ /// Caller must ensure source_dir != target_dir; use update_dir_meta for same-directory case.+ pub async fn update_dual_dir_meta<F>(+ &self,+ source_dir: &str,+ target_dir: &str,+ ctx: &FsContext,+ op: F,+ ) -> Result<()>+ where+ F: FnOnce(+ &mut RedirectMeta,+ &mut SyncLogMeta,+ &mut RedirectMeta,+ &mut SyncLogMeta,+ ) -> Result<()>,+ {+ // Acquire locks in lexicographic order to avoid deadlock.+ let (first_dir, second_dir) = if source_dir < target_dir {+ (source_dir, target_dir)+ } else {+ (target_dir, source_dir)+ };++ let lock1 = self.get_dir_lock(first_dir).await;+ let lock2 = self.get_dir_lock(second_dir).await;+ let _guard1 = lock1.lock().await;+ let _guard2 = lock2.lock().await;++ let (mut src_redirect, mut src_sync_log) = self.read_dir_meta_pair(source_dir, ctx).await?;+ let (mut tgt_redirect, mut tgt_sync_log) = self.read_dir_meta_pair(target_dir, ctx).await?;+ let original_src_redirect = src_redirect.clone();+ let original_src_sync_log = src_sync_log.clone();+ let original_tgt_redirect = tgt_redirect.clone();+ let original_tgt_sync_log = tgt_sync_log.clone();++ op(+ &mut src_redirect,+ &mut src_sync_log,+ &mut tgt_redirect,+ &mut tgt_sync_log,+ )?;++ if src_redirect != original_src_redirect {+ self.write_redirect_meta(source_dir, &src_redirect, ctx)+ .await?;+ }+ if src_sync_log != original_src_sync_log {+ self.write_sync_log_meta(source_dir, &src_sync_log, ctx)+ .await?;+ }+ if tgt_redirect != original_tgt_redirect {+ self.write_redirect_meta(target_dir, &tgt_redirect, ctx)+ .await?;+ }+ if tgt_sync_log != original_tgt_sync_log {+ self.write_sync_log_meta(target_dir, &tgt_sync_log, ctx)+ .await?;+ }+ Ok(())+ }++ /// Read redirect metadata for a directory (public, used by read_dir to merge redirect entries).+ pub async fn get_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result<RedirectMeta> {+ self.read_dir_meta_pair(dir, ctx)+ .await+ .map(|(redirect, _)| redirect)+ }++ /// Read sync log metadata for a directory (public, used by retry_loop).+ pub async fn get_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result<SyncLogMeta> {+ self.read_dir_meta_pair(dir, ctx)+ .await+ .map(|(_, sync_log)| sync_log)+ }++ /// Get a reference to the context resolver.+ pub fn ctx_resolver(&self) -> &Arc<dyn FsContextResolver> {+ &self.ctx_resolver+ }++ /// Get a reference to the primary backend.+ pub fn primary_backend(&self) -> &Arc<dyn FileSystem> {+ &self.primary_backend+ }++ /// Allocate and persist the next global sequence number.+ pub async fn next_seq(&self) -> Result<u64> {+ let _guard = self.global_state_lock.lock().await;+ let mut state = self.read_global_state().await?;+ let seq = state.next_seq;+ state.next_seq = state.next_seq.saturating_add(1);+ self.write_global_state(&state).await?;+ Ok(seq)+ }++ /// Read the persisted global state file.+ async fn read_global_state(&self) -> Result<GlobalMultiWriteState> {+ let ctx = Self::system_ctx();+ match FS_CTX+ .scope(ctx.clone(), async {+ self.primary_backend.read(GLOBAL_STATE_FILE, 0, 0).await+ })+ .await+ {+ Ok(data) => {+ if data.is_empty() {+ Ok(GlobalMultiWriteState::default())+ } else {+ let state: GlobalMultiWriteState = serde_json::from_slice(&data)?;+ if state.version != GLOBAL_STATE_VERSION {+ return Err(Error::config(format!(+ "unsupported multi-write global state version {}",+ state.version+ )));+ }+ Ok(state)+ }+ }+ Err(Error::NotFound(_)) => Ok(GlobalMultiWriteState::default()),+ Err(e) => Err(e),+ }+ }++ /// Persist the global sequence state through the primary backend.+ async fn write_global_state(&self, state: &GlobalMultiWriteState) -> Result<()> {+ let ctx = Self::system_ctx();+ let data = serde_json::to_vec(state)?;+ FS_CTX+ .scope(ctx.clone(), async {+ self.primary_backend+ .ensure_parent_dirs(GLOBAL_STATE_FILE, 0o755)+ .await?;+ self.primary_backend+ .write(GLOBAL_STATE_FILE, &data, 0, WriteFlag::Create)+ .await+ .map(|_| ())+ })+ .await+ }+}++/// Per-path serialization queue for async write ordering.+///+/// Ensures that multiple writes to the same path are executed in FIFO order+/// on backup backends, preventing out-of-order application.+pub struct PathSerializer {+ queues: Mutex<HashMap<String, Arc<TrackedLock>>>,+}++impl PathSerializer {+ /// Create a new PathSerializer.+ pub fn new() -> Self {+ Self::with_limits(0, Duration::ZERO)+ }++ /// Create a PathSerializer; limit arguments are ignored for compatibility.+ pub fn with_limits(_capacity: usize, _ttl: Duration) -> Self {+ Self {+ queues: Mutex::new(HashMap::new()),+ }+ }++ /// Run one async operation under the per-path serialization lock.+ pub async fn with_path_lock<F, Fut, T>(&self, path: &str, op: F) -> T+ where+ F: FnOnce() -> Fut,+ Fut: std::future::Future<Output = T>,+ {+ let mut queues = self.queues.lock().await;+ let lock = queues+ .entry(path.to_string())+ .or_insert_with(|| Arc::new(TrackedLock::new(())))+ .clone();+ drop(queues);++ let _guard = lock.lock().await;+ op().await+ }++ /// Return the number of tracked queue entries.+ #[cfg(test)]+ pub async fn len(&self) -> usize {+ self.queues.lock().await.len()+ }+}++impl Default for PathSerializer {+ fn default() -> Self {+ Self::new()+ }+}++/// Extract the directory path from a file path.+pub(crate) fn parent_dir(path: &str) -> String {+ match path.rfind('/') {+ Some(0) => "/".to_string(),+ Some(pos) => path[..pos].to_string(),+ None => "/".to_string(),+ }+}++/// Extract the file name from a path.+pub(crate) fn file_name(path: &str) -> &str {+ match path.rfind('/') {+ Some(pos) => &path[pos + 1..],+ None => path,+ }+}++/// Snapshot the current FsContext from the task-local, returning an error if unset.+pub fn current_required_ctx() -> Result<FsContext> {+ FS_CTX+ .try_with(|c| c.clone())+ .map_err(|_| Error::context_missing("FsContext not set in current task"))+}++#[cfg(test)]+mod tests {+ use super::*;+ use crate::plugins::memfs::MemFileSystem;+ use std::sync::Arc;++ #[test]+ fn test_parent_dir() {+ assert_eq!(parent_dir("/a/b/c.txt"), "/a/b");+ assert_eq!(parent_dir("/a"), "/");+ assert_eq!(parent_dir("/"), "/");+ }++ #[test]+ fn test_file_name() {+ assert_eq!(file_name("/a/b/c.txt"), "c.txt");+ assert_eq!(file_name("/a"), "a");+ }++ #[test]+ fn test_default_resolver() {+ let resolver = DefaultFsContextResolver;+ let ctx = resolver+ .resolve("/local/tenant-1/resources/file.txt")+ .unwrap();+ assert_eq!(ctx.account_id(), "tenant-1");+ }++ #[test]+ fn test_default_resolver_invalid_path() {+ let resolver = DefaultFsContextResolver;+ assert!(resolver.resolve("/invalid/path").is_err());+ }++ #[test]+ fn test_mount_root_resolver_extracts_account_from_non_local_path() {+ let resolver = MountRootFsContextResolver::new("/s3");+ let ctx = resolver.resolve("/s3/acct-1/resources/file.txt").unwrap();+ assert_eq!(ctx.account_id(), "acct-1");+ assert!(resolver+ .resolve("/local/acct-1/resources/file.txt")+ .is_err());+ }++ #[tokio::test]+ async fn test_invalid_sync_log_json_returns_error() {+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let store = MetaStateStore::new(primary.clone(), Arc::new(DefaultFsContextResolver));+ let ctx = Arc::new(FsContextInner::new("acct".to_string()));++ FS_CTX+ .scope(ctx.clone(), async {+ primary+ .ensure_parent_dirs("/local/acct/docs/.sync_log.json", 0o755)+ .await?;+ primary+ .write(+ "/local/acct/docs/.sync_log.json",+ b"{not valid json",+ 0,+ WriteFlag::Create,+ )+ .await?;+ Ok::<(), Error>(())+ })+ .await+ .unwrap();++ let result = store.get_sync_log_meta("/local/acct/docs", &ctx).await;+ assert!(result.is_err(), "corrupted metadata must fail fast");+ }++ #[tokio::test]+ async fn test_next_seq_uses_dedicated_global_lock() {+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());+ let store = MetaStateStore::new(primary, Arc::new(DefaultFsContextResolver));++ assert_eq!(store.next_seq().await.unwrap(), 1);+ assert_eq!(store.next_seq().await.unwrap(), 2);++ let dir_locks = store.dir_locks.lock().await;+ assert!(+ !dir_locks.contains_key(GLOBAL_STATE_FILE),+ "global sequence locking should not occupy the directory lock pool"+ );+ }+}diff --git a/crates/ragfs/src/multibackend/mod.rs b/crates/ragfs/src/multibackend/mod.rsnew file mode 100644index 000000000..44e8e4ab2--- /dev/null+++ b/crates/ragfs/src/multibackend/mod.rs@@ -0,0 +1,14 @@+//! Multi-backend configuration, factory, metadata, and admin support.++/// Multi-backend administrative APIs.+pub mod admin;+/// Multi-backend config validation and normalization helpers.+pub mod config;+/// Multi-backend runtime assembly from validated config.+pub mod factory;+/// Metadata state management shared by multi-backend runtime paths.+pub mod meta;+/// Shared build-time types for multi-backend assembly.+pub mod types;++pub use meta::{FsContextResolver, MetaStateStore};diff --git a/crates/ragfs/src/multibackend/types.rs b/crates/ragfs/src/multibackend/types.rsnew file mode 100644index 000000000..8e097ee4f--- /dev/null+++ b/crates/ragfs/src/multibackend/types.rs@@ -0,0 +1,17 @@+//! Shared build-time types for multi-backend assembly.++/// Encryption context used while assembling one multi-backend mount.+#[derive(Clone, Copy)]+pub struct MultiBackendBuildContext {+ /// Global root key, if server-side encryption is enabled.+ pub enc_root_key: Option<[u8; 32]>,+ /// Global provider type, if server-side encryption is enabled.+ pub enc_provider_type: Option<u8>,+}++impl MultiBackendBuildContext {+ /// Return whether global encryption is enabled for this mount.+ pub fn global_encryption_enabled(&self) -> bool {+ self.enc_root_key.is_some() && self.enc_provider_type.is_some()+ }+}diff --git a/crates/ragfs/src/plugins/kvfs/mod.rs b/crates/ragfs/src/plugins/kvfs/mod.rsindex 9f4d7b9b9..e2b1775b0 100644--- a/crates/ragfs/src/plugins/kvfs/mod.rs+++ b/crates/ragfs/src/plugins/kvfs/mod.rs@@ -67,11 +67,7 @@ impl KVFileSystem {/// List all keys with a given prefixfn list_keys_with_prefix(&self, store: &HashMap<String, KVEntry>, prefix: &str) -> Vec<String> {- let search_prefix = if prefix == "/" {- ""- } else {- prefix- };+ let search_prefix = if prefix == "/" { "" } else { prefix };store.keys()@@ -450,41 +446,41 @@ KVFS has no configuration parameters.mod tests {use super::*;+ /// Write one key in KVFS tests.+ async fn write_key(fs: &KVFileSystem, path: &str, data: &[u8]) {+ fs.write(path, data, 0, WriteFlag::Create).await.unwrap();+ }++ /// Read one key in KVFS tests.+ async fn read_key(fs: &KVFileSystem, path: &str) -> Vec<u8> {+ fs.read(path, 0, 0).await.unwrap()+ }+#[tokio::test]async fn test_kvfs_basic_operations() {let fs = KVFileSystem::new();// Create and write- fs.write("/key1", b"value1", 0, WriteFlag::Create)- .await- .unwrap();+ write_key(&fs, "/key1", b"value1").await;// Read- let data = fs.read("/key1", 0, 0).await.unwrap();- assert_eq!(data, b"value1");+ assert_eq!(read_key(&fs, "/key1").await, b"value1");// Updatefs.write("/key1", b"value2", 0, WriteFlag::Truncate).await.unwrap();- let data = fs.read("/key1", 0, 0).await.unwrap();- assert_eq!(data, b"value2");+ assert_eq!(read_key(&fs, "/key1").await, b"value2");}#[tokio::test]async fn test_kvfs_list_keys() {let fs = KVFileSystem::new();- fs.write("/key1", b"val1", 0, WriteFlag::Create)- .await- .unwrap();- fs.write("/key2", b"val2", 0, WriteFlag::Create)- .await- .unwrap();- fs.write("/key3", b"val3", 0, WriteFlag::Create)- .await- .unwrap();+ write_key(&fs, "/key1", b"val1").await;+ write_key(&fs, "/key2", b"val2").await;+ write_key(&fs, "/key3", b"val3").await;let entries = fs.read_dir("/").await.unwrap();assert_eq!(entries.len(), 3);@@ -497,12 +493,8 @@ mod tests {// Create parent "directory" firstfs.mkdir("/user", 0o755).await.unwrap();- fs.write("/user/123", b"alice", 0, WriteFlag::Create)- .await- .unwrap();- fs.write("/user/456", b"bob", 0, WriteFlag::Create)- .await- .unwrap();+ write_key(&fs, "/user/123", b"alice").await;+ write_key(&fs, "/user/456", b"bob").await;let entries = fs.read_dir("/user").await.unwrap();assert_eq!(entries.len(), 2);@@ -512,9 +504,7 @@ mod tests {async fn test_kvfs_delete() {let fs = KVFileSystem::new();- fs.write("/key1", b"value1", 0, WriteFlag::Create)- .await- .unwrap();+ write_key(&fs, "/key1", b"value1").await;fs.remove("/key1").await.unwrap();assert!(fs.read("/key1", 0, 0).await.is_err());@@ -524,14 +514,11 @@ mod tests {async fn test_kvfs_rename() {let fs = KVFileSystem::new();- fs.write("/oldkey", b"data", 0, WriteFlag::Create)- .await- .unwrap();+ write_key(&fs, "/oldkey", b"data").await;fs.rename("/oldkey", "/newkey").await.unwrap();assert!(fs.read("/oldkey", 0, 0).await.is_err());- let data = fs.read("/newkey", 0, 0).await.unwrap();- assert_eq!(data, b"data");+ assert_eq!(read_key(&fs, "/newkey").await, b"data");}#[tokio::test]@@ -539,11 +526,7 @@ mod tests {let plugin = KVFSPlugin;assert_eq!(plugin.name(), "kvfs");- let config = PluginConfig {- name: "kvfs".to_string(),- mount_path: "/kvfs".to_string(),- params: HashMap::new(),- };+ let config = PluginConfig::single_backend("kvfs", "/kvfs", HashMap::new());assert!(plugin.validate(&config).await.is_ok());assert!(plugin.initialize(config).await.is_ok());diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rsindex 4c98f9b4c..08cd2d6d3 100644--- a/crates/ragfs/src/plugins/localfs/mod.rs+++ b/crates/ragfs/src/plugins/localfs/mod.rs@@ -793,9 +793,9 @@ impl FileSystem for LocalFileSystem {// Get parent directoryif let Some(parent) = local_path.parent() {// Create all parent directories if they don't exist- tokio::fs::create_dir_all(parent)- .await- .map_err(|e| Error::plugin(format!("failed to create parent directories: {}", e)))?;+ tokio::fs::create_dir_all(parent).await.map_err(|e| {+ Error::plugin(format!("failed to create parent directories: {}", e))+ })?;}Ok(())@@ -869,6 +869,8 @@ impl FileSystem for LocalFileSystem {#[cfg(test)]mod tests {use super::*;+ use std::path::Path;+ use tempfile::TempDir;struct EnvVarGuard {key: &'static str,@@ -893,19 +895,52 @@ mod tests {}}- #[tokio::test]- async fn test_localfs_grep_fallback_respects_gitignore_and_hidden() {- use tempfile::TempDir;-+ /// Create a LocalFS fixture pinned to the fallback grep implementation.+ fn fallback_localfs() -> (TempDir, LocalFileSystem) {let dir = TempDir::new().unwrap();- std::fs::write(dir.path().join(".gitignore"), "ignored.txt\n").unwrap();- std::fs::write(dir.path().join("a.txt"), "hello\n").unwrap();- std::fs::write(dir.path().join("ignored.txt"), "hello\n").unwrap();- std::fs::create_dir_all(dir.path().join(".hidden")).unwrap();- std::fs::write(dir.path().join(".hidden/secret.txt"), "hello\n").unwrap();-let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();fs.has_rg = false;+ (dir, fs)+ }++ /// Write a test file, creating parent directories when needed.+ fn write_file(root: &Path, path: &str, content: &str) {+ let path = root.join(path);+ if let Some(parent) = path.parent() {+ std::fs::create_dir_all(parent).unwrap();+ }+ std::fs::write(path, content).unwrap();+ }++ /// Install a fake rg executable and prepend it to PATH for the current test.+ #[cfg(unix)]+ fn install_fake_rg(dir: &TempDir, script: &str) -> EnvVarGuard {+ let bin_dir = dir.path().join("bin");+ std::fs::create_dir_all(&bin_dir).unwrap();+ let rg_path = bin_dir.join("rg");+ std::fs::write(&rg_path, script).unwrap();++ use std::os::unix::fs::PermissionsExt;+ let mut perms = std::fs::metadata(&rg_path).unwrap().permissions();+ perms.set_mode(0o755);+ std::fs::set_permissions(&rg_path, perms).unwrap();++ let old_path = std::env::var_os("PATH").unwrap_or_default();+ let mut new_path = std::ffi::OsString::new();+ new_path.push(bin_dir.as_os_str());+ new_path.push(std::ffi::OsStr::new(":"));+ new_path.push(old_path);+ EnvVarGuard::set("PATH", &new_path)+ }++ #[tokio::test]+ async fn test_localfs_grep_fallback_respects_gitignore_and_hidden() {+ let (dir, fs) = fallback_localfs();+ write_file(dir.path(), ".gitignore", "ignored.txt\n");+ write_file(dir.path(), "a.txt", "hello\n");+ write_file(dir.path(), "ignored.txt", "hello\n");+ std::fs::create_dir_all(dir.path().join(".hidden")).unwrap();+ write_file(dir.path(), ".hidden/secret.txt", "hello\n");let result = fs.grep("/", "hello", true, false, None, None, None)@@ -918,13 +953,8 @@ mod tests {#[tokio::test]async fn test_localfs_grep_case_insensitive() {- use tempfile::TempDir;-- let dir = TempDir::new().unwrap();- std::fs::write(dir.path().join("a.txt"), "HELLO\n").unwrap();-- let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();- fs.has_rg = false;+ let (dir, fs) = fallback_localfs();+ write_file(dir.path(), "a.txt", "HELLO\n");let result = fs.grep("/", "hello", true, true, None, None, None)@@ -936,11 +966,7 @@ mod tests {#[tokio::test]async fn test_localfs_grep_missing_path_returns_not_found() {- use tempfile::TempDir;-- let dir = TempDir::new().unwrap();- let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();- fs.has_rg = false;+ let (_dir, fs) = fallback_localfs();let err = fs.grep("/does-not-exist", "hello", true, false, None, None, None)@@ -951,15 +977,13 @@ mod tests {#[tokio::test]async fn test_localfs_grep_does_not_inherit_parent_gitignore() {- use tempfile::TempDir;-let dir = TempDir::new().unwrap();let mount_dir = dir.path().join("mount");std::fs::create_dir_all(&mount_dir).unwrap();// Parent .gitignore ignores the mounted directory entirely.- std::fs::write(dir.path().join(".gitignore"), "mount/\n").unwrap();- std::fs::write(mount_dir.join("a.txt"), "hello\n").unwrap();+ write_file(dir.path(), ".gitignore", "mount/\n");+ write_file(&mount_dir, "a.txt", "hello\n");let mut fs = LocalFileSystem::new(mount_dir.to_str().unwrap()).unwrap();fs.has_rg = false;@@ -974,14 +998,8 @@ mod tests {#[tokio::test]async fn test_localfs_grep_returns_query_root_relative_paths() {- use tempfile::TempDir;-- let dir = TempDir::new().unwrap();- std::fs::create_dir_all(dir.path().join("sub")).unwrap();- std::fs::write(dir.path().join("sub/a.txt"), "hello\n").unwrap();-- let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();- fs.has_rg = false;+ let (dir, fs) = fallback_localfs();+ write_file(dir.path(), "sub/a.txt", "hello\n");let result = fs.grep("/sub", "hello", true, false, None, None, None)@@ -1001,29 +1019,12 @@ mod tests {#[cfg(unix)]#[tokio::test]async fn test_localfs_grep_node_limit_zero_does_not_invoke_rg() {- use tempfile::TempDir;-let dir = TempDir::new().unwrap();- std::fs::write(dir.path().join("a.txt"), "hello\n").unwrap();+ write_file(dir.path(), "a.txt", "hello\n");// Create a fake `rg` that fails if invoked. The expected behavior is to short-circuit// and return empty result when node_limit=0.- let bin_dir = dir.path().join("bin");- std::fs::create_dir_all(&bin_dir).unwrap();- let rg_path = bin_dir.join("rg");- std::fs::write(&rg_path, "#!/bin/sh\nexit 2\n").unwrap();- use std::os::unix::fs::PermissionsExt;- let mut perms = std::fs::metadata(&rg_path).unwrap().permissions();- perms.set_mode(0o755);- std::fs::set_permissions(&rg_path, perms).unwrap();-- // Prepend fake rg to PATH.- let old_path = std::env::var_os("PATH").unwrap_or_default();- let mut new_path = std::ffi::OsString::new();- new_path.push(bin_dir.as_os_str());- new_path.push(std::ffi::OsStr::new(":"));- new_path.push(old_path);- let _path_guard = EnvVarGuard::set("PATH", &new_path);+ let _path_guard = install_fake_rg(&dir, "#!/bin/sh\nexit 2\n");let fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();let result = fs@@ -1036,16 +1037,9 @@ mod tests {#[tokio::test]async fn test_localfs_grep_exclude_path_applies_before_node_limit() {- use tempfile::TempDir;-- let dir = TempDir::new().unwrap();- std::fs::create_dir_all(dir.path().join("excluded")).unwrap();- std::fs::create_dir_all(dir.path().join("ok")).unwrap();- std::fs::write(dir.path().join("excluded/x.txt"), "hit\n").unwrap();- std::fs::write(dir.path().join("ok/y.txt"), "hit\n").unwrap();-- let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();- fs.has_rg = false;+ let (dir, fs) = fallback_localfs();+ write_file(dir.path(), "excluded/x.txt", "hit\n");+ write_file(dir.path(), "ok/y.txt", "hit\n");let out = fs.grep("/", "hit", true, false, Some(1), Some("/excluded"), None)@@ -1059,17 +1053,10 @@ mod tests {#[tokio::test]async fn test_localfs_grep_exclude_path_is_query_root_relative_for_nested_query_root() {- use tempfile::TempDir;-- let dir = TempDir::new().unwrap();+ let (dir, fs) = fallback_localfs();let nested_root = dir.path().join("acct/resources/docs");- std::fs::create_dir_all(nested_root.join("excluded")).unwrap();- std::fs::create_dir_all(nested_root.join("ok")).unwrap();- std::fs::write(nested_root.join("excluded/x.txt"), "hit\n").unwrap();- std::fs::write(nested_root.join("ok/y.txt"), "hit\n").unwrap();-- let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();- fs.has_rg = false;+ write_file(&nested_root, "excluded/x.txt", "hit\n");+ write_file(&nested_root, "ok/y.txt", "hit\n");// Simulate MountableFS -> LocalFS: query root and exclude path are plugin-relative and nested.let out = fs@@ -1093,37 +1080,17 @@ mod tests {#[cfg(unix)]#[tokio::test]async fn test_localfs_grep_rg_fast_path_disables_parent_ignore_inheritance() {- use tempfile::TempDir;-let dir = TempDir::new().unwrap();- std::fs::write(dir.path().join(".gitignore"), "mount/\n").unwrap();+ write_file(dir.path(), ".gitignore", "mount/\n");let mount_dir = dir.path().join("mount");- std::fs::create_dir_all(&mount_dir).unwrap();- std::fs::write(mount_dir.join("a.txt"), "hello\n").unwrap();+ write_file(&mount_dir, "a.txt", "hello\n");// Fake `rg` verifies `--no-ignore-parent` is present, then emits one JSON match.- let bin_dir = dir.path().join("bin");- std::fs::create_dir_all(&bin_dir).unwrap();- let rg_path = bin_dir.join("rg");- std::fs::write(- &rg_path,- format!(+ let script = format!("#!/bin/sh\nfor arg in \"$@\"; do\n if [ \"$arg\" = \"--no-ignore-parent\" ]; then\n printf '%s\\n' '{{\"type\":\"match\",\"data\":{{\"path\":{{\"text\":\"a.txt\"}},\"line_number\":1,\"lines\":{{\"text\":\"hello\\\\n\"}}}}}}'\n exit 0\n fi\ndone\necho missing --no-ignore-parent 1>&2\nexit 2\n"- ),- )- .unwrap();- use std::os::unix::fs::PermissionsExt;- let mut perms = std::fs::metadata(&rg_path).unwrap().permissions();- perms.set_mode(0o755);- std::fs::set_permissions(&rg_path, perms).unwrap();-- let old_path = std::env::var_os("PATH").unwrap_or_default();- let mut new_path = std::ffi::OsString::new();- new_path.push(bin_dir.as_os_str());- new_path.push(std::ffi::OsStr::new(":"));- new_path.push(old_path);- let _path_guard = EnvVarGuard::set("PATH", &new_path);+ );+ let _path_guard = install_fake_rg(&dir, &script);let fs = LocalFileSystem::new(mount_dir.to_str().unwrap()).unwrap();let result = fsdiff --git a/crates/ragfs/src/plugins/memfs/mod.rs b/crates/ragfs/src/plugins/memfs/mod.rsindex 3d9757a73..444fa36c2 100644--- a/crates/ragfs/src/plugins/memfs/mod.rs+++ b/crates/ragfs/src/plugins/memfs/mod.rs@@ -65,10 +65,7 @@ impl MemFileSystem {let mut entries = HashMap::new();// Create root directory- entries.insert(- "/".to_string(),- FileEntry::new_dir(0o755),- );+ entries.insert("/".to_string(), FileEntry::new_dir(0o755));Self {entries: Arc::new(RwLock::new(entries)),@@ -117,11 +114,7 @@ impl MemFileSystem {}let normalized = Self::normalize_path(path);- normalized- .split('/')- .last()- .unwrap_or("")- .to_string()+ normalized.split('/').last().unwrap_or("").to_string()}/// List entries in a directory@@ -643,11 +636,7 @@ mod tests {let plugin = MemFSPlugin;assert_eq!(plugin.name(), "memfs");- let config = PluginConfig {- name: "memfs".to_string(),- mount_path: "/memfs".to_string(),- params: HashMap::new(),- };+ let config = PluginConfig::single_backend("memfs", "/memfs", HashMap::new());assert!(plugin.validate(&config).await.is_ok());assert!(plugin.initialize(config).await.is_ok());diff --git a/crates/ragfs/src/plugins/queuefs/backend.rs b/crates/ragfs/src/plugins/queuefs/backend.rsindex 49568bcbe..253c2281a 100644--- a/crates/ragfs/src/plugins/queuefs/backend.rs+++ b/crates/ragfs/src/plugins/queuefs/backend.rs@@ -9,11 +9,10 @@ use rusqlite::{params, types::ValueRef, Connection, Row};use serde::{Deserialize, Deserializer, Serialize};use std::collections::{HashMap, VecDeque};use std::sync::Mutex;-use std::time::{Duration, UNIX_EPOCH};use std::time::SystemTime;+use std::time::{Duration, UNIX_EPOCH};use uuid::Uuid;-/// A message in the queue#[derive(Debug, Clone, Serialize, Deserialize)]pub struct Message {@@ -220,7 +219,10 @@ impl MemoryBackend {impl QueueBackend for MemoryBackend {fn create_queue(&mut self, name: &str) -> Result<()> {if self.queues.contains_key(name) {- return Err(Error::AlreadyExists(format!("queue '{}' already exists", name)));+ return Err(Error::AlreadyExists(format!(+ "queue '{}' already exists",+ name+ )));}self.queues.insert(name.to_string(), Queue::new());Ok(())@@ -250,9 +252,10 @@ impl QueueBackend for MemoryBackend {}fn enqueue(&mut self, queue_name: &str, msg: Message) -> Result<()> {- let queue = self.queues.get_mut(queue_name).ok_or_else(|| {- Error::NotFound(format!("queue '{}' not found", queue_name))- })?;+ let queue = self+ .queues+ .get_mut(queue_name)+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;queue.last_enqueue_time = SystemTime::now();queue.messages.push_back(msg);@@ -260,50 +263,56 @@ impl QueueBackend for MemoryBackend {}fn dequeue(&mut self, queue_name: &str) -> Result<Option<Message>> {- let queue = self.queues.get_mut(queue_name).ok_or_else(|| {- Error::NotFound(format!("queue '{}' not found", queue_name))- })?;+ let queue = self+ .queues+ .get_mut(queue_name)+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;Ok(queue.messages.pop_front())}fn peek(&self, queue_name: &str) -> Result<Option<Message>> {- let queue = self.queues.get(queue_name).ok_or_else(|| {- Error::NotFound(format!("queue '{}' not found", queue_name))- })?;+ let queue = self+ .queues+ .get(queue_name)+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;Ok(queue.messages.front().cloned())}fn size(&self, queue_name: &str) -> Result<usize> {- let queue = self.queues.get(queue_name).ok_or_else(|| {- Error::NotFound(format!("queue '{}' not found", queue_name))- })?;+ let queue = self+ .queues+ .get(queue_name)+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;Ok(queue.messages.len())}fn clear(&mut self, queue_name: &str) -> Result<()> {- let queue = self.queues.get_mut(queue_name).ok_or_else(|| {- Error::NotFound(format!("queue '{}' not found", queue_name))- })?;+ let queue = self+ .queues+ .get_mut(queue_name)+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;queue.messages.clear();Ok(())}fn get_last_enqueue_time(&self, queue_name: &str) -> Result<SystemTime> {- let queue = self.queues.get(queue_name).ok_or_else(|| {- Error::NotFound(format!("queue '{}' not found", queue_name))- })?;+ let queue = self+ .queues+ .get(queue_name)+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;Ok(queue.last_enqueue_time)}fn ack(&mut self, queue_name: &str, msg_id: &str) -> Result<bool> {- let queue = self.queues.get_mut(queue_name).ok_or_else(|| {- Error::NotFound(format!("queue '{}' not found", queue_name))- })?;+ let queue = self+ .queues+ .get_mut(queue_name)+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;// Find and remove message by IDlet original_len = queue.messages.len();@@ -397,7 +406,12 @@ impl SQLiteQueueBackend {WHERE queue_name IS NOT NULL AND queue_name != ''",[],)- .map_err(|e| Error::internal(format!("sqlite migration backfill queue metadata error: {}", e)))?;+ .map_err(|e| {+ Error::internal(format!(+ "sqlite migration backfill queue metadata error: {}",+ e+ ))+ })?;Ok(())}@@ -472,7 +486,10 @@ impl QueueBackend for SQLiteQueueBackend {).map_err(|e| Error::internal(format!("sqlite create queue error: {}", e)))?;if changed == 0 {- return Err(Error::AlreadyExists(format!("queue '{}' already exists", name)));+ return Err(Error::AlreadyExists(format!(+ "queue '{}' already exists",+ name+ )));}Ok(())}@@ -518,10 +535,11 @@ impl QueueBackend for SQLiteQueueBackend {};if prefix.is_empty() {- let mut stmt = match conn.prepare("SELECT queue_name FROM queue_metadata ORDER BY queue_name") {- Ok(stmt) => stmt,- Err(_) => return Vec::new(),- };+ let mut stmt =+ match conn.prepare("SELECT queue_name FROM queue_metadata ORDER BY queue_name") {+ Ok(stmt) => stmt,+ Err(_) => return Vec::new(),+ };let rows = match stmt.query_map([], |row| row.get::<_, String>(0)) {Ok(rows) => rows,Err(_) => return Vec::new(),@@ -585,7 +603,12 @@ impl QueueBackend for SQLiteQueueBackend {let (id, raw_data) = match row {Ok(row) => row,Err(rusqlite::Error::QueryReturnedNoRows) => return Ok(None),- Err(e) => return Err(Error::internal(format!("sqlite dequeue query error: {}", e))),+ Err(e) => {+ return Err(Error::internal(format!(+ "sqlite dequeue query error: {}",+ e+ )))+ }};tx.execute(@@ -711,6 +734,24 @@ impl QueueBackend for SQLiteQueueBackend {mod tests {use super::*;use tempfile::tempdir;+ use tempfile::TempDir;++ /// Create an in-memory backend with a test queue.+ fn memory_backend_with_queue(queue: &str) -> MemoryBackend {+ let mut backend = MemoryBackend::new();+ backend.create_queue(queue).unwrap();+ backend+ }++ /// Create a SQLite backend backed by a temporary database file.+ fn sqlite_backend() -> (TempDir, String, SQLiteQueueBackend) {+ let dir = tempdir().unwrap();+ let db_path = dir.path().join("queue.db");+ let db_path_str = db_path.to_str().unwrap().to_string();+ let backend =+ SQLiteQueueBackend::open(&db_path_str, SQLiteQueueOptions::default()).unwrap();+ (dir, db_path_str, backend)+ }#[test]fn test_create_queue() {@@ -726,9 +767,7 @@ mod tests {#[test]fn test_remove_queue() {- let mut backend = MemoryBackend::new();-- backend.create_queue("test").unwrap();+ let mut backend = memory_backend_with_queue("test");backend.remove_queue("test").unwrap();assert!(!backend.queue_exists("test"));@@ -755,8 +794,7 @@ mod tests {#[test]fn test_enqueue_dequeue() {- let mut backend = MemoryBackend::new();- backend.create_queue("test").unwrap();+ let mut backend = memory_backend_with_queue("test");let msg1 = Message::new(b"message 1".to_vec());let msg2 = Message::new(b"message 2".to_vec());@@ -778,8 +816,7 @@ mod tests {#[test]fn test_peek() {- let mut backend = MemoryBackend::new();- backend.create_queue("test").unwrap();+ let mut backend = memory_backend_with_queue("test");let msg = Message::new(b"test message".to_vec());backend.enqueue("test", msg.clone()).unwrap();@@ -796,11 +833,14 @@ mod tests {#[test]fn test_clear() {- let mut backend = MemoryBackend::new();- backend.create_queue("test").unwrap();+ let mut backend = memory_backend_with_queue("test");- backend.enqueue("test", Message::new(b"msg1".to_vec())).unwrap();- backend.enqueue("test", Message::new(b"msg2".to_vec())).unwrap();+ backend+ .enqueue("test", Message::new(b"msg1".to_vec()))+ .unwrap();+ backend+ .enqueue("test", Message::new(b"msg2".to_vec()))+ .unwrap();assert_eq!(backend.size("test").unwrap(), 2);@@ -814,8 +854,12 @@ mod tests {backend.create_queue("queue1").unwrap();backend.create_queue("queue2").unwrap();- backend.enqueue("queue1", Message::new(b"msg1".to_vec())).unwrap();- backend.enqueue("queue2", Message::new(b"msg2".to_vec())).unwrap();+ backend+ .enqueue("queue1", Message::new(b"msg1".to_vec()))+ .unwrap();+ backend+ .enqueue("queue2", Message::new(b"msg2".to_vec()))+ .unwrap();assert_eq!(backend.size("queue1").unwrap(), 1);assert_eq!(backend.size("queue2").unwrap(), 1);@@ -831,7 +875,9 @@ mod tests {fn test_operations_on_nonexistent_queue() {let mut backend = MemoryBackend::new();- assert!(backend.enqueue("nonexistent", Message::new(b"data".to_vec())).is_err());+ assert!(backend+ .enqueue("nonexistent", Message::new(b"data".to_vec()))+ .is_err());assert!(backend.dequeue("nonexistent").is_err());assert!(backend.peek("nonexistent").is_err());assert!(backend.size("nonexistent").is_err());@@ -840,11 +886,7 @@ mod tests {#[test]fn test_sqlite_backend_basic_flow() {- let dir = tempdir().unwrap();- let db_path = dir.path().join("queue.db");- let mut backend =- SQLiteQueueBackend::open(db_path.to_str().unwrap(), SQLiteQueueOptions::default())- .unwrap();+ let (_dir, _db_path, mut backend) = sqlite_backend();backend.create_queue("test").unwrap();let msg1 = Message::new(b"message 1".to_vec());@@ -865,15 +907,13 @@ mod tests {#[test]fn test_sqlite_backend_preserves_non_utf8_payload_bytes() {- let dir = tempdir().unwrap();- let db_path = dir.path().join("queue.db");- let mut backend =- SQLiteQueueBackend::open(db_path.to_str().unwrap(), SQLiteQueueOptions::default())- .unwrap();+ let (_dir, _db_path, mut backend) = sqlite_backend();backend.create_queue("test").unwrap();let payload = vec![0xff, 0x00, 0x80, b'a'];- backend.enqueue("test", Message::new(payload.clone())).unwrap();+ backend+ .enqueue("test", Message::new(payload.clone()))+ .unwrap();let peeked = backend.peek("test").unwrap().unwrap();assert_eq!(peeked.data, payload);@@ -884,9 +924,7 @@ mod tests {#[test]fn test_sqlite_backend_recover_stale() {- let dir = tempdir().unwrap();- let db_path = dir.path().join("queue.db");- let db_path_str = db_path.to_str().unwrap().to_string();+ let (_dir, db_path_str, _backend) = sqlite_backend();let msg_id = {let mut backend =@@ -909,12 +947,8 @@ mod tests {#[test]fn test_sqlite_backend_dequeue_legacy_go_row() {- let dir = tempdir().unwrap();- let db_path = dir.path().join("queue.db");- let db_path_str = db_path.to_str().unwrap().to_string();+ let (_dir, db_path_str, mut backend) = sqlite_backend();- let mut backend =- SQLiteQueueBackend::open(&db_path_str, SQLiteQueueOptions::default()).unwrap();backend.create_queue("Semantic").unwrap();drop(backend);@@ -997,13 +1031,11 @@ mod tests {#[test]fn test_sqlite_backend_operations_on_nonexistent_queue() {- let dir = tempdir().unwrap();- let db_path = dir.path().join("queue.db");- let mut backend =- SQLiteQueueBackend::open(db_path.to_str().unwrap(), SQLiteQueueOptions::default())- .unwrap();+ let (_dir, _db_path, mut backend) = sqlite_backend();- assert!(backend.enqueue("nonexistent", Message::new(b"data".to_vec())).is_err());+ assert!(backend+ .enqueue("nonexistent", Message::new(b"data".to_vec()))+ .is_err());assert!(backend.dequeue("nonexistent").is_err());assert!(backend.peek("nonexistent").is_err());assert!(backend.size("nonexistent").is_err());diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rsindex 2e69e86c4..12724272a 100644--- a/crates/ragfs/src/plugins/queuefs/mod.rs+++ b/crates/ragfs/src/plugins/queuefs/mod.rs@@ -199,9 +199,7 @@ impl FileSystem for QueueFileSystem {async fn mkdir(&self, path: &str, _mode: u32) -> Result<()> {let parsed = Self::parse_queue_path(path)?;if !parsed.is_dir {- return Err(Error::InvalidOperation(- "not a directory path".to_string(),- ));+ return Err(Error::InvalidOperation("not a directory path".to_string()));}if let Some(queue_name) = parsed.queue_name {self.backend.lock().await.create_queue(&queue_name)?;@@ -260,13 +258,7 @@ impl FileSystem for QueueFileSystem {}}- async fn write(- &self,- path: &str,- data: &[u8],- _offset: u64,- _flags: WriteFlag,- ) -> Result<u64> {+ async fn write(&self, path: &str, data: &[u8], _offset: u64, _flags: WriteFlag) -> Result<u64> {let parsed = Self::parse_queue_path(path)?;let queue_name = parsed@@ -369,10 +361,7 @@ impl FileSystem for QueueFileSystem {// Leaf queue: return control filesif !backend.queue_exists(&queue_name) {- return Err(Error::NotFound(format!(- "queue not found: {}",- queue_name- )));+ return Err(Error::NotFound(format!("queue not found: {}", queue_name)));}Ok(Self::leaf_control_files(now))@@ -399,7 +388,11 @@ impl FileSystem for QueueFileSystem {let queue_name = parsed.queue_name.unwrap();if backend.queue_exists(&queue_name) {Ok(FileInfo {- name: queue_name.split('/').last().unwrap_or(&queue_name).to_string(),+ name: queue_name+ .split('/')+ .last()+ .unwrap_or(&queue_name)+ .to_string(),size: 0,mode: 0o755,mod_time: SystemTime::now(),@@ -414,8 +407,9 @@ impl FileSystem for QueueFileSystem {Ok(FileInfo {name: operation.clone(),size: 0,- mode: Self::control_file_mode(operation)- .ok_or_else(|| Error::NotFound(format!("control file not found: {}", operation)))?,+ mode: Self::control_file_mode(operation).ok_or_else(|| {+ Error::NotFound(format!("control file not found: {}", operation))+ })?,mod_time: SystemTime::now(),is_dir: false,})@@ -444,9 +438,7 @@ impl FileSystem for QueueFileSystem {let parsed = Self::parse_queue_path(path)?;if !parsed.is_dir {- return Err(Error::InvalidOperation(- "not a directory".to_string(),- ));+ return Err(Error::InvalidOperation("not a directory".to_string()));}if let Some(queue_name) = parsed.queue_name {@@ -665,32 +657,48 @@ mod tests {data: String,}- #[tokio::test]- async fn test_queuefs_enqueue_dequeue() {+ /// Create a queue filesystem with one initialized queue.+ async fn queuefs_with(queue: &str) -> QueueFileSystem {let fs = QueueFileSystem::new();+ fs.mkdir(&format!("/{queue}"), 0o755).await.unwrap();+ fs+ }- // Create a queue first- fs.mkdir("/test", 0o755).await.unwrap();+ /// Enqueue one message into a test queue.+ async fn enqueue(fs: &dyn FileSystem, queue: &str, data: &[u8]) {+ fs.write(&format!("/{queue}/enqueue"), data, 0, WriteFlag::None)+ .await+ .unwrap();+ }++ /// Dequeue and deserialize one message from a test queue.+ async fn dequeue_msg(fs: &dyn FileSystem, queue: &str) -> TestQueueMessage {+ let result = fs.read(&format!("/{queue}/dequeue"), 0, 0).await.unwrap();+ serde_json::from_slice(&result).unwrap()+ }++ /// Read the queue size control file as usize.+ async fn queue_size(fs: &dyn FileSystem, queue: &str) -> usize {+ let size = fs.read(&format!("/{queue}/size"), 0, 0).await.unwrap();+ String::from_utf8(size).unwrap().parse().unwrap()+ }++ #[tokio::test]+ async fn test_queuefs_enqueue_dequeue() {+ let fs = queuefs_with("test").await;// Enqueue messageslet data1 = b"message 1";let data2 = b"message 2";-- fs.write("/test/enqueue", data1, 0, WriteFlag::None)- .await- .unwrap();- fs.write("/test/enqueue", data2, 0, WriteFlag::None)- .await- .unwrap();+ enqueue(&fs, "test", data1).await;+ enqueue(&fs, "test", data2).await;// Dequeue messages- let result1 = fs.read("/test/dequeue", 0, 0).await.unwrap();- let msg1: TestQueueMessage = serde_json::from_slice(&result1).unwrap();+ let msg1 = dequeue_msg(&fs, "test").await;assert!(!msg1.id.is_empty());assert_eq!(msg1.data.as_bytes(), data1);- let result2 = fs.read("/test/dequeue", 0, 0).await.unwrap();- let msg2: TestQueueMessage = serde_json::from_slice(&result2).unwrap();+ let msg2 = dequeue_msg(&fs, "test").await;assert!(!msg2.id.is_empty());assert_eq!(msg2.data.as_bytes(), data2);@@ -701,16 +709,11 @@ mod tests {#[tokio::test]async fn test_queuefs_peek() {- let fs = QueueFileSystem::new();-- // Create a queue first- fs.mkdir("/test", 0o755).await.unwrap();+ let fs = queuefs_with("test").await;// Enqueue a messagelet data = b"test message";- fs.write("/test/enqueue", data, 0, WriteFlag::None)- .await- .unwrap();+ enqueue(&fs, "test", data).await;// Peek should return the message without removing itlet result1 = fs.read("/test/peek", 0, 0).await.unwrap();@@ -722,54 +725,36 @@ mod tests {assert_eq!(msg2.data.as_bytes(), data);// Dequeue should still work- let result3 = fs.read("/test/dequeue", 0, 0).await.unwrap();- let msg3: TestQueueMessage = serde_json::from_slice(&result3).unwrap();+ let msg3 = dequeue_msg(&fs, "test").await;assert_eq!(msg3.data.as_bytes(), data);}#[tokio::test]async fn test_queuefs_size() {- let fs = QueueFileSystem::new();-- // Create a queue first- fs.mkdir("/test", 0o755).await.unwrap();+ let fs = queuefs_with("test").await;// Initially empty- let size = fs.read("/test/size", 0, 0).await.unwrap();- assert_eq!(String::from_utf8(size).unwrap(), "0");+ assert_eq!(queue_size(&fs, "test").await, 0);// Add messages- fs.write("/test/enqueue", b"msg1", 0, WriteFlag::None)- .await- .unwrap();- fs.write("/test/enqueue", b"msg2", 0, WriteFlag::None)- .await- .unwrap();+ enqueue(&fs, "test", b"msg1").await;+ enqueue(&fs, "test", b"msg2").await;- let size = fs.read("/test/size", 0, 0).await.unwrap();- assert_eq!(String::from_utf8(size).unwrap(), "2");+ assert_eq!(queue_size(&fs, "test").await, 2);// Dequeue onefs.read("/test/dequeue", 0, 0).await.unwrap();- let size = fs.read("/test/size", 0, 0).await.unwrap();- assert_eq!(String::from_utf8(size).unwrap(), "1");+ assert_eq!(queue_size(&fs, "test").await, 1);}#[tokio::test]async fn test_queuefs_clear() {- let fs = QueueFileSystem::new();-- // Create a queue first- fs.mkdir("/test", 0o755).await.unwrap();+ let fs = queuefs_with("test").await;// Add messages- fs.write("/test/enqueue", b"msg1", 0, WriteFlag::None)- .await- .unwrap();- fs.write("/test/enqueue", b"msg2", 0, WriteFlag::None)- .await- .unwrap();+ enqueue(&fs, "test", b"msg1").await;+ enqueue(&fs, "test", b"msg2").await;// Clear the queuefs.write("/test/clear", b"", 0, WriteFlag::None)@@ -777,8 +762,7 @@ mod tests {.unwrap();// Queue should be empty- let size = fs.read("/test/size", 0, 0).await.unwrap();- assert_eq!(String::from_utf8(size).unwrap(), "0");+ assert_eq!(queue_size(&fs, "test").await, 0);let result = fs.read("/test/dequeue", 0, 0).await;assert_eq!(result.unwrap(), b"{}");@@ -786,10 +770,7 @@ mod tests {#[tokio::test]async fn test_queuefs_read_dir() {- let fs = QueueFileSystem::new();-- // Create a queue- fs.mkdir("/test", 0o755).await.unwrap();+ let fs = queuefs_with("test").await;// Root should list the queuelet entries = fs.read_dir("/").await.unwrap();@@ -812,10 +793,7 @@ mod tests {#[tokio::test]async fn test_queuefs_stat() {- let fs = QueueFileSystem::new();-- // Create a queue- fs.mkdir("/test", 0o755).await.unwrap();+ let fs = queuefs_with("test").await;// Stat rootlet info = fs.stat("/").await.unwrap();@@ -837,10 +815,7 @@ mod tests {#[tokio::test]async fn test_queuefs_invalid_operations() {- let fs = QueueFileSystem::new();-- // Create a queue- fs.mkdir("/test", 0o755).await.unwrap();+ let fs = queuefs_with("test").await;// Cannot read from enqueuelet result = fs.read("/test/enqueue", 0, 0).await;@@ -886,8 +861,7 @@ mod tests {}// Check size- let size = fs.read("/test/size", 0, 0).await.unwrap();- assert_eq!(String::from_utf8(size).unwrap(), "10");+ assert_eq!(queue_size(fs.as_ref(), "test").await, 10);// Dequeue all messagesfor _ in 0..10 {@@ -895,8 +869,7 @@ mod tests {}// Queue should be empty- let size = fs.read("/test/size", 0, 0).await.unwrap();- assert_eq!(String::from_utf8(size).unwrap(), "0");+ assert_eq!(queue_size(fs.as_ref(), "test").await, 0);}#[tokio::test]@@ -907,11 +880,8 @@ mod tests {assert!(!plugin.readme().is_empty());assert_eq!(plugin.config_params().len(), 4);- let config = PluginConfig {- name: "queuefs".to_string(),- mount_path: "/queue".to_string(),- params: std::collections::HashMap::new(),- };+ let config =+ PluginConfig::single_backend("queuefs", "/queue", std::collections::HashMap::new());plugin.validate(&config).await.unwrap();let fs = plugin.initialize(config).await.unwrap();@@ -920,11 +890,8 @@ mod tests {fs.mkdir("/test", 0o755).await.unwrap();// Test basic operation- fs.write("/test/enqueue", b"test", 0, WriteFlag::None)- .await- .unwrap();- let result = fs.read("/test/dequeue", 0, 0).await.unwrap();- let msg: TestQueueMessage = serde_json::from_slice(&result).unwrap();+ enqueue(fs.as_ref(), "test", b"test").await;+ let msg = dequeue_msg(fs.as_ref(), "test").await;assert_eq!(msg.data, "test");}@@ -937,27 +904,19 @@ mod tests {fs.mkdir("/Semantic", 0o755).await.unwrap();// Enqueue to both- fs.write("/Embedding/enqueue", b"embed1", 0, WriteFlag::None)- .await- .unwrap();- fs.write("/Semantic/enqueue", b"semantic1", 0, WriteFlag::None)- .await- .unwrap();+ enqueue(&fs, "Embedding", b"embed1").await;+ enqueue(&fs, "Semantic", b"semantic1").await;// Verify isolation- let size1 = fs.read("/Embedding/size", 0, 0).await.unwrap();- let size2 = fs.read("/Semantic/size", 0, 0).await.unwrap();- assert_eq!(String::from_utf8(size1).unwrap(), "1");- assert_eq!(String::from_utf8(size2).unwrap(), "1");+ assert_eq!(queue_size(&fs, "Embedding").await, 1);+ assert_eq!(queue_size(&fs, "Semantic").await, 1);// Dequeue from specific queue- let msg = fs.read("/Embedding/dequeue", 0, 0).await.unwrap();- let msg: TestQueueMessage = serde_json::from_slice(&msg).unwrap();+ let msg = dequeue_msg(&fs, "Embedding").await;assert_eq!(msg.data, "embed1");// Other queue unaffected- let size2 = fs.read("/Semantic/size", 0, 0).await.unwrap();- assert_eq!(String::from_utf8(size2).unwrap(), "1");+ assert_eq!(queue_size(&fs, "Semantic").await, 1);}#[tokio::test]@@ -977,11 +936,8 @@ mod tests {assert!(names.contains(&"warnings"));// Can enqueue to nested queue- fs.write("/logs/errors/enqueue", b"error1", 0, WriteFlag::None)- .await- .unwrap();- let msg = fs.read("/logs/errors/dequeue", 0, 0).await.unwrap();- let msg: TestQueueMessage = serde_json::from_slice(&msg).unwrap();+ enqueue(&fs, "logs/errors", b"error1").await;+ let msg = dequeue_msg(&fs, "logs/errors").await;assert_eq!(msg.data, "error1");}@@ -991,13 +947,10 @@ mod tests {// Create queuefs.mkdir("/temp", 0o755).await.unwrap();- fs.write("/temp/enqueue", b"data", 0, WriteFlag::None)- .await- .unwrap();+ enqueue(&fs, "temp", b"data").await;// Verify exists- let size = fs.read("/temp/size", 0, 0).await.unwrap();- assert_eq!(String::from_utf8(size).unwrap(), "1");+ assert_eq!(queue_size(&fs, "temp").await, 1);// Delete queuefs.remove_all("/temp").await.unwrap();diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rsindex f9c29b663..79f0bca49 100644--- a/crates/ragfs/src/plugins/s3fs/client.rs+++ b/crates/ragfs/src/plugins/s3fs/client.rs@@ -301,14 +301,26 @@ impl S3Client {/// Get an object's contentspub async fn get_object(&self, key: &str) -> Result<Vec<u8>> {- let resp = self+ let resp = match self.client.get_object().bucket(&self.bucket).key(key).send().await- .map_err(|e| Error::internal(format!("S3 GetObject error: {}", e)))?;+ {+ Ok(resp) => resp,+ Err(sdk_err) => {+ let service_err = sdk_err.into_service_error();+ if service_err.is_no_such_key() {+ return Err(Error::NotFound(key.to_string()));+ }+ return Err(Error::internal(format!(+ "S3 GetObject error: {}",+ service_err+ )));+ }+ };let bytes = resp.body@@ -320,19 +332,14 @@ impl S3Client {}/// Get an object's contents with range request- pub async fn get_object_range(- &self,- key: &str,- offset: u64,- size: u64,- ) -> Result<Vec<u8>> {+ pub async fn get_object_range(&self, key: &str, offset: u64, size: u64) -> Result<Vec<u8>> {let range = if size == 0 {format!("bytes={}-", offset)} else {format!("bytes={}-{}", offset, offset + size - 1)};- let resp = self+ let resp = match self.client.get_object().bucket(&self.bucket)@@ -340,7 +347,19 @@ impl S3Client {.range(range).send().await- .map_err(|e| Error::internal(format!("S3 GetObject range error: {}", e)))?;+ {+ Ok(resp) => resp,+ Err(sdk_err) => {+ let service_err = sdk_err.into_service_error();+ if service_err.is_no_such_key() {+ return Err(Error::NotFound(key.to_string()));+ }+ return Err(Error::internal(format!(+ "S3 GetObject range error: {}",+ service_err+ )));+ }+ };let bytes = resp.body@@ -471,11 +490,7 @@ impl S3Client {}/// List objects with prefix and delimiter- pub async fn list_objects(- &self,- prefix: &str,- delimiter: Option<&str>,- ) -> Result<ListResult> {+ pub async fn list_objects(&self, prefix: &str, delimiter: Option<&str>) -> Result<ListResult> {let mut files = Vec::new();let mut directories = Vec::new();let mut continuation_token: Option<String> = None;diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rsindex 2aa82aaf1..5be685a0d 100644--- a/crates/ragfs/src/plugins/s3fs/mod.rs+++ b/crates/ragfs/src/plugins/s3fs/mod.rs@@ -1243,64 +1243,45 @@ mod tests {}#[test]- fn test_relative_match_file_same_path() {- assert_eq!(relative_match_file("/foo", "/foo"), ".");- assert_eq!(relative_match_file("/", "/"), ".");- }-- #[test]- fn test_relative_match_file_root_base() {- assert_eq!(relative_match_file("/", "/a"), "a");- assert_eq!(relative_match_file("/", "/a/b"), "a/b");- assert_eq!(- relative_match_file("/", "/deep/nested/file"),- "deep/nested/file"- );- }-- #[test]- fn test_relative_match_file_subdirectory() {- assert_eq!(relative_match_file("/a", "/a/b"), "b");- assert_eq!(relative_match_file("/a", "/a/b/c"), "b/c");- assert_eq!(relative_match_file("/dir", "/dir/file.txt"), "file.txt");- }-- #[test]- fn test_relative_match_file_no_prefix_match() {- assert_eq!(relative_match_file("/a", "/b"), "b");- assert_eq!(relative_match_file("/x", "/y/z"), "y/z");- }-- #[test]- fn test_relative_match_file_trailing_slash_base() {- assert_eq!(relative_match_file("/foo/", "/foo/bar"), "bar");- assert_eq!(relative_match_file("/foo/", "/foo"), ".");- }-- #[test]- fn test_relative_match_file_empty_remainder() {- assert_eq!(relative_match_file("/dir", "/dir/"), ".");- }-- #[test]- fn test_relative_match_file_leading_slash_in_rest() {- assert_eq!(relative_match_file("/dir", "/dir//sub"), "sub");- }-- #[test]- fn test_relative_depth() {- assert_eq!(relative_depth("."), 0);- assert_eq!(relative_depth(""), 0);- assert_eq!(relative_depth("a"), 1);- assert_eq!(relative_depth("a/b"), 2);- assert_eq!(relative_depth("a/b/c"), 3);- assert_eq!(relative_depth("deep/nested/path/to/file"), 5);+ fn test_relative_match_file_cases() {+ for (base, path, expected) in [+ ("/foo", "/foo", "."),+ ("/", "/", "."),+ ("/", "/a", "a"),+ ("/", "/a/b", "a/b"),+ ("/", "/deep/nested/file", "deep/nested/file"),+ ("/a", "/a/b", "b"),+ ("/a", "/a/b/c", "b/c"),+ ("/dir", "/dir/file.txt", "file.txt"),+ ("/a", "/b", "b"),+ ("/x", "/y/z", "y/z"),+ ("/foo/", "/foo/bar", "bar"),+ ("/foo/", "/foo", "."),+ ("/dir", "/dir/", "."),+ ("/dir", "/dir//sub", "sub"),+ ] {+ assert_eq!(+ relative_match_file(base, path),+ expected,+ "{base} -> {path}"+ );+ }}#[test]- fn test_relative_depth_filters_empty_segments() {- assert_eq!(relative_depth("a//b"), 2);- assert_eq!(relative_depth("///"), 0);+ fn test_relative_depth_cases() {+ for (path, expected) in [+ (".", 0),+ ("", 0),+ ("a", 1),+ ("a/b", 2),+ ("a/b/c", 3),+ ("deep/nested/path/to/file", 5),+ ("a//b", 2),+ ("///", 0),+ ] {+ assert_eq!(relative_depth(path), expected, "{path}");+ }}#[test]@@ -1334,6 +1315,29 @@ mod tests {Regex::new(pattern).unwrap()}+ /// Run grep_stream against in-memory chunk data.+ async fn grep_chunks(+ data: impl AsRef<[u8]>,+ pattern: &str,+ chunk_size: u64,+ max_partial: usize,+ ) -> Result<Vec<GrepMatch>> {+ let data = data.as_ref().to_vec();+ let file_size = data.len() as u64;+ let mut reader = MockChunkReader { data };+ let re = build_re(pattern);+ grep_stream(+ "f",+ file_size,+ &re,+ 100,+ chunk_size,+ max_partial,+ &mut reader,+ )+ .await+ }+// ── Case 3: 正则无效 ──#[test]@@ -1367,14 +1371,7 @@ mod tests {#[tokio::test]async fn test_grep_stream_exact_chunk_single() {let data = b"hello\nworld\n";- let file_size = data.len() as u64;- let chunk_size = file_size;- let mut reader = MockChunkReader {- data: data.to_vec(),- };- let re = build_re(".");-- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)+ let matches = grep_chunks(data, ".", data.len() as u64, 1024).await.unwrap();@@ -1390,16 +1387,7 @@ mod tests {#[tokio::test]async fn test_grep_stream_last_chunk_exact_size() {let data = b"line1\nline2\nline3\nline4\n";- let file_size = data.len() as u64; // 24- let chunk_size = 12u64;- let mut reader = MockChunkReader {- data: data.to_vec(),- };- let re = build_re("line");-- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)- .await- .unwrap();+ let matches = grep_chunks(data, "line", 12, 1024).await.unwrap();assert_eq!(matches.len(), 4, "all 4 lines should match");assert_eq!(matches[3].content, "line4");@@ -1410,16 +1398,7 @@ mod tests {#[tokio::test]async fn test_grep_stream_cross_chunk_line() {let data = b"hel\nlo wo\nrld\n";- let file_size = data.len() as u64; // 15- let chunk_size = 8u64;- let mut reader = MockChunkReader {- data: data.to_vec(),- };- let re = build_re("lo");-- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)- .await- .unwrap();+ let matches = grep_chunks(data, "lo", 8, 1024).await.unwrap();assert_eq!(matches.len(), 1, "only 'lo wo' contains 'lo'");assert_eq!(matches[0].line, 2, "line number should be 2");@@ -1431,16 +1410,7 @@ mod tests {#[tokio::test]async fn test_grep_stream_multi_chunk_no_newline() {let data = b"abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()"; // 46 bytes, no \n- let file_size = data.len() as u64;- let chunk_size = 10u64;- let mut reader = MockChunkReader {- data: data.to_vec(),- };- let re = build_re("xyz");-- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)- .await- .unwrap();+ let matches = grep_chunks(data, "xyz", 10, 1024).await.unwrap();assert_eq!(matches.len(), 1, "one long line should match");assert_eq!(matches[0].line, 1);@@ -1456,26 +1426,7 @@ mod tests {#[tokio::test]async fn test_grep_stream_line_exceeds_max_partial() {let content = "a".repeat(200);- let data = content.as_bytes();- let file_size = data.len() as u64; // 200- let chunk_size = 64u64;- let max_partial = 100usize;- let mut reader = MockChunkReader {- data: data.to_vec(),- };- let re = build_re("a");-- let matches = grep_stream(- "f",- file_size,- &re,- 100,- chunk_size,- max_partial,- &mut reader,- )- .await- .unwrap();+ let matches = grep_chunks(content.as_bytes(), "a", 64, 100).await.unwrap();assert!(matches.is_empty(),@@ -1490,14 +1441,7 @@ mod tests {let mut data: Vec<u8> = b"hello\nworld\n".to_vec();data.extend_from_slice(&[0xff, 0xfe, 0xfd]);data.extend_from_slice(b"\nvalid\n");- let file_size = data.len() as u64;- let chunk_size = 64u64;- let mut reader = MockChunkReader { data };- let re = build_re("world");-- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)- .await- .unwrap();+ let matches = grep_chunks(data, "world", 64, 1024).await.unwrap();assert_eq!(matches.len(),@@ -1513,14 +1457,7 @@ mod tests {let mut data: Vec<u8> = b"first\n".to_vec();data.extend_from_slice(&[0xff, 0xfe, 0xfd, b'\n']);data.extend_from_slice(b"last\n");- let file_size = data.len() as u64;- let chunk_size = 64u64;- let mut reader = MockChunkReader { data };- let re = build_re("last");-- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)- .await- .unwrap();+ let matches = grep_chunks(data, "last", 64, 1024).await.unwrap();assert_eq!(matches.len(), 1, "should match 'last' after binary line");assert_eq!(matches[0].content, "last");diff --git a/crates/ragfs/src/plugins/serverinfofs/mod.rs b/crates/ragfs/src/plugins/serverinfofs/mod.rsindex 13cdabadb..72df71aea 100644--- a/crates/ragfs/src/plugins/serverinfofs/mod.rs+++ b/crates/ragfs/src/plugins/serverinfofs/mod.rs@@ -49,7 +49,13 @@ impl ServerInfoFileSystem {}}"#,self.version,format_duration(uptime),- format_timestamp(UNIX_EPOCH.elapsed().unwrap_or(Duration::from_secs(0)).as_secs() - uptime_secs),+ format_timestamp(+ UNIX_EPOCH+ .elapsed()+ .unwrap_or(Duration::from_secs(0))+ .as_secs()+ - uptime_secs+ ),env!("CARGO_PKG_RUST_VERSION"))}@@ -128,19 +134,27 @@ VERSION: 1.0.0#[async_trait]impl FileSystem for ServerInfoFileSystem {async fn create(&self, _path: &str) -> Result<()> {- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))+ Err(Error::plugin(+ "operation not permitted: serverinfofs is read-only".to_string(),+ ))}async fn mkdir(&self, _path: &str, _mode: u32) -> Result<()> {- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))+ Err(Error::plugin(+ "operation not permitted: serverinfofs is read-only".to_string(),+ ))}async fn remove(&self, _path: &str) -> Result<()> {- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))+ Err(Error::plugin(+ "operation not permitted: serverinfofs is read-only".to_string(),+ ))}async fn remove_all(&self, _path: &str) -> Result<()> {- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))+ Err(Error::plugin(+ "operation not permitted: serverinfofs is read-only".to_string(),+ ))}async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {@@ -185,8 +199,16 @@ impl FileSystem for ServerInfoFileSystem {}}- async fn write(&self, _path: &str, _data: &[u8], _offset: u64, _flags: WriteFlag) -> Result<u64> {- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))+ async fn write(+ &self,+ _path: &str,+ _data: &[u8],+ _offset: u64,+ _flags: WriteFlag,+ ) -> Result<u64> {+ Err(Error::plugin(+ "operation not permitted: serverinfofs is read-only".to_string(),+ ))}async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {@@ -205,9 +227,21 @@ impl FileSystem for ServerInfoFileSystem {Ok(vec![FileInfo::new("README".to_string(), readme.len() as u64, 0o444, now, false),- FileInfo::new("server_info".to_string(), server_info.len() as u64, 0o444, now, false),+ FileInfo::new(+ "server_info".to_string(),+ server_info.len() as u64,+ 0o444,+ now,+ false,+ ),FileInfo::new("uptime".to_string(), uptime.len() as u64, 0o444, now, false),- FileInfo::new("version".to_string(), version.len() as u64, 0o444, now, false),+ FileInfo::new(+ "version".to_string(),+ version.len() as u64,+ 0o444,+ now,+ false,+ ),FileInfo::new("stats".to_string(), stats.len() as u64, 0o444, now, false),])}@@ -234,15 +268,25 @@ impl FileSystem for ServerInfoFileSystem {};let name = path.strip_prefix('/').unwrap_or(path);- Ok(FileInfo::new(name.to_string(), data.len() as u64, 0o444, now, false))+ Ok(FileInfo::new(+ name.to_string(),+ data.len() as u64,+ 0o444,+ now,+ false,+ ))}async fn rename(&self, _old_path: &str, _new_path: &str) -> Result<()> {- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))+ Err(Error::plugin(+ "operation not permitted: serverinfofs is read-only".to_string(),+ ))}async fn chmod(&self, _path: &str, _mode: u32) -> Result<()> {- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))+ Err(Error::plugin(+ "operation not permitted: serverinfofs is read-only".to_string(),+ ))}}diff --git a/crates/ragfs/src/plugins/sqlfs/backend.rs b/crates/ragfs/src/plugins/sqlfs/backend.rsindex 7c32dc3e3..5fa01efb7 100644--- a/crates/ragfs/src/plugins/sqlfs/backend.rs+++ b/crates/ragfs/src/plugins/sqlfs/backend.rs@@ -37,11 +37,8 @@ pub trait DatabaseBackend: Send + Sync {fn delete_entry(&self, path: &str) -> Result<()>;/// Delete entries matching a pattern (for recursive delete)- fn delete_entries_by_pattern(- &self,- pattern: &str,- exclude_path: Option<&str>,- ) -> Result<usize>;+ fn delete_entries_by_pattern(&self, pattern: &str, exclude_path: Option<&str>)+ -> Result<usize>;/// Read file datafn read_file(&self, path: &str) -> Result<Option<(bool, Vec<u8>)>>;@@ -152,7 +149,10 @@ impl DatabaseBackend for SQLiteBackend {}fn path_exists(&self, path: &str) -> Result<bool> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let mut stmt = conn.prepare_cached("SELECT COUNT(*) FROM files WHERE path = ?1").map_err(|e| Error::internal(format!("prepare error: {}", e)))?;@@ -167,7 +167,10 @@ impl DatabaseBackend for SQLiteBackend {}fn is_directory(&self, path: &str) -> Result<bool> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let mut stmt = conn.prepare_cached("SELECT is_dir FROM files WHERE path = ?1").map_err(|e| Error::internal(format!("prepare error: {}", e)))?;@@ -180,7 +183,10 @@ impl DatabaseBackend for SQLiteBackend {}fn create_file(&self, path: &str, mode: u32, data: &[u8]) -> Result<()> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let now = chrono::Utc::now().timestamp();conn.execute("INSERT INTO files (path, is_dir, mode, size, mod_time, data) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",@@ -191,7 +197,10 @@ impl DatabaseBackend for SQLiteBackend {}fn create_directory(&self, path: &str, mode: u32) -> Result<()> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let now = chrono::Utc::now().timestamp();conn.execute("INSERT INTO files (path, is_dir, mode, size, mod_time, data) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",@@ -202,7 +211,10 @@ impl DatabaseBackend for SQLiteBackend {}fn delete_entry(&self, path: &str) -> Result<()> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;conn.execute("DELETE FROM files WHERE path = ?1", params![path]).map_err(|e| Error::internal(format!("delete error: {}", e)))?;Ok(())@@ -213,7 +225,10 @@ impl DatabaseBackend for SQLiteBackend {pattern: &str,exclude_path: Option<&str>,) -> Result<usize> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let result = if let Some(exclude) = exclude_path {conn.execute(@@ -230,7 +245,10 @@ impl DatabaseBackend for SQLiteBackend {}fn read_file(&self, path: &str) -> Result<Option<(bool, Vec<u8>)>> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let mut stmt = conn.prepare_cached("SELECT is_dir, data FROM files WHERE path = ?1").map_err(|e| Error::internal(format!("prepare error: {}", e)))?;@@ -247,7 +265,10 @@ impl DatabaseBackend for SQLiteBackend {}fn update_file(&self, path: &str, data: &[u8]) -> Result<()> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let now = chrono::Utc::now().timestamp();conn.execute("UPDATE files SET data = ?1, size = ?2, mod_time = ?3 WHERE path = ?4",@@ -258,11 +279,12 @@ impl DatabaseBackend for SQLiteBackend {}fn get_metadata(&self, path: &str) -> Result<Option<FileMetadata>> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let mut stmt = conn- .prepare_cached(- "SELECT path, is_dir, mode, size, mod_time FROM files WHERE path = ?1",- )+ .prepare_cached("SELECT path, is_dir, mode, size, mod_time FROM files WHERE path = ?1").map_err(|e| Error::internal(format!("prepare error: {}", e)))?;match stmt.query_row(params![path], |row| {@@ -282,7 +304,10 @@ impl DatabaseBackend for SQLiteBackend {}fn update_mode(&self, path: &str, mode: u32) -> Result<()> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let now = chrono::Utc::now().timestamp();conn.execute("UPDATE files SET mode = ?1, mod_time = ?2 WHERE path = ?3",@@ -293,7 +318,10 @@ impl DatabaseBackend for SQLiteBackend {}fn rename_path(&self, old_path: &str, new_path: &str) -> Result<()> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;conn.execute("UPDATE files SET path = ?1 WHERE path = ?2",params![new_path, old_path],@@ -303,7 +331,10 @@ impl DatabaseBackend for SQLiteBackend {}fn rename_children(&self, old_path: &str, new_path: &str) -> Result<()> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let old_pattern = format!("{}/%", old_path);let old_len = (old_path.len() + 1) as i32;let sql = "UPDATE files SET path = ?1 || SUBSTR(path, ?2) WHERE path LIKE ?3";@@ -313,7 +344,10 @@ impl DatabaseBackend for SQLiteBackend {}fn list_directory(&self, path: &str) -> Result<Vec<FileMetadata>> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;// Build pattern for direct children only// For root "/": children are like "/<name>" (no further slashes)@@ -350,8 +384,7 @@ impl DatabaseBackend for SQLiteBackend {.map_err(|e| Error::internal(format!("query error: {}", e)))?;for row_result in rows {- let meta =- row_result.map_err(|e| Error::internal(format!("row error: {}", e)))?;+ let meta = row_result.map_err(|e| Error::internal(format!("row error: {}", e)))?;// Only include direct children (no further '/' after the prefix)let remainder = &meta.path[prefix_len..];@@ -364,7 +397,10 @@ impl DatabaseBackend for SQLiteBackend {}fn count_by_pattern(&self, pattern: &str) -> Result<i64> {- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;+ let conn = self+ .conn+ .lock()+ .map_err(|e| Error::internal(e.to_string()))?;let mut stmt = conn.prepare_cached("SELECT COUNT(*) FROM files WHERE path LIKE ?1").map_err(|e| Error::internal(format!("prepare error: {}", e)))?;@@ -451,7 +487,9 @@ mod tests {assert!(backend.is_directory("/testdir").unwrap());// Create a file- backend.create_file("/testdir/file.txt", 0o644, b"hello").unwrap();+ backend+ .create_file("/testdir/file.txt", 0o644, b"hello")+ .unwrap();assert!(backend.path_exists("/testdir/file.txt").unwrap());assert!(!backend.is_directory("/testdir/file.txt").unwrap());diff --git a/crates/ragfs/src/plugins/sqlfs/cache.rs b/crates/ragfs/src/plugins/sqlfs/cache.rsindex dc4fa105d..c2a46fa88 100644--- a/crates/ragfs/src/plugins/sqlfs/cache.rs+++ b/crates/ragfs/src/plugins/sqlfs/cache.rs@@ -147,9 +147,7 @@ impl ListDirCache {let to_invalidate: Vec<String> = inner.cache.iter()- .filter(|(path, _)| {- *path == prefix || is_descendant(path, prefix)- })+ .filter(|(path, _)| *path == prefix || is_descendant(path, prefix)).map(|(path, _)| path.clone()).collect();diff --git a/crates/ragfs/src/plugins/sqlfs/mod.rs b/crates/ragfs/src/plugins/sqlfs/mod.rsindex 6639908f6..f990850c8 100644--- a/crates/ragfs/src/plugins/sqlfs/mod.rs+++ b/crates/ragfs/src/plugins/sqlfs/mod.rs@@ -38,10 +38,7 @@ impl SQLFileSystem {// Create database backend (schema init and optimizations happen inside)let backend = create_backend(&config.params)?;- tracing::info!(- "SQLFS backend created: {}",- backend.driver_name(),- );+ tracing::info!("SQLFS backend created: {}", backend.driver_name(),);// Create cache from configlet cache_enabled = config@@ -111,22 +108,14 @@ impl SQLFileSystem {}let normalized = Self::normalize_path(path);- normalized- .rsplit('/')- .next()- .unwrap_or("")- .to_string()+ normalized.rsplit('/').next().unwrap_or("").to_string()}}impl Default for SQLFileSystem {fn default() -> Self {// Create with default SQLite in-memory database- let config = PluginConfig {- name: "sqlfs".to_string(),- mount_path: "/sqlfs".to_string(),- params: HashMap::new(),- };+ let config = PluginConfig::single_backend("sqlfs", "/sqlfs", HashMap::new());Self::new(&config).expect("Failed to create default SQLFS")}@@ -519,12 +508,7 @@ impl SQLFSPlugin {"Database host (MySQL/TiDB)",),ConfigParameter::optional("port", "int", "3306", "Database port (MySQL/TiDB)"),- ConfigParameter::optional(- "user",- "string",- "root",- "Database user (MySQL/TiDB)",- ),+ ConfigParameter::optional("user", "string", "root", "Database user (MySQL/TiDB)"),ConfigParameter::optional("password","string",@@ -543,18 +527,8 @@ impl SQLFSPlugin {"true","Enable directory listing cache",),- ConfigParameter::optional(- "cache_max_size",- "int",- "1000",- "Maximum cache entries",- ),- ConfigParameter::optional(- "cache_ttl_seconds",- "int",- "5",- "Cache TTL in seconds",- ),+ ConfigParameter::optional("cache_max_size", "int", "1000", "Maximum cache entries"),+ ConfigParameter::optional("cache_ttl_seconds", "int", "5", "Cache TTL in seconds"),],}}@@ -706,13 +680,20 @@ agfs ls /sqlfs/mydirmod tests {use super::*;+ /// Write one test file in SQLFS.+ async fn write_file(fs: &SQLFileSystem, path: &str, data: &[u8]) {+ fs.write(path, data, 0, WriteFlag::Create).await.unwrap();+ }++ /// Read one test file from SQLFS.+ async fn read_file(fs: &SQLFileSystem, path: &str) -> Vec<u8> {+ fs.read(path, 0, 0).await.unwrap()+ }+#[tokio::test]async fn test_sqlfs_basic() {- let config = PluginConfig {- name: "sqlfs".to_string(),- mount_path: "/sqlfs".to_string(),- params: std::collections::HashMap::new(),- };+ let config =+ PluginConfig::single_backend("sqlfs", "/sqlfs", std::collections::HashMap::new());let plugin = SQLFSPlugin::new();assert!(plugin.validate(&config).await.is_ok());@@ -742,9 +723,7 @@ mod tests {fs.mkdir("/testdir", 0o755).await.unwrap();// Create file in directory- fs.write("/testdir/file.txt", b"data", 0, WriteFlag::Create)- .await- .unwrap();+ write_file(&fs, "/testdir/file.txt", b"data").await;// List directorylet entries = fs.read_dir("/testdir").await.unwrap();@@ -763,29 +742,23 @@ mod tests {async fn test_sqlfs_rename() {let fs = SQLFileSystem::default();- fs.write("/old.txt", b"data", 0, WriteFlag::Create)- .await- .unwrap();+ write_file(&fs, "/old.txt", b"data").await;fs.rename("/old.txt", "/new.txt").await.unwrap();assert!(fs.stat("/old.txt").await.is_err());- let data = fs.read("/new.txt", 0, 0).await.unwrap();- assert_eq!(data, b"data");+ assert_eq!(read_file(&fs, "/new.txt").await, b"data");}#[tokio::test]async fn test_sqlfs_truncate() {let fs = SQLFileSystem::default();- fs.write("/trunc.txt", b"hello world", 0, WriteFlag::Create)- .await- .unwrap();+ write_file(&fs, "/trunc.txt", b"hello world").await;fs.truncate("/trunc.txt", 5).await.unwrap();- let data = fs.read("/trunc.txt", 0, 0).await.unwrap();- assert_eq!(data, b"hello");+ assert_eq!(read_file(&fs, "/trunc.txt").await, b"hello");}#[tokio::test]@@ -813,9 +786,7 @@ mod tests {fs.mkdir("/a", 0o755).await.unwrap();fs.mkdir("/a/b", 0o755).await.unwrap();- fs.write("/a/b/file.txt", b"nested", 0, WriteFlag::Create)- .await- .unwrap();+ write_file(&fs, "/a/b/file.txt", b"nested").await;// List /a should only show /a/blet entries = fs.read_dir("/a").await.unwrap();@@ -824,17 +795,14 @@ mod tests {assert!(entries[0].is_dir);// Read nested file- let data = fs.read("/a/b/file.txt", 0, 0).await.unwrap();- assert_eq!(data, b"nested");+ assert_eq!(read_file(&fs, "/a/b/file.txt").await, b"nested");}#[tokio::test]async fn test_sqlfs_read_with_offset_and_size() {let fs = SQLFileSystem::default();- fs.write("/range.txt", b"hello world", 0, WriteFlag::Create)- .await- .unwrap();+ write_file(&fs, "/range.txt", b"hello world").await;// Read with offsetlet data = fs.read("/range.txt", 6, 0).await.unwrap();@@ -853,9 +821,7 @@ mod tests {async fn test_sqlfs_chmod() {let fs = SQLFileSystem::default();- fs.write("/perm.txt", b"data", 0, WriteFlag::Create)- .await- .unwrap();+ write_file(&fs, "/perm.txt", b"data").await;fs.chmod("/perm.txt", 0o600).await.unwrap();diff --git a/crates/ragfs/src/shape/manifest.rs b/crates/ragfs/src/shape/manifest.rsnew file mode 100644index 000000000..56557520a--- /dev/null+++ b/crates/ragfs/src/shape/manifest.rs@@ -0,0 +1,16 @@+/// Shape guard file stored at the backend root.+pub const SHAPE_MANIFEST_PATH: &str = "/backend_meta.json";++/// Physical storage layout of one backend.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum StorageShape {+ /// Backend stores plaintext user files.+ Plaintext,+ /// Backend stores encrypted user files.+ Encrypted {+ /// Crypto provider identifier embedded in the envelope.+ provider_type: u8,+ /// Envelope format version.+ envelope_version: u8,+ },+}diff --git a/crates/ragfs/src/shape/mod.rs b/crates/ragfs/src/shape/mod.rsnew file mode 100644index 000000000..4cd3fecfa--- /dev/null+++ b/crates/ragfs/src/shape/mod.rs@@ -0,0 +1,10 @@+//! Backend storage shape probing and validation.++/// Shape manifest definitions and constants.+pub mod manifest;+/// Legacy backend shape probing helpers.+pub mod probe;+/// Backend shape validation helpers.+pub mod validate;++pub use manifest::{StorageShape, SHAPE_MANIFEST_PATH};diff --git a/crates/ragfs/src/shape/probe.rs b/crates/ragfs/src/shape/probe.rsnew file mode 100644index 000000000..f176c530c--- /dev/null+++ b/crates/ragfs/src/shape/probe.rs@@ -0,0 +1,129 @@+use std::sync::Arc;++use crate::core::errors::{Error, Result};+use crate::core::filesystem::FileSystem;+use crate::core::WriteFlag;+use crate::crypto;++use super::manifest::{StorageShape, SHAPE_MANIFEST_PATH};++const SYSTEM_ACCOUNT_ID: &[u8] = b"_system";++fn normalize_shape_path(path: &str) -> String {+ let mut normalized = path.trim().to_string();+ if !normalized.starts_with('/') {+ normalized.insert(0, '/');+ }+ if normalized.len() > 1 && normalized.ends_with('/') {+ normalized.pop();+ }+ normalized+}++/// Read the raw guard-file bytes from the backend root.+async fn read_shape_guard_raw(raw_fs: &Arc<dyn FileSystem>) -> Result<Option<Vec<u8>>> {+ match raw_fs.read(SHAPE_MANIFEST_PATH, 0, 0).await {+ Ok(bytes) => Ok(Some(bytes)),+ Err(Error::NotFound(_)) => Ok(None),+ Err(err) => Err(err),+ }+}++/// Infer one file's physical storage shape from its raw bytes.+fn shape_from_raw_bytes(path: &str, bytes: &[u8]) -> Result<StorageShape> {+ if crypto::is_encrypted(bytes) {+ if bytes.len() < 6 {+ return Err(Error::config(format!(+ "encrypted file '{}' is too short to inspect shape",+ path+ )));+ }+ Ok(StorageShape::Encrypted {+ envelope_version: bytes[4],+ provider_type: bytes[5],+ })+ } else {+ Ok(StorageShape::Plaintext)+ }+}++/// Encrypt one empty guard payload so its physical shape matches the backend.+fn encrypt_shape_guard_bytes(root_key: &[u8; 32], provider_type: u8) -> Result<Vec<u8>> {+ let account_key = crypto::hkdf_sha256(root_key, SYSTEM_ACCOUNT_ID);+ let file_key: [u8; 32] = rand::random();+ let key_iv: [u8; 12] = rand::random();+ let data_iv: [u8; 12] = rand::random();+ let ciphertext = crypto::aes_gcm_encrypt(&file_key, &data_iv, &[])?;+ let enc_key = crypto::aes_gcm_encrypt(&account_key, &key_iv, &file_key)?;+ Ok(crypto::build_envelope(+ provider_type,+ &enc_key,+ &key_iv,+ &data_iv,+ &ciphertext,+ ))+}++/// Probe the physical shape of the persisted backend-shape guard file.+pub async fn probe_shape_guard_storage(+ raw_fs: &Arc<dyn FileSystem>,+) -> Result<Option<StorageShape>> {+ let Some(bytes) = read_shape_guard_raw(raw_fs).await? else {+ return Ok(None);+ };+ Ok(Some(shape_from_raw_bytes(SHAPE_MANIFEST_PATH, &bytes)?))+}++/// Persist the backend-shape guard file in the same physical shape as the backend.+pub async fn write_shape_guard(+ raw_fs: &Arc<dyn FileSystem>,+ shape: &StorageShape,+ root_key: Option<&[u8; 32]>,+) -> Result<()> {+ let payload = match shape {+ StorageShape::Plaintext => Vec::new(),+ StorageShape::Encrypted { provider_type, .. } => {+ let root_key = root_key.ok_or_else(|| {+ Error::config("encrypted backend_meta.json requires a root key to be written")+ })?;+ encrypt_shape_guard_bytes(root_key, *provider_type)?+ }+ };+ raw_fs+ .write(SHAPE_MANIFEST_PATH, &payload, 0, WriteFlag::Create)+ .await?;+ Ok(())+}++/// Probe existing backend files to infer the legacy storage shape.+pub async fn detect_legacy_shape(raw_fs: &Arc<dyn FileSystem>) -> Result<Option<StorageShape>> {+ let entries = raw_fs.tree_directory("/", true, None, None).await?;+ let mut detected: Option<StorageShape> = None;++ for entry in entries {+ if entry.info.is_dir {+ continue;+ }++ let normalized = normalize_shape_path(&entry.path);+ if normalized == SHAPE_MANIFEST_PATH {+ continue;+ }++ let header = raw_fs.read(&normalized, 0, 6).await?;+ let current = shape_from_raw_bytes(&normalized, &header)?;++ match &detected {+ None => detected = Some(current),+ Some(existing) if existing == ¤t => {}+ Some(existing) => {+ return Err(Error::config(format!(+ "backend contains mixed storage shapes: previously {:?}, found {:?} at '{}'",+ existing, current, normalized+ )));+ }+ }+ }++ Ok(detected)+}diff --git a/crates/ragfs/src/shape/validate.rs b/crates/ragfs/src/shape/validate.rsnew file mode 100644index 000000000..de157d045--- /dev/null+++ b/crates/ragfs/src/shape/validate.rs@@ -0,0 +1,54 @@+use std::sync::Arc;++use crate::core::errors::{Error, Result};+use crate::core::FileSystem;+use crate::crypto;++use super::manifest::StorageShape;+use super::probe::{detect_legacy_shape, probe_shape_guard_storage, write_shape_guard};++/// Compute the expected backend shape from encryption settings.+pub fn expected_shape(encrypted: bool, provider_type: Option<u8>) -> Result<StorageShape> {+ if encrypted {+ let provider_type = provider_type+ .ok_or_else(|| Error::config("encrypted backend shape requires provider_type"))?;+ Ok(StorageShape::Encrypted {+ provider_type,+ envelope_version: crypto::VERSION,+ })+ } else {+ Ok(StorageShape::Plaintext)+ }+}++/// Validate that one backend matches the expected storage shape.+pub async fn ensure_backend_shape(+ raw_fs: &Arc<dyn FileSystem>,+ backend_type: &str,+ encrypted: bool,+ provider_type: Option<u8>,+ root_key: Option<[u8; 32]>,+) -> Result<()> {+ let expected = expected_shape(encrypted, provider_type)?;+ if let Some(observed) = probe_shape_guard_storage(raw_fs).await? {+ if observed != expected {+ return Err(Error::config(format!(+ "backend storage shape mismatch for '{}': expected {:?}, found {:?}",+ backend_type, expected, observed+ )));+ }+ return Ok(());+ }++ let detected = detect_legacy_shape(raw_fs).await?;+ match detected {+ None => write_shape_guard(raw_fs, &expected, root_key.as_ref()).await,+ Some(found) if found == expected => {+ write_shape_guard(raw_fs, &found, root_key.as_ref()).await+ }+ Some(found) => Err(Error::config(format!(+ "backend storage shape mismatch for '{}': expected {:?}, found {:?}",+ backend_type, expected, found+ ))),+ }+}diff --git a/docs/en/api/03-filesystem.md b/docs/en/api/03-filesystem.mdindex 4ec8ac649..e7b46fbce 100644--- a/docs/en/api/03-filesystem.md+++ b/docs/en/api/03-filesystem.md@@ -15,7 +15,7 @@ Phase 1 intentionally keeps the scope narrow:- Resources only. Memories, skills, sessions, and other namespaces are not exposed.- Text-first writes. `PUT` currently accepts UTF-8 text content only.- WebDAV subset only. `OPTIONS`, `PROPFIND`, `GET`, `HEAD`, `PUT`, `DELETE`, `MKCOL`, and `MOVE` are supported.-- Semantic sidecars stay internal. Derived files such as `.abstract.md`, `.overview.md`, `.relations.json`, and `.path.ovlock` are hidden from listings and cannot be accessed directly through WebDAV.+- Semantic sidecars and internal system files stay internal. Derived or internal files such as `.abstract.md`, `.overview.md`, `.relations.json`, `.path.ovlock`, `.redirect.json`, and `.sync_log.json` are hidden from listings and cannot be accessed directly through WebDAV.Behavior notes:@@ -23,6 +23,7 @@ Behavior notes:- Replacing an existing file through WebDAV refreshes related semantics and vectors, same as `write()`.- `PUT` does not create parent collections automatically. Create missing directories with `MKCOL` first.- User-created dot-directories and dot-files remain visible unless they match one of the reserved internal filenames above.+- When multi-write storage is enabled, files redirected to a backup are still exposed through the filesystem APIs as normal files; internal redirect and sync metadata never become visible to callers.## API Referencediff --git a/docs/en/concepts/05-storage.md b/docs/en/concepts/05-storage.mdindex 11ad4f5c6..421002cab 100644--- a/docs/en/concepts/05-storage.md+++ b/docs/en/concepts/05-storage.md@@ -80,6 +80,17 @@ relations = viking_fs.relations("viking://resources/docs/auth")AGFS provides POSIX-style file operations with multiple backend support.+### Single-Backend and Multi-Write Modes++By default, AGFS uses a single backend for content storage. Once `storage.agfs.backups` is configured, OpenViking enters multi-write mode:++- Top-level `storage.agfs.backend` is the primary backend and remains the authoritative write target.+- `storage.agfs.backups.items[]` defines backup backends for replicas, migration, or read acceleration.+- The Python SDK, HTTP API, and CLI filesystem interfaces stay unchanged.+- Multi-write uses `.redirect.json` and `.sync_log.json` internally to track redirect mappings and sync progress. These files are not visible to users.++For the conceptual model, see [Multi-Write Storage](./14-multi-write-storage.md). For examples, see the [Multi-Write Storage Guide](../guides/13-multi-write-storage.md).+### Backend Types| Backend | Description | Config |@@ -165,4 +176,5 @@ viking_fs.mv(- [Architecture Overview](./01-architecture.md) - System architecture- [Context Layers](./03-context-layers.md) - L0/L1/L2 model- [Viking URI](./04-viking-uri.md) - URI specification+- [Multi-Write Storage](./14-multi-write-storage.md) - Primary/backup roles, routing, and consistency- [Retrieval Mechanism](./07-retrieval.md) - Retrieval process detailsdiff --git a/docs/en/concepts/14-multi-write-storage.md b/docs/en/concepts/14-multi-write-storage.mdnew file mode 100644index 000000000..231acd0cb--- /dev/null+++ b/docs/en/concepts/14-multi-write-storage.md@@ -0,0 +1,134 @@+# Multi-Write Storage++Multi-write storage lets OpenViking use one primary storage backend together with multiple backup backends under a unified filesystem abstraction. It is suitable for high availability, cross-region replicas, read acceleration, and storage migration.++From the API user's point of view, interfaces such as `read()`, `write()`, `ls()`, and `stat()` do not change. Multi-write logic lives inside RAGFS, so callers do not need to care which underlying backend ultimately stores a file.++## Core Model++Multi-write storage consists of one primary backend and multiple backup backends:++| Role | Configuration | Description |+| --- | --- | --- |+| primary | `storage.agfs.backend` | Authoritative write target and final read fallback |+| backup | `storage.agfs.backups.items[]` | Receives replicated writes and may optionally participate in reads |++If `backups` is not configured, OpenViking continues to use the original single-backend mode.++## Write Path++By default, writes land on the primary backend first and are then replicated to write-enabled backup backends.++```text+Client+ -> OpenViking API+ -> RAGFS MultiWrite+ -> primary+ -> backup1 / backup2 / ...+```++If a backup does not define `operations`, it participates in writes by default. This keeps cold-backup setup simple.++## Sync Modes++Multi-write supports two consistency modes.++| Mode | Config value | Behavior | Suitable for |+| --- | --- | --- | --- |+| Async multi-write | `async` | Return as soon as the primary write succeeds; backups sync in the background | Low-latency writes, eventual consistency |+| Sync multi-write | `sync` | Wait for backup acknowledgements after the primary write succeeds | Stronger write confirmation when extra latency is acceptable |++In async mode, backups may lag behind the primary for a short time. In sync mode, `write_ack_count` and `write_ack_timeout_ms` control how many backup acknowledgements are required and how long the system waits.++Even in sync mode, backups that timed out or did not confirm are still retried in the background.++## Read Path++Reads do not hit every backup by default. Only backups that explicitly declare the `read` operation join the read route.++Read order:++```text+1. Read-enabled backups in ascending priority order+2. Fallback to the primary backend+3. If the file is redirected, access the redirect target+4. Return NotFound if the file is still missing+```++This avoids having cold-backup nodes participate in reads by default and reduces the risk of serving stale data.++## Redirect++Redirect means "certain files are not written to the primary backend and are written to a specified backup instead."++Common cases:++- Large files go to object storage.+- Specific file extensions go to a dedicated backend.+- The primary backend keeps standard content, while special files live elsewhere.++Redirect policies are configured on the primary backend. When a file matches a policy, OpenViking records the mapping in internal metadata. Calls such as `ls()`, `stat()`, and `read()` still expose a normal filesystem view.++## Exclude++Exclude means "a specific backup backend does not receive matching files."++Common cases:++- A memory or cache backend should not keep large files.+- One backup only stores text resources.+- A lower-cost backend excludes temporary or oversized files.++Exclude policies are configured on each backup and affect only whether that backup receives a write.++## Internal Metadata++Multi-write uses two internal metadata files:++| File | Purpose |+| --- | --- |+| `.redirect.json` | Records which backend stores each redirected file |+| `.sync_log.json` | Records per-file sync version and backup acknowledgement progress |++These files are hidden from normal users, do not appear in standard directory listings, and should not be read or written through public APIs.++If the primary backend enables at-rest encryption, these internal metadata files follow the same encryption policy.++## Encryption Relationship++Multi-write does not change OpenViking's transparent encryption model.++Rules:++- The Python layer and public APIs stay unaware of encryption details.+- The primary backend must be encrypted when global encryption is enabled.+- Each backup may decide independently whether to enable encryption.+- Internal metadata must go through the primary backend's encryption path.++That means enabling multi-write does not change how clients call the system; encryption remains a configuration concern per backend.++## Relationship with OVPack++Multi-write only handles new writes after it is enabled. It does not automatically sync historical files that already exist in the primary backend.++Recommended migration flow:++1. Use OVPack or another controlled process for a full data migration.+2. Validate the target backend data.+3. Enable the multi-write configuration.+4. Let future new writes and updates continue to replicate through multi-write.++## Limitations++- In async mode, backups may lag temporarily.+- Historical files that existed before multi-write was enabled need a separate migration or backfill process.+- Redirected files rely on internal metadata to reconstruct the directory view.+- Concurrent writes from multiple processes to the same primary backend still need future distributed metadata locking.+- Hot directories may update internal metadata frequently and can introduce additional write amplification.++## Related Documents++- [Storage Architecture](./05-storage.md)+- [Configuration Guide](../guides/01-configuration.md)+- [Multi-Write Storage Guide](../guides/13-multi-write-storage.md)+- [OVPack Import and Export](../guides/09-ovpack.md)diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex c2f49b125..544e3e6b5 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -986,6 +986,8 @@ Storage configuration for context data, including file storage (RAGFS) and vecto|-----------|------|-------------|---------|| `backend` | str | `"local"`, `"s3"`, or `"memory"` | `"local"` || `timeout` | float | Request timeout in seconds | `10.0` |+| `backups` | object | Multi-write storage configuration. When set, the top-level `backend` acts as the primary backend and `backups.items[]` defines backup backends | `null` |+| `redirects` | array | File redirect policies for multi-write storage. Matching files are written to the specified backup instead of the primary backend | `[]` || `queuefs` | object | QueueFS configuration. Controls the namespace mode, backend, and runtime options for `/queue` | `{ "mode": "shared", "backend": "sqlite", "recover_stale_sec": 0, "busy_timeout_ms": 5000 }` || `queue_db_path` | str (optional) | Legacy compatibility field for QueueFS sqlite DB path. Superseded by `storage.agfs.queuefs.db_path`. Defaults to `{storage.workspace}/_system/queue/queue.db` when not set. Useful when the workspace volume does not support sqlite (e.g. some network filesystems) | `null` || `s3` | object | S3 backend configuration (when backend is 's3') | - |@@ -997,6 +999,82 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro> [!WARNING]> `storage.agfs` no longer supports the AGFS HTTP client mode, and the old HTTP client entry should not be configured anymore. AGFS / RAGFS filesystem access now happens only through the in-process Rust binding (`RAGFSBindingClient`). This does not affect the OpenViking server HTTP API, the `ov` CLI, or `AsyncHTTPClient` / `SyncHTTPClient` when they connect to an OpenViking server.+##### Multi-Write Storage Configuration++`storage.agfs.backups` enables multi-write storage. If it is not configured, OpenViking stays in single-backend mode.++```json+{+ "storage": {+ "workspace": "./data",+ "agfs": {+ "backend": "local",+ "redirects": [+ {+ "type": "FileExtensionPolicy",+ "extensions": ["(pdf|ppt|zip)"],+ "target": ["s3-backup"]+ }+ ],+ "backups": {+ "sync_type": "async",+ "items": [+ {+ "name": "s3-backup",+ "backend": "s3",+ "s3": {+ "bucket": "openviking-backup",+ "region": "cn-beijing",+ "endpoint": "https://tos-s3-cn-beijing.volces.com",+ "access_key": "your-ak",+ "secret_key": "your-sk",+ "prefix": "multi-write"+ }+ }+ ]+ }+ }+ }+}+```++Common `backups` fields:++| Parameter | Type | Description | Default |+|-----------|------|-------------|---------|+| `sync_type` | str | Multi-write sync mode. Supports `"async"` or `"sync"` | `"async"` |+| `write_ack_count` | int | Number of backup acknowledgements required before a `sync` write returns | all backups |+| `write_ack_timeout_ms` | int | Timeout in milliseconds while waiting for backup acknowledgements in `sync` mode | `null` |+| `write_concurrency` | int | Maximum async backup write concurrency | `null` |+| `items` | array | Backup backend list. Each item reuses normal backend configuration and adds fields such as `name`, `operations`, `excludes`, and `encryption` | `[]` |++Common `redirects` fields:++| Parameter | Type | Description | Default |+|-----------|------|-------------|---------|+| `type` | str | Policy type. Supports `"FileExtensionPolicy"` or `"FileOverSizePolicy"` | required |+| `extensions` | array | Extension regex list used by `FileExtensionPolicy`, for example `["(pdf\\|ppt)"]` | `[]` |+| `max_size_mb` | int | File size threshold in MB used by `FileOverSizePolicy` | `null` |+| `target` | array | Backup `name` list that receives matched files | required |++File-size redirect example:++```json+{+ "type": "FileOverSizePolicy",+ "max_size_mb": 100,+ "target": ["s3-backup"]+}+```++Notes:++- `redirects` is configured at top-level `storage.agfs` and defines redirect policies for the primary backend.+- `target` must reference an existing backup `name` from `backups.items[]`.+- Files matched by redirect still appear as normal readable and listable files through the filesystem APIs.++See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more examples.+##### QueueFS Configuration| Parameter | Type | Description | Default |diff --git a/docs/en/guides/08-encryption.md b/docs/en/guides/08-encryption.mdindex 0b10c821c..f9b661929 100644--- a/docs/en/guides/08-encryption.md+++ b/docs/en/guides/08-encryption.md@@ -13,6 +13,19 @@ OpenViking provides transparent at-rest data encryption to ensure data securitySee [Data Encryption](../concepts/10-encryption.md) for conceptual explanations.+## Encryption in Multi-Write Storage++Multi-write storage reuses the same transparent encryption model. Encryption still happens inside RAGFS, so the Python SDK, HTTP API, and CLI do not need to handle encryption or decryption directly.++Rules:++- When global `encryption.enabled=true`, the primary backend must be encrypted.+- Each backup backend may control its own encryption through `encryption.enabled`.+- Multi-write internal metadata such as `.redirect.json` and `.sync_log.json` follows the primary backend's encryption policy.+- OpenViking does not expose and does not need public encryption APIs for operating on these internal files.++See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more multi-write configuration details.+## Quick Start### 1. Initialize Root Key (Local Mode)diff --git a/docs/en/guides/09-ovpack.md b/docs/en/guides/09-ovpack.mdindex 6e88063dc..70796f954 100644--- a/docs/en/guides/09-ovpack.md+++ b/docs/en/guides/09-ovpack.md@@ -28,6 +28,25 @@ scope roots together:Internal or runtime data such as `temp`, `queue`, `upload`, lock files, watchcontrol files, and `.relations.json` are outside the OVPack migration scope.+## Working with Multi-Write Storage++Multi-write storage only replicates writes that happen after it is enabled. It+does not automatically copy historical files that already existed before+`storage.agfs.backups` was turned on.++When migrating an existing environment to multi-write mode, first move the+existing dataset with OVPack, then enable multi-write storage.++Recommended flow:++1. Use `ov backup` or `ov export` to export the current dataset.+2. Restore or import the dataset into the target storage environment.+3. Validate data and index integrity in the target environment.+4. Configure and enable multi-write storage.+5. Resume normal writes and let multi-write handle future incremental copies.++For more details, see the [Multi-Write Storage Guide](./13-multi-write-storage.md).+## Quick Start### Export and Import a Resource Directorydiff --git a/docs/en/guides/13-multi-write-storage.md b/docs/en/guides/13-multi-write-storage.mdnew file mode 100644index 000000000..c3f8c66ef--- /dev/null+++ b/docs/en/guides/13-multi-write-storage.md@@ -0,0 +1,388 @@+# Multi-Write Storage Guide++This guide explains how to configure multi-write storage in OpenViking. Multi-write storage lets one primary backend replicate writes to multiple backup backends for high availability, cross-region replicas, read acceleration, and storage migration.++Multi-write lives inside RAGFS. The Python SDK, HTTP API, and CLI usage remain unchanged.++## Prerequisites++- You already have a working `ov.conf`.+- The primary backend has been verified to read and write correctly.+- If you plan to use S3-compatible storage, prepare the bucket, endpoint, and access credentials first.+- If you need to migrate existing data, migrate that dataset before enabling multi-write.++## Minimal Configuration++The following example uses a local directory as the primary backend and replicates writes to another local directory.++```json+{+ "storage": {+ "workspace": "./data",+ "agfs": {+ "backend": "local",+ "backups": {+ "sync_type": "async",+ "items": [+ {+ "name": "local-backup",+ "backend": "local",+ "local": {+ "local_dir": "./data/backup"+ }+ }+ ]+ }+ }+ }+}+```++Notes:++- The top-level `backend` is the primary backend.+- `backups.items[]` is the backup backend list.+- `name` is the stable identity of a backup; later sync metadata refers to it.+- If `sync_type` is omitted, treat it as async by default.++## Configuring Multiple Backups++You can configure more than one backup. The following example writes to both a local replica and S3-compatible object storage.++```json+{+ "storage": {+ "workspace": "./data",+ "agfs": {+ "backend": "local",+ "backups": {+ "sync_type": "async",+ "items": [+ {+ "name": "local-az2",+ "backend": "local",+ "local": {+ "local_dir": "./data/local-az2"+ }+ },+ {+ "name": "object-store",+ "backend": "s3",+ "s3": {+ "bucket": "openviking-backup",+ "region": "us-east-1",+ "endpoint": "https://s3.example.com",+ "access_key": "your-access-key",+ "secret_key": "your-secret-key",+ "prefix": "openviking"+ }+ }+ ]+ }+ }+ }+}+```++Recommendations:++- Do not use unstable hostnames or temporary IDs for `name`.+- The backup path or bucket should not point to the same physical location as the primary backend.+- Changing a backup `name` affects historical sync metadata recognition, so treat that as a production change.++## Choosing a Sync Mode++### Async Mode++Async mode fits most deployments.++```json+{+ "backups": {+ "sync_type": "async",+ "items": []+ }+}+```++Characteristics:++- Returns immediately after the primary write succeeds.+- Backup writes run in the background.+- Low write latency.+- Backups may lag temporarily.++Suitable for:++- Write throughput first.+- Backups mainly used for disaster recovery.+- Eventual consistency is acceptable.++### Sync Mode++Sync mode waits for backup acknowledgements.++```json+{+ "backups": {+ "sync_type": "sync",+ "write_ack_count": 1,+ "write_ack_timeout_ms": 5000,+ "items": []+ }+}+```++Parameters:++| Parameter | Description |+| --- | --- |+| `write_ack_count` | Minimum number of backup acknowledgements required before the write returns |+| `write_ack_timeout_ms` | Timeout in milliseconds while waiting for backup acknowledgements |++Characteristics:++- Stronger write confirmation.+- Write latency depends on backup responsiveness.+- Unconfirmed backups continue to be retried in the background.+- The client may still see an error after the primary write has already succeeded when the required backup acknowledgements are not met.++Suitable for:++- Narrowing the confirmation window between the primary and backups.+- Backup latency is predictable.+- The caller can accept the extra latency of synchronous writes.++## Configuring Read Acceleration++Backups do not participate in reads by default. To let a backup serve reads, explicitly configure `operations`.++```json+{+ "name": "cache-backend",+ "backend": "memfs",+ "operations": [+ {+ "operation": "read",+ "priority": 10+ }+ ]+}+```++Read priority rules:++- Lower `priority` values are tried first.+- Only backups with `read` configured join the read route.+- The primary backend always remains the final fallback.+- Cold-backup nodes usually should not be read-enabled.++If a backup defines only `read` but not `write`, it does not receive normal multi-write replication. Use that only when you explicitly control how the backend gets its data.++## Redirect Configuration++Redirect sends matching files to a specified backup instead of the primary backend.++Redirect by file extension:++```json+{+ "storage": {+ "agfs": {+ "backend": "local",+ "redirects": [+ {+ "type": "FileExtensionPolicy",+ "extensions": ["(pdf|ppt|zip)"],+ "target": ["object-store"]+ }+ ],+ "backups": {+ "items": [+ {+ "name": "object-store",+ "backend": "s3",+ "s3": {+ "bucket": "openviking-large-files",+ "endpoint": "https://s3.example.com"+ }+ }+ ]+ }+ }+ }+}+```++Redirect by file size:++```json+{+ "type": "FileOverSizePolicy",+ "max_size_mb": 100,+ "target": ["object-store"]+}+```++Notes:++- `target` must reference an existing backup `name`.+- Redirected files still appear as normal readable, listable, and queryable files through the public APIs.+- Redirect mappings are stored in internal metadata on the primary backend.++## Exclude Configuration++Exclude makes one backup skip matching files.++```json+{+ "name": "cache-backend",+ "backend": "memfs",+ "excludes": [+ {+ "type": "FileOverSizePolicy",+ "max_size_mb": 50+ },+ {+ "type": "FileExtensionPolicy",+ "extensions": ["(mp4|zip)"]+ }+ ]+}+```++Common uses:++- Exclude large files from cache backends.+- Exclude file types that do not need to be preserved on a lower-cost backup.+- Keep one backup focused on text or configuration resources only.++If the target backup of a redirect also excludes the same file, the configuration is self-contradictory. Fix the configuration instead of expecting the system to guess another target.++## Encryption Configuration++Multi-write storage reuses OpenViking's transparent at-rest encryption.++Example with global encryption enabled:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local",+ "local": {+ "key_file": "~/.openviking/master.key"+ }+ },+ "storage": {+ "workspace": "./data",+ "agfs": {+ "backend": "local",+ "backups": {+ "items": [+ {+ "name": "plain-cache",+ "backend": "memfs",+ "encryption": {+ "enabled": false+ }+ },+ {+ "name": "encrypted-backup",+ "backend": "local",+ "local": {+ "local_dir": "./data/encrypted-backup"+ },+ "encryption": {+ "enabled": true+ }+ }+ ]+ }+ }+ }+}+```++Rules:++- When global `encryption.enabled=true`, the primary backend must be encrypted.+- Each backup may independently control encryption through `encryption.enabled`.+- The Python SDK, HTTP API, and CLI do not need to handle encryption or decryption.+- Internal metadata such as `.redirect.json` and `.sync_log.json` follows the primary backend's encryption policy.++## Migrating Existing Data++Multi-write only replicates writes that happen after it is enabled. It does not automatically copy historical files.++Recommended migration flow:++1. Stop writes or freeze the write window.+2. Use OVPack or another controlled tool to migrate historical data to the target backup.+3. Validate the target backend's data integrity.+4. Configure and enable `storage.agfs.backups`.+5. Resume writes.+6. Observe sync state and error logs.++If freezing writes is not possible, do one full migration first, then a short write pause for incremental validation, and only then enable multi-write.++## Verifying the Configuration++Before startup, it is recommended to run:++```bash+openviking-server doctor+```++After startup, verify with ordinary file APIs:++```bash+openviking write viking://resources/multiwrite-check.txt \+ --content "multi-write check" \+ --wait++openviking read viking://resources/multiwrite-check.txt+```++If you use a local backup, you can also inspect the backup directory directly. In production, system health checks and sync-status commands are preferable.++## FAQ++### Why is a backup not serving reads?++Backups are write-only by default. To make a backup serve reads, configure:++```json+{+ "operations": [+ {+ "operation": "read",+ "priority": 10+ }+ ]+}+```++### Why do historical files not appear in the backup after enabling multi-write?++Multi-write only handles new writes after it is enabled. Historical data must be migrated separately through OVPack, object-storage copy workflows, or future backfill capabilities.++### Can async mode guarantee that the newest data is immediately readable from a backup?++No. Async mode provides eventual consistency only. If you need stronger read consistency, let reads fall back to the primary backend or avoid routing reads to backups that may lag.++### Will internal metadata files appear in normal user listings?++No. `.redirect.json` and `.sync_log.json` are internal files and are hidden from ordinary directory listings.++### If sync mode returns a failure, does that mean the primary backend definitely did not write the data?++No. The primary write may already have succeeded while the required backup acknowledgements were not met. In that case the client can see a failure even though the data already exists on the primary backend, and lagging backups will continue to be repaired in the background.++## Related Documents++- [Multi-Write Storage](../concepts/14-multi-write-storage.md)+- [Storage Architecture](../concepts/05-storage.md)+- [Configuration Guide](./01-configuration.md)+- [Encryption Guide](./08-encryption.md)+- [OVPack Import and Export](./09-ovpack.md)diff --git a/docs/zh/api/03-filesystem.md b/docs/zh/api/03-filesystem.mdindex 508aa9dcc..39b9e5304 100644--- a/docs/zh/api/03-filesystem.md+++ b/docs/zh/api/03-filesystem.md@@ -15,7 +15,7 @@ Phase 1 有意把范围控制得比较小:- 仅开放 `resources` 命名空间,不暴露 memories、skills、sessions 等其他空间。- 以文本写入为主,当前 `PUT` 只接受 UTF-8 文本内容。- 只实现一小部分 WebDAV 方法:`OPTIONS`、`PROPFIND`、`GET`、`HEAD`、`PUT`、`DELETE`、`MKCOL`、`MOVE`。-- 语义侧边文件保持内部可见。`.abstract.md`、`.overview.md`、`.relations.json`、`.path.ovlock` 这些派生文件不会出现在 WebDAV 列表中,也不能被直接访问。+- 语义侧边文件和系统内部文件保持内部可见。`.abstract.md`、`.overview.md`、`.relations.json`、`.path.ovlock`、`.redirect.json`、`.sync_log.json` 这些派生或内部文件不会出现在 WebDAV 列表中,也不能被直接访问。行为说明:@@ -23,6 +23,7 @@ Phase 1 有意把范围控制得比较小:- 通过 WebDAV 覆盖已有文件时,会像 `write()` 一样刷新相关语义和向量。- `PUT` 不会自动创建父目录。缺失的目录需要先用 `MKCOL` 创建。- 用户自己创建的点目录或点文件仍然可见,只有上面列出的保留内部文件名会被隐藏。+- 启用多写存储时,被 redirect 到 backup 的文件仍会通过文件系统 API 呈现为普通文件;内部 redirect 和同步元数据不会暴露给调用方。## API 参考diff --git a/docs/zh/concepts/05-storage.md b/docs/zh/concepts/05-storage.mdindex e4b13351f..5af13966b 100644--- a/docs/zh/concepts/05-storage.md+++ b/docs/zh/concepts/05-storage.md@@ -78,6 +78,17 @@ relations = viking_fs.relations("viking://resources/docs/auth")AGFS 提供 POSIX 风格的文件操作,支持多种后端。+### 单后端与多写模式++默认情况下,AGFS 使用一个后端作为内容存储。配置 `storage.agfs.backups` 后,OpenViking 会启用多写模式:++- 顶层 `storage.agfs.backend` 是 primary,作为权威写入目标。+- `storage.agfs.backups.items[]` 是 backup,用于副本、迁移或读加速。+- Python SDK、HTTP API 和 CLI 的文件系统接口保持不变。+- 多写内部使用 `.redirect.json` 和 `.sync_log.json` 维护 redirect 映射与同步进度,这些文件对用户不可见。++更多概念说明见 [多写存储](./14-multi-write-storage.md),配置示例见 [多写存储指南](../guides/13-multi-write-storage.md)。+### 后端类型| 后端 | 说明 | 配置 |@@ -163,4 +174,5 @@ viking_fs.mv(- [架构概述](./01-architecture.md) - 系统整体架构- [上下文层级](./03-context-layers.md) - L0/L1/L2 模型- [Viking URI](./04-viking-uri.md) - URI 规范+- [多写存储](./14-multi-write-storage.md) - primary/backup、多写路由与一致性- [检索机制](./07-retrieval.md) - 检索流程详解diff --git a/docs/zh/concepts/14-multi-write-storage.md b/docs/zh/concepts/14-multi-write-storage.mdnew file mode 100644index 000000000..6cd24d574--- /dev/null+++ b/docs/zh/concepts/14-multi-write-storage.md@@ -0,0 +1,134 @@+# 多写存储++多写存储让 OpenViking 在一个统一的文件系统抽象下,同时使用一个主存储和多个备份存储。它适合数据高可用、跨区域副本、读加速、存储迁移等场景。++从 API 使用者视角看,`read()`、`write()`、`ls()`、`stat()` 等接口不变。多写逻辑位于 RAGFS 内部,调用方不需要关心文件最终落在哪个底层后端。++## 核心模型++多写存储由一个 primary 和多个 backup 组成:++| 角色 | 配置位置 | 说明 |+| --- | --- | --- |+| primary | `storage.agfs.backend` | 权威写入目标,也是读取兜底 |+| backup | `storage.agfs.backups.items[]` | 接收复制写入,可选参与读取 |++没有配置 `backups` 时,OpenViking 继续使用原有单后端模式。++## 写入路径++默认情况下,写入先落到 primary,再复制到 write-enabled backup。++```text+Client+ -> OpenViking API+ -> RAGFS MultiWrite+ -> primary+ -> backup1 / backup2 / ...+```++backup 未配置 `operations` 时默认参与写入。这样可以用最少配置得到冷备能力。++## 同步模式++多写支持两种一致性模式。++| 模式 | 配置值 | 行为 | 适用场景 |+| --- | --- | --- | --- |+| 异步多写 | `async` | primary 写成功后立即返回,backup 后台同步 | 低延迟写入、最终一致 |+| 同步多写 | `sync` | primary 写成功后等待 backup 确认 | 更强写入确认、可接受额外延迟 |++异步模式下,backup 可能在短时间内落后于 primary。同步模式下,可以通过 `write_ack_count` 和 `write_ack_timeout_ms` 控制需要等待多少 backup 确认,以及等待多久。++即使使用同步模式,未确认或超时的 backup 仍会由后台重试修复。++## 读取路径++读取不会默认访问所有 backup。只有显式声明 `read` 操作的 backup 才会进入读路由。++读取顺序如下:++```text+1. 按 priority 升序访问 read-enabled backup+2. 回退到 primary+3. 如果文件被 redirect,则访问 redirect target+4. 仍未命中则返回 NotFound+```++这种设计避免冷备节点默认参与读取,降低读到旧数据的风险。++## Redirect++Redirect 表示“某些文件不写入 primary,而是写入指定 backup”。++常见用途:++- 大文件进入对象存储。+- 特定后缀文件进入专门 backend。+- 主存储只保存常规内容,特殊文件由其他 backend 保存。++Redirect 策略配置在 primary 上。命中策略后,OpenViking 会把映射记录到内部元数据中。用户执行 `ls()`、`stat()`、`read()` 时仍能看到正常的文件系统视图。++## Exclude++Exclude 表示“某个 backup 不接收匹配的文件”。++常见用途:++- 内存或缓存 backend 不保存大文件。+- 某个 backup 只保存文本类资源。+- 某个低成本 backend 排除临时或超大文件。++Exclude 策略配置在 backup 上,只影响该 backup 是否接收写入。++## 内部元数据++多写使用两个内部元数据文件:++| 文件 | 作用 |+| --- | --- |+| `.redirect.json` | 记录 redirect 文件对应的目标 backend |+| `.sync_log.json` | 记录每个文件的同步版本和 backup 确认进度 |++这些文件对普通用户不可见,不会出现在常规列表结果中,也不应通过公开 API 直接读写。++如果 primary 开启静态数据加密,这些内部元数据也会跟随 primary 加密策略写入。++## 加密关系++多写不会改变 OpenViking 的透明加密模型。++规则如下:++- Python 层和公共 API 不感知加密实现。+- primary 在全局加密开启时必须加密。+- backup 可以独立决定是否加密。+- 内部元数据必须走 primary 的加密入口。++这意味着启用多写后,调用方式仍然不变;只需要通过配置决定每个 backend 的加密策略。++## 与 OVPack 的关系++多写只负责“启用之后的新写入”。它不会自动同步启用之前已经存在于 primary 中的历史文件。++如果要迁移存量数据,推荐流程是:++1. 使用 OVPack 或其他受控方式完成全量迁移。+2. 校验目标 backend 数据。+3. 启用多写配置。+4. 后续新增和修改的数据由多写持续复制。++## 限制++- 异步模式下 backup 可能短暂落后。+- 启用多写前的历史文件需要单独迁移或回填。+- redirect 文件依赖内部元数据恢复目录视图。+- 多进程同时写同一 primary 时,需要未来的分布式元数据锁能力。+- 热点目录会频繁更新内部元数据,可能带来额外写放大。++## 相关文档++- [存储架构](./05-storage.md)+- [配置指南](../guides/01-configuration.md)+- [多写存储指南](../guides/13-multi-write-storage.md)+- [OVPack 导入导出](../guides/09-ovpack.md)diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex ec970b37c..af6a86ff2 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -957,6 +957,8 @@ AST 提取支持:Python、JavaScript/TypeScript、Rust、Go、Java、C/C++。|------|------|------|--------|| `backend` | str | `"local"`、`"s3"` 或 `"memory"` | `"local"` || `timeout` | float | 请求超时时间(秒) | `10.0` |+| `backups` | object | 多写存储配置。配置后顶层 `backend` 作为 primary,`backups.items[]` 作为 backup | `null` |+| `redirects` | array | 多写存储的文件重定向策略。命中后文件写入指定 backup,而不是 primary | `[]` || `queuefs` | object | QueueFS 配置。控制 `/queue` 的命名空间模式、后端和运行时参数 | `{ "mode": "shared", "backend": "sqlite", "recover_stale_sec": 0, "busy_timeout_ms": 5000 }` || `queue_db_path` | str(可选)| 旧版兼容字段,用于覆盖 QueueFS 的 sqlite 数据库文件路径。已被 `storage.agfs.queuefs.db_path` 取代。未设置时默认为 `{storage.workspace}/_system/queue/queue.db`。适用于 workspace 卷不支持 sqlite 的场景(例如某些网络文件系统) | `null` || `s3` | object | S3 backend configuration (when backend is 's3') | - |@@ -969,6 +971,82 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系> [!WARNING]> `storage.agfs` 已不再支持 AGFS HTTP client 模式,也无需再配置旧的 HTTP client 入口。当前 AGFS / RAGFS 文件系统访问仅通过 Rust binding(`RAGFSBindingClient`)在进程内完成。这不影响 OpenViking server 的 HTTP API、`ov` CLI,或 `AsyncHTTPClient` / `SyncHTTPClient` 访问 OpenViking 服务端的能力。+##### 多写存储配置++`storage.agfs.backups` 用于启用多写存储。未配置时,OpenViking 保持单 backend 模式。++```json+{+ "storage": {+ "workspace": "./data",+ "agfs": {+ "backend": "local",+ "redirects": [+ {+ "type": "FileExtensionPolicy",+ "extensions": ["(pdf|ppt|zip)"],+ "target": ["s3-backup"]+ }+ ],+ "backups": {+ "sync_type": "async",+ "items": [+ {+ "name": "s3-backup",+ "backend": "s3",+ "s3": {+ "bucket": "openviking-backup",+ "region": "cn-beijing",+ "endpoint": "https://tos-s3-cn-beijing.volces.com",+ "access_key": "your-ak",+ "secret_key": "your-sk",+ "prefix": "multi-write"+ }+ }+ ]+ }+ }+ }+}+```++`backups` 常用字段:++| 参数 | 类型 | 说明 | 默认值 |+|------|------|------|--------|+| `sync_type` | str | 多写同步模式,支持 `"async"` 或 `"sync"` | `"async"` |+| `write_ack_count` | int | `sync` 模式下返回前需要的 backup 确认数 | 全部 backup |+| `write_ack_timeout_ms` | int | `sync` 模式下等待 backup 确认的超时时间,单位毫秒 | `null` |+| `write_concurrency` | int | 异步 backup 写入并发上限 | `null` |+| `items` | array | backup backend 列表,每个 item 复用普通 backend 配置并增加 `name`、`operations`、`excludes`、`encryption` 等字段 | `[]` |++`redirects` 常用字段:++| 参数 | 类型 | 说明 | 默认值 |+|------|------|------|--------|+| `type` | str | 策略类型,支持 `"FileExtensionPolicy"` 或 `"FileOverSizePolicy"` | 必填 |+| `extensions` | array | `FileExtensionPolicy` 使用的扩展名正则列表,例如 `["(pdf\|ppt)"]` | `[]` |+| `max_size_mb` | int | `FileOverSizePolicy` 使用的文件大小阈值,单位 MB | `null` |+| `target` | array | 命中策略后写入的 backup `name` 列表 | 必填 |++按文件大小重定向示例:++```json+{+ "type": "FileOverSizePolicy",+ "max_size_mb": 100,+ "target": ["s3-backup"]+}+```++注意:++- `redirects` 配置在顶层 `storage.agfs`,表示 primary 的重定向策略。+- `target` 必须引用 `backups.items[]` 中已经定义的 backup `name`。+- 命中 redirect 的文件仍会通过普通文件系统 API 呈现为可读、可列举的文件。++更多配置示例见 [多写存储指南](./13-multi-write-storage.md)。+##### QueueFS 配置| 参数 | 类型 | 说明 | 默认值 |diff --git a/docs/zh/guides/08-encryption.md b/docs/zh/guides/08-encryption.mdindex 355bc6fa9..15dd2d6ca 100644--- a/docs/zh/guides/08-encryption.md+++ b/docs/zh/guides/08-encryption.md@@ -13,6 +13,19 @@ OpenViking 提供透明的静态数据加密,确保多租户环境下的数据加密功能的概念说明见 [数据加密](../concepts/10-encryption.md)。+## 多写存储中的加密++多写存储复用同一套透明加密机制。加密仍在 RAGFS 内部完成,Python SDK、HTTP API 和 CLI 不需要处理加解密。++规则:++- 全局 `encryption.enabled=true` 时,primary backend 必须加密。+- backup backend 可以通过自己的 `encryption.enabled` 控制是否加密。+- `.redirect.json` 和 `.sync_log.json` 等多写内部元数据跟随 primary 加密策略。+- OpenViking 不提供也不需要公开的加解密 API 来操作这些内部文件。++更多多写配置见 [多写存储指南](./13-multi-write-storage.md)。+## 快速开始### 1. 初始化根密钥(Local 模式)diff --git a/docs/zh/guides/09-ovpack.md b/docs/zh/guides/09-ovpack.mdindex eb0fa58ff..e97eee9b4 100644--- a/docs/zh/guides/09-ovpack.md+++ b/docs/zh/guides/09-ovpack.md@@ -24,6 +24,20 @@ checksum,保证包内容没有偏离 manifest;如果攻击者能同时篡改`temp`、`queue`、`upload`、锁文件、watch control 文件、`.relations.json` 等内部或运行态数据不属于 OVPack 迁移范围。+## 与多写存储配合++多写存储只复制启用之后的新写入,不会自动同步启用之前已经存在的历史文件。将已有环境迁移到多写模式时,建议先用 OVPack 完成存量数据迁移,再开启 `storage.agfs.backups`。++推荐流程:++1. 使用 `ov backup` 或 `ov export` 导出现有内容。+2. 在目标存储环境恢复或导入数据。+3. 校验目标环境内容和索引状态。+4. 配置并启用多写存储。+5. 恢复正常写入,让后续增量由多写机制复制。++更多说明见 [多写存储指南](./13-multi-write-storage.md)。+## 快速开始### 导出和导入资源目录diff --git a/docs/zh/guides/13-multi-write-storage.md b/docs/zh/guides/13-multi-write-storage.mdnew file mode 100644index 000000000..934a543e6--- /dev/null+++ b/docs/zh/guides/13-multi-write-storage.md@@ -0,0 +1,388 @@+# 多写存储指南++本指南介绍如何配置 OpenViking 的多写存储能力。多写存储允许一个 primary 后端同时复制写入多个 backup 后端,用于高可用、跨区域副本、读加速和存储迁移。++多写能力位于 RAGFS 内部。OpenViking 的 Python SDK、HTTP API 和 CLI 使用方式保持不变。++## 前置条件++- 已有可用的 `ov.conf`。+- 已确认 primary backend 可以正常读写。+- 如果要接入 S3 兼容存储,已准备好 bucket、endpoint 和访问凭据。+- 如果要迁移已有数据,先完成存量数据迁移,再启用多写。++## 最小配置++下面示例使用本地目录作为 primary,并把写入复制到另一个本地目录。++```json+{+ "storage": {+ "workspace": "./data",+ "agfs": {+ "backend": "local",+ "backups": {+ "sync_type": "async",+ "items": [+ {+ "name": "local-backup",+ "backend": "local",+ "local": {+ "local_dir": "./data/backup"+ }+ }+ ]+ }+ }+ }+}+```++说明:++- 顶层 `backend` 是 primary。+- `backups.items[]` 是 backup 列表。+- `name` 是 backup 的稳定身份,后续同步元数据会引用它。+- `sync_type` 不配置时默认按异步模式理解。++## 多 Backup 配置++可以配置多个 backup。下面示例同时写入本地副本和 S3 兼容对象存储。++```json+{+ "storage": {+ "workspace": "./data",+ "agfs": {+ "backend": "local",+ "backups": {+ "sync_type": "async",+ "items": [+ {+ "name": "local-az2",+ "backend": "local",+ "local": {+ "local_dir": "./data/local-az2"+ }+ },+ {+ "name": "object-store",+ "backend": "s3",+ "s3": {+ "bucket": "openviking-backup",+ "region": "us-east-1",+ "endpoint": "https://s3.example.com",+ "access_key": "your-access-key",+ "secret_key": "your-secret-key",+ "prefix": "openviking"+ }+ }+ ]+ }+ }+ }+}+```++建议:++- `name` 不要使用会频繁变化的机器名或临时编号。+- backup 的底层路径或 bucket 应避免与 primary 指向同一物理位置。+- 修改 backup `name` 会影响历史同步元数据的识别,生产环境应谨慎变更。++## 同步模式选择++### 异步模式++异步模式适合大多数场景。++```json+{+ "backups": {+ "sync_type": "async",+ "items": []+ }+}+```++特点:++- primary 写入成功后立即返回。+- backup 写入在后台执行。+- 写入延迟低。+- backup 可能短暂落后。++适合:++- 写入吞吐优先。+- backup 主要用于灾备。+- 可以接受最终一致性。++### 同步模式++同步模式会等待 backup 确认。++```json+{+ "backups": {+ "sync_type": "sync",+ "write_ack_count": 1,+ "write_ack_timeout_ms": 5000,+ "items": []+ }+}+```++参数说明:++| 参数 | 说明 |+| --- | --- |+| `write_ack_count` | 写入返回前至少需要多少个 backup 确认 |+| `write_ack_timeout_ms` | 等待 backup 确认的超时时间,单位毫秒 |++特点:++- 写入确认更强。+- 写入延迟受 backup 影响。+- 未确认的 backup 会继续由后台重试修复。+- primary 已写成功但 backup 未达确认数时,客户端可能收到错误;此时 primary 中可能已经存在数据。++适合:++- 希望尽量减少 primary 与 backup 的确认窗口。+- backup 延迟可控。+- 调用方能接受同步写入带来的额外延迟。++## 配置读加速++backup 默认不参与读取。要让 backup 服务读取,需要显式配置 `operations`。++```json+{+ "name": "cache-backend",+ "backend": "memfs",+ "operations": [+ {+ "operation": "read",+ "priority": 10+ }+ ]+}+```++读取优先级规则:++- `priority` 越小越优先。+- 只有声明 `read` 的 backup 才参与读取。+- primary 始终作为最终兜底。+- 冷备 backup 不建议配置读能力。++如果一个 backup 只配置了 `read`,没有配置 `write`,它不会接收普通多写复制。只有在你明确知道该 backend 的数据来源时,才应使用这种配置。++## Redirect 配置++Redirect 用于把匹配的文件写入指定 backup,而不是写入 primary。++按扩展名重定向:++```json+{+ "storage": {+ "agfs": {+ "backend": "local",+ "redirects": [+ {+ "type": "FileExtensionPolicy",+ "extensions": ["(pdf|ppt|zip)"],+ "target": ["object-store"]+ }+ ],+ "backups": {+ "items": [+ {+ "name": "object-store",+ "backend": "s3",+ "s3": {+ "bucket": "openviking-large-files",+ "endpoint": "https://s3.example.com"+ }+ }+ ]+ }+ }+ }+}+```++按大小重定向:++```json+{+ "type": "FileOverSizePolicy",+ "max_size_mb": 100,+ "target": ["object-store"]+}+```++注意:++- `target` 必须引用已有 backup 的 `name`。+- redirect 文件仍会通过普通 API 呈现为可读、可列举、可查询状态。+- redirect 映射保存在 primary 的内部元数据中。++## Exclude 配置++Exclude 用于让某个 backup 跳过匹配文件。++```json+{+ "name": "cache-backend",+ "backend": "memfs",+ "excludes": [+ {+ "type": "FileOverSizePolicy",+ "max_size_mb": 50+ },+ {+ "type": "FileExtensionPolicy",+ "extensions": ["(mp4|zip)"]+ }+ ]+}+```++常见用法:++- 缓存 backend 排除大文件。+- 低成本备份排除无需保存的文件类型。+- 某个 backup 只保存文本或配置类资源。++如果 redirect 的目标 backup 同时 exclude 了该文件,说明配置互相冲突。请优先修正配置,不要依赖系统自动猜测其他目标。++## 加密配置++多写存储复用 OpenViking 的透明静态加密能力。++全局加密开启示例:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local",+ "local": {+ "key_file": "~/.openviking/master.key"+ }+ },+ "storage": {+ "workspace": "./data",+ "agfs": {+ "backend": "local",+ "backups": {+ "items": [+ {+ "name": "plain-cache",+ "backend": "memfs",+ "encryption": {+ "enabled": false+ }+ },+ {+ "name": "encrypted-backup",+ "backend": "local",+ "local": {+ "local_dir": "./data/encrypted-backup"+ },+ "encryption": {+ "enabled": true+ }+ }+ ]+ }+ }+ }+}+```++规则:++- 全局 `encryption.enabled=true` 时,primary 必须加密。+- backup 可以通过 `encryption.enabled` 单独控制是否加密。+- Python SDK、HTTP API 和 CLI 不需要处理加解密。+- `.redirect.json` 和 `.sync_log.json` 等内部元数据会跟随 primary 加密策略。++## 存量数据迁移++多写只复制启用之后的新写入,不会自动复制历史文件。++推荐迁移流程:++1. 停止或冻结写入窗口。+2. 使用 OVPack 或其他受控工具把存量数据迁移到目标 backup。+3. 校验目标 backend 的数据完整性。+4. 配置并启用 `storage.agfs.backups`。+5. 恢复写入。+6. 观察同步状态和错误日志。++如果无法冻结写入,可以先做一次全量迁移,再短暂停写做增量校验,最后启用多写。++## 验证配置++启动前建议运行:++```bash+openviking-server doctor+```++启动后可以用普通文件 API 验证:++```bash+openviking write viking://resources/multiwrite-check.txt \+ --content "multi-write check" \+ --wait++openviking read viking://resources/multiwrite-check.txt+```++如果使用本地 backup,可以直接检查 backup 目录中是否出现对应文件。生产环境更推荐使用系统健康检查和同步状态命令。++## 常见问题++### 为什么 backup 没有参与读取?++backup 默认只参与写入,不参与读取。需要在 backup 上显式配置:++```json+{+ "operations": [+ {+ "operation": "read",+ "priority": 10+ }+ ]+}+```++### 为什么启用多写后历史文件没有出现在 backup?++多写只处理启用后的新写入。历史文件需要先通过 OVPack、对象存储复制或后续 backfill 能力迁移。++### 异步模式下能否保证立即读到 backup 的最新数据?++不能。异步模式只保证最终一致。需要强读一致时,应让读取回退到 primary,或避免让可能滞后的 backup 参与读路由。++### 内部元数据文件会出现在用户列表里吗?++不会。`.redirect.json` 和 `.sync_log.json` 是内部文件,会被普通目录列表隐藏。++### sync 模式返回失败是否表示 primary 一定没写入?++不是。primary 写成功但 backup 未达到确认数时,客户端可能收到失败。此时 primary 数据可能已经存在,落后的 backup 会由后台重试修复。++## 相关文档++- [多写存储](../concepts/14-multi-write-storage.md)+- [存储架构](../concepts/05-storage.md)+- [配置指南](./01-configuration.md)+- [加密指南](./08-encryption.md)+- [OVPack 导入导出](./09-ovpack.md)diff --git a/openviking/pyagfs/__init__.py b/openviking/pyagfs/__init__.pyindex 1c2145ae3..7e0a07c37 100644--- a/openviking/pyagfs/__init__.py+++ b/openviking/pyagfs/__init__.py@@ -97,9 +97,18 @@ def _find_ragfs_so():def _load_rust_binding():"""Attempt to load the Rust (PyO3) binding client.- Searches openviking/lib/ for the pre-built native extension first,- then falls back to a pip-installed ``ragfs_python`` package.+ Prefers the pip-installed ``ragfs_python`` package (e.g. from maturin develop),+ then falls back to the vendored native extension in openviking/lib/."""+ # Prefer pip-installed version (handles @rpath correctly)+ try:+ from ragfs_python import RAGFSBindingClient as _Rust++ return _Rust, None+ except ImportError:+ pass++ # Fallback: vendored .so in openviking/lib/try:so_path = _find_ragfs_so()if so_path:@@ -107,13 +116,10 @@ def _load_rust_binding():mod = importlib.util.module_from_spec(spec)spec.loader.exec_module(mod)return mod.RAGFSBindingClient, None-- # Fallback: maybe ragfs_python was pip-installed (dev environment)- from ragfs_python import RAGFSBindingClient as _Rust-- return _Rust, Noneexcept Exception:- raise ImportError("Rust binding not available")+ pass++ raise ImportError("Rust binding not available")def get_binding_client():diff --git a/openviking/pyagfs/async_client.py b/openviking/pyagfs/async_client.pyindex e23a67700..09e57d43f 100644--- a/openviking/pyagfs/async_client.py+++ b/openviking/pyagfs/async_client.py@@ -45,13 +45,12 @@ def encryption_account_id_from_agfs_path(path: str) -> str:def ensure_same_encryption_account(src_path: str, dst_path: str) -> None:- """Reject raw moves/copies that would preserve ciphertext under a different account key."""+ """Reject AGFS moves/copies across encryption-account domains."""src_account = encryption_account_id_from_agfs_path(src_path)dst_account = encryption_account_id_from_agfs_path(dst_path)if src_account != dst_account:raise ValueError(- "cross-account AGFS move/copy is not supported by raw path operations: "- f"{src_account!r} -> {dst_account!r}"+ f"cross-account AGFS move/copy is not supported: {src_account!r} -> {dst_account!r}")@@ -126,19 +125,6 @@ class AsyncAGFSClient:kwargs["stream"] = streamreturn await self.run("cat", path, **kwargs, ctx=_fs_ctx_or_default(path, fs_ctx))- async def read_raw(self, path: str, offset: int = 0, size: int = -1) -> Any:- """Read raw bytes, bypassing the encryption layer (cp/persist whole-blob copy)."""- kwargs: Dict[str, Any] = {}- if offset != 0:- kwargs["offset"] = offset- if size != -1:- kwargs["size"] = size- return await self.run("read_raw", path, **kwargs)-- async def write_raw(self, path: str, data: bytes) -> str:- """Write raw bytes, bypassing the encryption layer (cp/persist whole-blob copy)."""- return await self.run("write_raw", path, data)-async def write(self,path: str,@@ -193,10 +179,25 @@ class AsyncAGFSClient:ensure_same_encryption_account(old_path, new_path)return await self.run("mv", old_path, new_path, ctx=_fs_ctx_or_default(old_path, fs_ctx))- async def cp(self, src_path: str, dst_path: str, recursive: bool = False) -> Any:+ async def cp(+ self,+ src_path: str,+ dst_path: str,+ recursive: bool = False,+ *,+ fs_ctx: Dict[str, str] | None = None,+ ) -> Any:+ """Copy a path within AGFS while preserving the caller's FsContext."""from .helpers import cp- return await asyncio.to_thread(cp, self._client, src_path, dst_path, recursive=recursive)+ return await asyncio.to_thread(+ cp,+ self._client,+ src_path,+ dst_path,+ recursive=recursive,+ fs_ctx=_fs_ctx_or_default(src_path, fs_ctx),+ )async def grep(self, **kwargs: Any) -> Dict[str, Any]:if "ctx" not in kwargs or kwargs["ctx"] is None:@@ -222,3 +223,15 @@ class AsyncAGFSClient:level_limit=level_limit,ctx=_fs_ctx_or_default(path, fs_ctx),)++ async def system_sync_status(+ self, path: str, *, fs_ctx: Dict[str, str] | None = None+ ) -> Dict[str, Any]:+ """Return multi-write sync status for a file or directory path."""+ return await self.run("system_sync_status", path, ctx=_fs_ctx_or_default(path, fs_ctx))++ async def system_sync_retry(+ self, path: str, *, fs_ctx: Dict[str, str] | None = None+ ) -> Dict[str, Any]:+ """Retry pending multi-write sync work for a file or directory path."""+ return await self.run("system_sync_retry", path, ctx=_fs_ctx_or_default(path, fs_ctx))diff --git a/openviking/pyagfs/helpers.py b/openviking/pyagfs/helpers.pyindex 0683a6559..5483a05f0 100644--- a/openviking/pyagfs/helpers.py+++ b/openviking/pyagfs/helpers.py@@ -8,17 +8,51 @@ This module provides high-level helper functions for common operations:from collections.abc import Iteratorfrom pathlib import Path+from typing import Anyfrom .async_client import ensure_same_encryption_accountfrom .protocols import AGFSByteStream, AGFSSyncClientProtocol+def _call_with_optional_ctx(+ method: Any, *args: Any, ctx: dict[str, str] | None = None, **kwargs: Any+) -> Any:+ """Call a sync AGFS method with ctx when the backend supports it."""+ if ctx is None:+ return method(*args, **kwargs)+ try:+ return method(*args, ctx=ctx, **kwargs)+ except TypeError as exc:+ if "unexpected keyword argument 'ctx'" not in str(exc):+ raise+ return method(*args, **kwargs)+++def _try_copy_within_mount_fast_path(+ client: AGFSSyncClientProtocol,+ src: str,+ dst: str,+ *,+ fs_ctx: dict[str, str] | None = None,+) -> bool:+ """Attempt a same-mount verbatim copy and return whether the fast-path was used."""+ copy_within_mount = getattr(client, "copy_within_mount", None)+ if not callable(copy_within_mount):+ return False++ result = _call_with_optional_ctx(copy_within_mount, src, dst, ctx=fs_ctx)+ if isinstance(result, dict):+ return bool(result.get("performed", False))+ return bool(result)++def cp(client: AGFSSyncClientProtocol,src: str,dst: str,recursive: bool = False,stream: bool = False,+ fs_ctx: dict[str, str] | None = None,) -> None:"""Copy a file or directory within AGFS.@@ -39,15 +73,15 @@ def cp(ensure_same_encryption_account(src, dst)# Check if source exists and get its type- src_info = client.stat(src)+ src_info = _call_with_optional_ctx(client.stat, src, ctx=fs_ctx)is_dir = src_info.get("isDir", False)if is_dir:if not recursive:raise ValueError(f"Cannot copy directory '{src}' without recursive=True")- _copy_directory(client, src, dst, stream)+ _copy_directory(client, src, dst, stream, fs_ctx=fs_ctx)else:- _copy_file(client, src, dst, stream)+ _copy_file(client, src, dst, stream, fs_ctx=fs_ctx)def upload(@@ -125,43 +159,58 @@ def download(# Internal helper functions-def _copy_file(client: AGFSSyncClientProtocol, src: str, dst: str, stream: bool) -> None:+def _copy_file(+ client: AGFSSyncClientProtocol,+ src: str,+ dst: str,+ stream: bool,+ *,+ fs_ctx: dict[str, str] | None = None,+) -> None:"""Copy a single file within AGFS.- Uses the raw byte channel (``read_raw``/``write_raw``) when available so a whole blob is moved- verbatim, bypassing the encryption layer (ciphertext is copied as-is, never decrypted/re-encrypted).- Falls back to ``cat``/``write`` for clients that do not expose the raw channel.+ Copy is always expressed in logical AGFS operations so encryption remains+ transparent to the business layer."""# Ensure parent directory exists- _ensure_remote_parent_dir(client, dst)+ _ensure_remote_parent_dir(client, dst, fs_ctx=fs_ctx)- read_raw = getattr(client, "read_raw", None)- write_raw = getattr(client, "write_raw", None)- if callable(read_raw) and callable(write_raw):- write_raw(dst, read_raw(src))+ if _try_copy_within_mount_fast_path(client, src, dst, fs_ctx=fs_ctx):returnif stream:# The binding client returns bytes or an iterator of bytes, not an# HTTP response object with iter_content().- client.write(dst, _iter_file_bytes(client.cat(src, stream=True)))+ _call_with_optional_ctx(+ client.write,+ dst,+ _iter_file_bytes(_call_with_optional_ctx(client.cat, src, stream=True, ctx=fs_ctx)),+ ctx=fs_ctx,+ )else:# Read entire file and write- data = client.cat(src)- client.write(dst, data)+ data = _call_with_optional_ctx(client.cat, src, ctx=fs_ctx)+ _call_with_optional_ctx(client.write, dst, data, ctx=fs_ctx)-def _copy_directory(client: AGFSSyncClientProtocol, src: str, dst: str, stream: bool) -> None:+def _copy_directory(+ client: AGFSSyncClientProtocol,+ src: str,+ dst: str,+ stream: bool,+ *,+ fs_ctx: dict[str, str] | None = None,+) -> None:"""Recursively copy a directory within AGFS."""# Create destination directorytry:- client.mkdir(dst)+ _call_with_optional_ctx(client.mkdir, dst, ctx=fs_ctx)except Exception:# Directory might already exist, continuepass# List source directory contents- items = client.ls(src)+ items = _call_with_optional_ctx(client.ls, src, ctx=fs_ctx)for item in items:item_name = item["name"]@@ -170,10 +219,10 @@ def _copy_directory(client: AGFSSyncClientProtocol, src: str, dst: str, stream:if item.get("isDir", False):# Recursively copy subdirectory- _copy_directory(client, src_path, dst_path, stream)+ _copy_directory(client, src_path, dst_path, stream, fs_ctx=fs_ctx)else:# Copy file- _copy_file(client, src_path, dst_path, stream)+ _copy_file(client, src_path, dst_path, stream, fs_ctx=fs_ctx)def _upload_file(@@ -268,22 +317,32 @@ def _download_directory(_download_file(client, remote_item_path, local_item_path, stream)-def _ensure_remote_parent_dir(client: AGFSSyncClientProtocol, path: str) -> None:+def _ensure_remote_parent_dir(+ client: AGFSSyncClientProtocol,+ path: str,+ *,+ fs_ctx: dict[str, str] | None = None,+) -> None:"""Ensure the parent directory exists for a remote path."""parent = "/".join(path.rstrip("/").split("/")[:-1])if parent and parent != "/":# Try to create parent directory (and its parents)- _ensure_remote_dir_recursive(client, parent)+ _ensure_remote_dir_recursive(client, parent, fs_ctx=fs_ctx)-def _ensure_remote_dir_recursive(client: AGFSSyncClientProtocol, path: str) -> None:+def _ensure_remote_dir_recursive(+ client: AGFSSyncClientProtocol,+ path: str,+ *,+ fs_ctx: dict[str, str] | None = None,+) -> None:"""Recursively ensure a directory exists in AGFS."""if not path or path == "/":return# Check if directory already existstry:- info = client.stat(path)+ info = _call_with_optional_ctx(client.stat, path, ctx=fs_ctx)if info.get("isDir", False):return # Directory existsexcept Exception:@@ -293,11 +352,11 @@ def _ensure_remote_dir_recursive(client: AGFSSyncClientProtocol, path: str) -> N# Ensure parent exists firstparent = "/".join(path.rstrip("/").split("/")[:-1])if parent and parent != "/":- _ensure_remote_dir_recursive(client, parent)+ _ensure_remote_dir_recursive(client, parent, fs_ctx=fs_ctx)# Create this directorytry:- client.mkdir(path)+ _call_with_optional_ctx(client.mkdir, path, ctx=fs_ctx)except Exception:# Might already exist due to race condition, ignorepassdiff --git a/openviking/pyagfs/protocols.py b/openviking/pyagfs/protocols.pyindex 04c145e26..b6b6c19c8 100644--- a/openviking/pyagfs/protocols.py+++ b/openviking/pyagfs/protocols.py@@ -78,6 +78,9 @@ class AGFSSyncClientProtocol(Protocol):def mv(self, old_path: str, new_path: str) -> Dict[str, Any]:"""Move or rename a path inside AGFS."""+ def copy_within_mount(self, src_path: str, dst_path: str) -> Dict[str, Any]:+ """Attempt a same-mount verbatim copy and report whether it was used."""+def grep(self, **kwargs: Any) -> Dict[str, Any]:"""Run a grep-like search through the AGFS backend."""@@ -89,3 +92,9 @@ class AGFSSyncClientProtocol(Protocol):level_limit: int | None = None,) -> list[Dict[str, Any]]:"""Return a tree view for the given AGFS directory."""++ def system_sync_status(self, path: str) -> Dict[str, Any]:+ """Return multi-write sync status for a file or directory path."""++ def system_sync_retry(self, path: str) -> Dict[str, Any]:+ """Retry pending multi-write sync work for a file or directory path."""diff --git a/openviking/server/api_keys/legacy.py b/openviking/server/api_keys/legacy.pyindex 446d95b1f..9b4694f3d 100644--- a/openviking/server/api_keys/legacy.py+++ b/openviking/server/api_keys/legacy.py@@ -13,7 +13,7 @@ from typing import Dict, Optionalfrom argon2 import PasswordHasherfrom argon2.exceptions import VerifyMismatchError-from openviking.pyagfs import AGFSNotFoundError, AsyncAGFSClient+from openviking.pyagfs import AGFSAlreadyExistsError, AGFSNotFoundError, AsyncAGFSClientfrom openviking.server.api_keys.models import AccountInfo, UserKeyEntryfrom openviking.server.identity import ResolvedIdentity, Rolefrom openviking.storage.viking_fs import VikingFS@@ -70,6 +70,40 @@ class LegacyAPIKeyManager:# Prefix index: key_prefix -> list[UserKeyEntry]self._prefix_index: Dict[str, list[UserKeyEntry]] = {}+ def _discard_account_state(self, account_id: str) -> None:+ """Remove an account and its key index entries from in-memory state."""+ account = self._accounts.pop(account_id, None)+ if account is None:+ return++ for user_id, user_info in account.users.items():+ key_or_hash = user_info.get("key", "")+ if not key_or_hash:+ continue++ key_prefix = user_info.get("key_prefix", "")+ if not key_prefix:+ key_prefix = self._get_key_prefix(key_or_hash)++ if key_prefix not in self._prefix_index:+ continue++ self._prefix_index[key_prefix] = [+ entry+ for entry in self._prefix_index[key_prefix]+ if not (entry.account_id == account_id and entry.user_id == user_id)+ ]+ if not self._prefix_index[key_prefix]:+ del self._prefix_index[key_prefix]++ async def _rollback_create_account(self, account_id: str) -> None:+ """Best-effort rollback for partially persisted account creation."""+ self._discard_account_state(account_id)+ try:+ await self._save_accounts_json()+ except Exception:+ logger.exception("Failed to persist rollback for account %s", account_id)+async def load(self) -> None:"""Load accounts and user keys from VikingFS into memory."""accounts_data = await self._read_json(ACCOUNTS_PATH)@@ -229,8 +263,12 @@ class LegacyAPIKeyManager:self._prefix_index[key_prefix] = []self._prefix_index[key_prefix].append(entry)- await self._save_accounts_json()- await self._save_users_json(account_id)+ try:+ await self._save_accounts_json()+ await self._save_users_json(account_id)+ except Exception:+ await self._rollback_create_account(account_id)+ raisereturn keyasync def delete_account(self, account_id: str) -> None:@@ -238,25 +276,7 @@ class LegacyAPIKeyManager:if account_id not in self._accounts:raise NotFoundError(account_id, "account")- account = self._accounts.pop(account_id)- # Remove all keys for this account from prefix index- for user_info in account.users.values():- key_or_hash = user_info.get("key", "")- if key_or_hash:- # Get key_prefix - if not in user_info, compute from key- key_prefix = user_info.get("key_prefix", "")- if not key_prefix:- key_prefix = self._get_key_prefix(key_or_hash)-- if key_prefix in self._prefix_index:- self._prefix_index[key_prefix] = [- entry- for entry in self._prefix_index[key_prefix]- if entry.account_id != account_id- ]- # Remove prefix if index is empty- if not self._prefix_index[key_prefix]:- del self._prefix_index[key_prefix]+ self._discard_account_state(account_id)await self._save_accounts_json()@@ -599,18 +619,10 @@ class LegacyAPIKeyManager:async def _ensure_parent_dirs_async(self, path: str) -> None:"""Recursively create all parent directories for a file path."""- # Handle direct AGFS paths- if path.startswith("/local/"):- # Extract path parts from /local/{account_id}/_system/...- parts = path.lstrip("/").split("/")- if parts:- # Create directory for each parent path- for i in range(1, len(parts)):- parent = "/" + "/".join(parts[:i])- try:- await self._async_agfs.mkdir(parent)- except Exception:- pass+ try:+ await self._async_agfs.ensure_parent_dirs(path)+ except AGFSAlreadyExistsError:+ returnasync def _save_accounts_json(self) -> None:"""Persist the global accounts list."""diff --git a/openviking/server/api_keys/new.py b/openviking/server/api_keys/new.pyindex 10cf5f26a..6f97d1dfa 100644--- a/openviking/server/api_keys/new.py+++ b/openviking/server/api_keys/new.py@@ -227,8 +227,12 @@ class NewAPIKeyManager:self._legacy._prefix_index[key_prefix] = []self._legacy._prefix_index[key_prefix].append(entry)- await self._legacy._save_accounts_json()- await self._legacy._save_users_json(account_id)+ try:+ await self._legacy._save_accounts_json()+ await self._legacy._save_users_json(account_id)+ except Exception:+ await self._legacy._rollback_create_account(account_id)+ raisereturn keyasync def delete_account(self, account_id: str) -> None:diff --git a/openviking/server/app.py b/openviking/server/app.pyindex 58a2499d1..02558009b 100644--- a/openviking/server/app.py+++ b/openviking/server/app.py@@ -75,6 +75,55 @@ def _on_deferred_init_done(task):os._exit(1)+async def _initialize_api_key_manager(+ app: FastAPI,+ service: OpenVikingService,+ config: ServerConfig,+) -> None:+ """Initialize API key state before the app serves authenticated requests."""+ effective_auth_mode = config.get_effective_auth_mode()+ if config.root_api_key and config.root_api_key != "":+ api_key_manager = APIKeyManager(+ root_key=config.root_api_key,+ viking_fs=service.viking_fs,+ api_key_hashing_enabled=config.api_key_hashing_enabled,+ )+ await api_key_manager.load()+ app.state.api_key_manager = api_key_manager+ logger.info(+ "APIKeyManager initialized with api_key_hashing_enabled=%s",+ config.api_key_hashing_enabled,+ )+ if effective_auth_mode == AuthMode.TRUSTED:+ logger.info(+ "Trusted mode enabled: authentication trusts X-OpenViking-Account/User "+ "headers and requires the configured server API key on each request. "+ "Only expose this server behind a trusted network boundary or "+ "identity-injecting gateway."+ )+ return++ app.state.api_key_manager = None+ if effective_auth_mode == AuthMode.TRUSTED:+ logger.warning(+ "Trusted mode enabled: authentication uses X-OpenViking-Account/User "+ "headers without API keys. This is only allowed on localhost. "+ "Only expose this server behind a trusted network boundary or "+ "identity-injecting gateway after configuring server.root_api_key."+ )+++async def _initialize_runtime_state(+ app: FastAPI,+ service: OpenVikingService,+ config: ServerConfig,+) -> None:+ """Initialize service and auth dependencies before traffic is accepted."""+ await service.initialize()+ await _initialize_api_key_manager(app, service, config)+ logger.info("OpenVikingService initialization complete")++def _format_error_location(loc: object) -> str:if not isinstance(loc, (list, tuple)):return "request"@@ -173,51 +222,8 @@ def create_app(_configure_session_tool_outputs(service)async def _deferred_init(service, app, config):- """Run heavy initialization in background after server starts accepting requests."""- await service.initialize()-- # Initialize APIKeyManager after service (needs VikingFS)- effective_auth_mode = config.get_effective_auth_mode()- if config.root_api_key and config.root_api_key != "":- api_key_manager = APIKeyManager(- root_key=config.root_api_key,- viking_fs=service.viking_fs,- api_key_hashing_enabled=config.api_key_hashing_enabled,- )- await api_key_manager.load()- app.state.api_key_manager = api_key_manager- logger.info(- "APIKeyManager initialized with api_key_hashing_enabled=%s",- config.api_key_hashing_enabled,- )- if effective_auth_mode == AuthMode.TRUSTED:- logger.info(- "Trusted mode enabled: authentication trusts X-OpenViking-Account/User "- "headers and requires the configured server API key on each request. "- "Only expose this server behind a trusted network boundary or "- "identity-injecting gateway."- )- elif effective_auth_mode == AuthMode.TRUSTED:- app.state.api_key_manager = None- if config.root_api_key and config.root_api_key != "":- logger.info(- "Trusted mode enabled: authentication trusts X-OpenViking-Account/User "- "headers and requires the configured server API key on each request. "- "Only expose this server behind a trusted network boundary or "- "identity-injecting gateway."- )- else:- logger.warning(- "Trusted mode enabled: authentication uses X-OpenViking-Account/User "- "headers without API keys. This is only allowed on localhost. "- "Only expose this server behind a trusted network boundary or "- "identity-injecting gateway after configuring server.root_api_key."- )- else:- # AuthMode.DEV - logging already handled in validate_server_config- app.state.api_key_manager = None-- logger.info("OpenVikingService initialization complete")+ """Retained for tests that validate deferred-init callback behavior."""+ await _initialize_runtime_state(app, service, config)@asynccontextmanagerasync def lifespan(app: FastAPI):@@ -277,22 +283,11 @@ def create_app(# Start MCP session manager (must be active before /mcp requests)from openviking.server.mcp_endpoint import mcp_lifespan- deferred_task = None-async with mcp_lifespan():- # Start heavy initialization as background task after yieldif service is not None:- deferred_task = asyncio.create_task(_deferred_init(service, app, config))- deferred_task.add_done_callback(_on_deferred_init_done)+ await _initialize_runtime_state(app, service, config)yield- # Wait for deferred initialization to complete before shutdown- if deferred_task and not deferred_task.done():- try:- await deferred_task- except Exception:- logger.exception("Deferred initialization failed")-# Cleanupfrom openviking.metrics.global_api import shutdown_metrics_asyncfrom openviking.observability.usage_audit import shutdown_usage_audit@@ -329,6 +324,7 @@ def create_app()app.state.config = config+ app.state.api_key_manager = Noneset_server_config(config)# Add CORS middlewarediff --git a/openviking/server/auth.py b/openviking/server/auth.pyindex 30826f56b..9f961013b 100644--- a/openviking/server/auth.py+++ b/openviking/server/auth.py@@ -38,6 +38,8 @@ _API_KEY_ROOT_ALLOWED_PREFIXES = ("/api/v1/admin","/api/v1/observer","/api/v1/console",+ "/api/v1/system/backend",+ "/api/v1/system/sync",)diff --git a/openviking/server/routers/system.py b/openviking/server/routers/system.pyindex 0b0058ab0..12af14117 100644--- a/openviking/server/routers/system.py+++ b/openviking/server/routers/system.py@@ -9,10 +9,12 @@ from fastapi import APIRouter, Depends, Requestfrom fastapi.responses import JSONResponsefrom pydantic import BaseModel+from openviking.core.path_variables import resolve_path_variablesfrom openviking.core.uri_validation import validate_viking_uri-from openviking.server.auth import get_request_context, resolve_identity+from openviking.pyagfs.exceptions import AGFSInvalidOperationError, AGFSNotSupportedError+from openviking.server.auth import get_request_context, require_role, resolve_identityfrom openviking.server.dependencies import get_service-from openviking.server.identity import AuthMode, RequestContext+from openviking.server.identity import AuthMode, RequestContext, Rolefrom openviking.server.models import Responsefrom openviking.storage.viking_fs import get_viking_fsfrom openviking_cli.utils import get_logger@@ -22,6 +24,36 @@ logger = get_logger(__name__)router = APIRouter()+def _is_ready_check_ok(value) -> bool:+ """Return whether one readiness check value represents a healthy state."""+ if isinstance(value, dict):+ status = value.get("status")+ if status not in ("ok", "not_configured", "not_supported"):+ return False+ nested = value.get("checks")+ if nested is None:+ return True+ return all(_is_ready_check_ok(item) for item in nested.values())+ return value in ("ok", "not_configured", "not_supported")+++async def _probe_agfs_readiness() -> dict[str, object]:+ """Return structured AGFS readiness, including multi-write sync health when available."""+ viking_fs = get_viking_fs()+ checks: dict[str, object] = {}++ await viking_fs.ls("viking://", ctx=None)+ checks["filesystem"] = "ok"++ try:+ await viking_fs.system_sync_status("viking://", ctx=None)+ checks["multiwrite_sync"] = "ok"+ except (AGFSInvalidOperationError, AGFSNotSupportedError):+ checks["multiwrite_sync"] = "not_supported"++ return {"status": "ok", "checks": checks}++async def _embedding_probe(embedder) -> str:"""Quick embedding probe: embed a single token and check for errors."""from openviking.models.embedder.base import embed_compat@@ -101,13 +133,11 @@ async def readiness_check(request: Request):checks = {}- # 1. AGFS: try to list root+ # 1. AGFS: probe filesystem access and multi-write sync healthtry:- viking_fs = get_viking_fs()- await viking_fs.ls("viking://", ctx=None)- checks["agfs"] = "ok"+ checks["agfs"] = await _probe_agfs_readiness()except Exception as e:- checks["agfs"] = f"error: {e}"+ checks["agfs"] = {"status": "error", "checks": {"filesystem": f"error: {e}"}}# 2. VectorDB: health_check()try:@@ -164,7 +194,7 @@ async def readiness_check(request: Request):except Exception as e:checks["ollama"] = f"error: {e}"- all_ok = all(v in ("ok", "not_configured") for v in checks.values())+ all_ok = all(_is_ready_check_ok(v) for v in checks.values())status_code = 200 if all_ok else 503return JSONResponse(status_code=status_code,@@ -204,6 +234,12 @@ class ConsistencyRequest(BaseModel):uri: str+class BackendSyncRequest(BaseModel):+ """Request model for backend sync status and retry operations."""++ uri: str++@router.post("/api/v1/system/wait", tags=["system"])async def wait_processed(request: WaitRequest,@@ -228,3 +264,51 @@ async def check_consistency(ctx=ctx,)return Response(status="ok", result=result)++[email protected]("/api/v1/system/backend/sync-status", tags=["system"])+async def backend_sync_status(+ request: BackendSyncRequest,+ ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),+):+ """Return multi-write backend sync status for a Viking URI subtree."""+ service = get_service()+ uri = validate_viking_uri(resolve_path_variables(request.uri))+ result = await service.fs.system_sync_status(uri, ctx=ctx)+ return Response(status="ok", result=result)++[email protected]("/api/v1/system/backend/sync-retry", tags=["system"])+async def backend_sync_retry(+ request: BackendSyncRequest,+ ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),+):+ """Retry pending multi-write backend sync work for a Viking URI subtree."""+ service = get_service()+ uri = validate_viking_uri(resolve_path_variables(request.uri))+ result = await service.fs.system_sync_retry(uri, ctx=ctx)+ return Response(status="ok", result=result)++[email protected]("/api/v1/system/sync/{sync_path:path}", tags=["system"])+async def admin_sync_status(+ sync_path: str,+ ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),+):+ """Return multi-write backend sync status for one URI subtree through the admin API."""+ service = get_service()+ uri = validate_viking_uri(resolve_path_variables(sync_path))+ result = await service.fs.system_sync_status(uri, ctx=ctx)+ return Response(status="ok", result=result)++[email protected]("/api/v1/system/sync/{sync_path:path}/retry", tags=["system"])+async def admin_sync_retry(+ sync_path: str,+ ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),+):+ """Retry pending multi-write backend sync work for one URI subtree through the admin API."""+ service = get_service()+ uri = validate_viking_uri(resolve_path_variables(sync_path))+ result = await service.fs.system_sync_retry(uri, ctx=ctx)+ return Response(status="ok", result=result)diff --git a/openviking/server/routers/webdav.py b/openviking/server/routers/webdav.pyindex c0944366c..88311e8b4 100644--- a/openviking/server/routers/webdav.py+++ b/openviking/server/routers/webdav.py@@ -18,6 +18,7 @@ from fastapi.responses import Response as FastAPIResponsefrom openviking.server.auth import get_request_contextfrom openviking.server.dependencies import get_servicefrom openviking.server.identity import RequestContext+from openviking.storage.internal_names import WEBDAV_RESERVED_FILENAMESfrom openviking.utils.time_utils import parse_iso_datetimefrom openviking_cli.exceptions import InvalidArgumentError, NotFoundErrorfrom openviking_cli.utils.uri import VikingURI@@ -25,14 +26,6 @@ from openviking_cli.utils.uri import VikingURIrouter = APIRouter(prefix="/webdav/resources", tags=["webdav"])_DAV_NAMESPACE = "DAV:"-_RESERVED_FILENAMES = frozenset(- {- ".abstract.md",- ".overview.md",- ".relations.json",- ".path.ovlock",- }-)_WEBDAV_METHODS = "OPTIONS, PROPFIND, GET, HEAD, PUT, DELETE, MKCOL, MOVE"_TEXT_MEDIA_FALLBACK = "text/plain; charset=utf-8"@@ -72,7 +65,7 @@ def _ensure_exposed_path(resource_path: str) -> None:if not resource_path:returnparts = resource_path.split("/")- if any(part in _RESERVED_FILENAMES for part in parts):+ if any(part in WEBDAV_RESERVED_FILENAMES for part in parts):raise NotFoundError(resource_path, "resource")@@ -206,7 +199,7 @@ def _exposed_child_entries(entries: list[dict[str, Any]]) -> list[dict[str, Any]name = str(entry.get("name", "") or "")if not name or name in {".", ".."}:continue- if name in _RESERVED_FILENAMES:+ if name in WEBDAV_RESERVED_FILENAMES:continueexposed.append(entry)return exposeddiff --git a/openviking/service/core.py b/openviking/service/core.pyindex 5c8f42e2e..80697ea60 100644--- a/openviking/service/core.py+++ b/openviking/service/core.py@@ -14,7 +14,6 @@ from typing import TYPE_CHECKING, Any, Optionalfrom openviking.core.directories import DirectoryInitializerfrom openviking.crypto.config import bootstrap_encryptionfrom openviking.privacy import UserPrivacyConfigService-from openviking.pyagfs.exceptions import AGFSNotFoundErrorfrom openviking.resource.watch_scheduler import WatchSchedulerfrom openviking.server.identity import RequestContext, Rolefrom openviking.service.debug_service import DebugService@@ -163,10 +162,6 @@ class OpenVikingService:# Create RAGFS client using utilityruntime_binding_config = binding_config or RagfsBindingConfig(agfs=config.agfs)self._agfs_client = create_agfs_client(runtime_binding_config)- self._probe_storage_shape(- self._agfs_client,- encrypted_mode=runtime_binding_config.encryption_enabled(),- )# Initialize QueueManager with agfs_clientif self._agfs_client:@@ -240,25 +235,6 @@ class OpenVikingService:)self._data_dir_lock_acquired = True- def _probe_storage_shape(self, agfs_client: Any, encrypted_mode: bool) -> None:- """Fail fast if existing system metadata shape conflicts with current encryption mode."""- try:- raw = agfs_client.read_raw("/local/_system/accounts.json")- except AGFSNotFoundError:- return-- is_encrypted = raw.startswith(b"OVE1")- if encrypted_mode and not is_encrypted:- raise RuntimeError(- "Storage shape mismatch: encryption is enabled but existing "- "remained data is plaintext"- )- if not encrypted_mode and is_encrypted:- raise RuntimeError(- "Storage shape mismatch: encryption is disabled but existing "- "remained data is encrypted"- )-@propertydef _agfs(self) -> Any:"""Internal access to AGFS client for APIKeyManager."""diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.pyindex 64a9d8355..c0cb47446 100644--- a/openviking/service/fs_service.py+++ b/openviking/service/fs_service.py@@ -157,7 +157,9 @@ class FSService:directory_uri = VikingURI(abstract_uri).parent.urireturn directory_uri, abstract_uri- async def rm(self, uri: str, ctx: RequestContext, recursive: bool = False) -> Optional[Dict[str, Any]]:+ async def rm(+ self, uri: str, ctx: RequestContext, recursive: bool = False+ ) -> Optional[Dict[str, Any]]:"""Remove resource."""uri = validate_viking_uri(uri)viking_fs = self._ensure_initialized()@@ -199,6 +201,18 @@ class FSService:uri = validate_viking_uri(uri)return await viking_fs.stat(uri, ctx=ctx)+ async def system_sync_status(self, uri: str, ctx: RequestContext) -> Dict[str, Any]:+ """Return multi-write sync status for one Viking URI subtree."""+ viking_fs = self._ensure_initialized()+ uri = validate_viking_uri(uri)+ return await viking_fs.system_sync_status(uri, ctx=ctx)++ async def system_sync_retry(self, uri: str, ctx: RequestContext) -> Dict[str, Any]:+ """Retry multi-write sync work for one Viking URI subtree."""+ viking_fs = self._ensure_initialized()+ uri = validate_viking_uri(uri)+ return await viking_fs.system_sync_retry(uri, ctx=ctx)+async def read(self, uri: str, ctx: RequestContext, offset: int = 0, limit: int = -1) -> str:"""Read file content."""viking_fs = self._ensure_initialized()diff --git a/openviking/storage/internal_names.py b/openviking/storage/internal_names.pynew file mode 100644index 000000000..db020d28d--- /dev/null+++ b/openviking/storage/internal_names.py@@ -0,0 +1,33 @@+"""Shared internal file-name constants for Python storage code."""++from __future__ import annotations++MULTIWRITE_PATH_LOCK_FILE = ".path.ovlock"+MULTIWRITE_EXACT_LOCK_FILE_PREFIX = ".exact.ovlock."+MULTIWRITE_REDIRECT_FILE = ".redirect.json"+MULTIWRITE_SYNC_LOG_FILE = ".sync_log.json"++MULTIWRITE_INTERNAL_FILE_NAMES = frozenset(+ {+ MULTIWRITE_PATH_LOCK_FILE,+ MULTIWRITE_REDIRECT_FILE,+ MULTIWRITE_SYNC_LOG_FILE,+ }+)++STORAGE_INTERNAL_ENTRY_NAMES = frozenset(+ {+ "_system",+ "tasks",+ *MULTIWRITE_INTERNAL_FILE_NAMES,+ }+)++WEBDAV_RESERVED_FILENAMES = frozenset(+ {+ ".abstract.md",+ ".overview.md",+ ".relations.json",+ *MULTIWRITE_INTERNAL_FILE_NAMES,+ }+)diff --git a/openviking/storage/transaction/path_lock.py b/openviking/storage/transaction/path_lock.pyindex ef45dd20c..12e40ff42 100644--- a/openviking/storage/transaction/path_lock.py+++ b/openviking/storage/transaction/path_lock.py@@ -7,14 +7,18 @@ from typing import Any, Callable, Optional, Tuplefrom openviking.pyagfs import AGFSSyncClientProtocol, AsyncAGFSClientfrom openviking.pyagfs.async_client import fs_ctx_from_agfs_path+from openviking.storage.internal_names import (+ MULTIWRITE_EXACT_LOCK_FILE_PREFIX,+ MULTIWRITE_PATH_LOCK_FILE,+)from openviking.storage.transaction.lock_handle import LockOwnerfrom openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)# Lock file name-LOCK_FILE_NAME = ".path.ovlock"-EXACT_LOCK_FILE_PREFIX = ".exact.ovlock."+LOCK_FILE_NAME = MULTIWRITE_PATH_LOCK_FILE+EXACT_LOCK_FILE_PREFIX = MULTIWRITE_EXACT_LOCK_FILE_PREFIX# Lock type constantsLOCK_TYPE_EXACT = "E"diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex 59f4acb0c..3a46653a5 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -40,6 +40,10 @@ from openviking.resource.watch_storage import is_watch_task_control_urifrom openviking.server.error_mapping import is_not_found_error, map_exceptionfrom openviking.server.identity import RequestContext, Rolefrom openviking.storage.expr import PathScope+from openviking.storage.internal_names import (+ MULTIWRITE_PATH_LOCK_FILE,+ STORAGE_INTERNAL_ENTRY_NAMES,+)from openviking.telemetry import get_current_telemetryfrom openviking.utils.time_utils import format_simplified, get_current_timestamp, parse_iso_datetimefrom openviking_cli.exceptions import (@@ -588,7 +592,7 @@ class VikingFS:# Remove carried lock file from the copy (directory only)if is_dir:- carried_lock = new_path.rstrip("/") + "/.path.ovlock"+ carried_lock = new_path.rstrip("/") + f"/{MULTIWRITE_PATH_LOCK_FILE}"try:await self._async_agfs.rm(carried_lock)except Exception:@@ -611,6 +615,30 @@ class VikingFS:await self._async_agfs.rm(old_path, recursive=is_dir)return {}+ async def system_sync_status(+ self, uri: str, ctx: Optional[RequestContext] = None+ ) -> Dict[str, Any]:+ """Return multi-write sync status for one Viking URI subtree."""+ self._ensure_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ return await self._async_agfs.system_sync_status(+ path,+ fs_ctx={"account_id": real_ctx.account_id},+ )++ async def system_sync_retry(+ self, uri: str, ctx: Optional[RequestContext] = None+ ) -> Dict[str, Any]:+ """Retry multi-write sync for one Viking URI subtree."""+ self._ensure_mutable_access(uri, ctx)+ real_ctx = self._ctx_or_default(ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ return await self._async_agfs.system_sync_retry(+ path,+ fs_ctx={"account_id": real_ctx.account_id},+ )+async def _copy_for_mv(self,old_uri: str,@@ -623,7 +651,12 @@ class VikingFS:) -> None:"""Copy source to destination for mv without deleting source."""if is_temp:- await self._async_agfs.cp(old_path, new_path, recursive=is_dir)+ await self._async_agfs.cp(+ old_path,+ new_path,+ recursive=is_dir,+ fs_ctx={"account_id": self._ctx_or_default(ctx).account_id},+ )returnif is_dir:@@ -1481,12 +1514,12 @@ class VikingFS:"""Check if name would appear in _ls_entries(parent_path).At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.- At other levels, uses _INTERNAL_NAMES blacklist.+ At other levels, uses the shared storage internal-name blacklist."""parts = [p for p in parent_path.strip("/").split("/") if p]if len(parts) == 2 and parts[0] == "local":return name in VikingURI.LISTABLE_SCOPES- return name not in self._INTERNAL_NAMES+ return name not in STORAGE_INTERNAL_ENTRY_NAMESdef _ancestor_is_filtered(self, entry_path: str, base_path: str) -> bool:"""Check if any ancestor directory of entry_path would be filtered by _ls_entries.@@ -1644,7 +1677,6 @@ class VikingFS:safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]return f"/local/{account_id}/{'/'.join(safe_parts)}"- _INTERNAL_NAMES = {"_system", "tasks", ".path.ovlock"}_ROOT_PATH = "/local"async def _ls_entries(@@ -1653,13 +1685,13 @@ class VikingFS:"""List directory entries, filtering out internal directories.At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.- At other levels, uses _INTERNAL_NAMES blacklist.+ At other levels, uses the shared storage internal-name blacklist."""entries = await self._async_agfs.ls(path)parts = [p for p in path.strip("/").split("/") if p]if len(parts) == 2 and parts[0] == "local":return [e for e in entries if e.get("name") in VikingURI.LISTABLE_SCOPES]- return [e for e in entries if e.get("name") not in self._INTERNAL_NAMES]+ return [e for e in entries if e.get("name") not in STORAGE_INTERNAL_ENTRY_NAMES]def _path_to_uri(self, path: str, ctx: Optional[RequestContext] = None) -> str:"""/local/{account}/... -> viking://...@@ -2321,7 +2353,12 @@ class VikingFS:src_path = self._uri_to_path(temp_uri, ctx=ctx)dst_path = self._uri_to_path(target_uri, ctx=ctx)await self._ensure_parent_dirs(dst_path, ctx=ctx)- await self._async_agfs.cp(src_path, dst_path, recursive=True)+ await self._async_agfs.cp(+ src_path,+ dst_path,+ recursive=True,+ fs_ctx={"account_id": self._ctx_or_default(ctx).account_id},+ )async def delete_temp(self, temp_uri: str, ctx: Optional[RequestContext] = None) -> None:"""Delete temp directory and its contents."""diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.pyindex dff1cd169..04b507f74 100644--- a/openviking/utils/agfs_utils.py+++ b/openviking/utils/agfs_utils.py@@ -8,7 +8,7 @@ import multiprocessingimport osfrom dataclasses import dataclassfrom pathlib import Path-from typing import Any, Dict+from typing import Any, Callable, Dictfrom openviking_cli.utils.logger import get_logger@@ -106,7 +106,9 @@ def _build_queuefs_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str,return plugin_config-def _generate_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str, Any]:+def _generate_plugin_config(+ agfs_config: Any, data_path: Path, server_encryption_enabled: bool = False+) -> Dict[str, Any]:"""Dynamically generate RAGFS plugin configuration based on backend type."""config = {"serverinfofs": {@@ -127,44 +129,204 @@ def _generate_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str, Any]s3_config = getattr(agfs_config, "s3", None)vikingfs_path = data_path / "viking"+ # Check for multi-write configuration+ backups_config = getattr(agfs_config, "backups", None)+ redirects_config = getattr(agfs_config, "redirects", None)+ if redirects_config is not None and backups_config is None:+ raise ValueError(+ "redirects requires backups; single-backend mode does not support redirects"+ )++ # Build primary backend plugin config+ primary_plugin_config: Dict[str, Any] = {}+if backend == "local":- config["localfs"] = {- "enabled": True,- "path": "/local",- "config": {- "local_dir": str(vikingfs_path),- },+ primary_plugin_config = {+ "local_dir": str(vikingfs_path),}elif backend == "s3" and s3_config:- s3_plugin_config = {- "bucket": s3_config.bucket,- "region": s3_config.region,- "access_key_id": s3_config.access_key,- "secret_access_key": s3_config.secret_key,- "endpoint": s3_config.endpoint,- "prefix": s3_config.prefix,- "disable_ssl": not s3_config.use_ssl,- "use_path_style": s3_config.use_path_style,- "directory_marker_mode": s3_config.directory_marker_mode.value- if hasattr(s3_config.directory_marker_mode, "value")- else s3_config.directory_marker_mode,- "disable_batch_delete": s3_config.disable_batch_delete,- "normalize_encoding_chars": s3_config.normalize_encoding_chars,- }+ primary_plugin_config = _serialize_s3_plugin_params(s3_config)- config["s3fs"] = {- "enabled": True,- "path": "/local",- "config": s3_plugin_config,- }+ # Build the mount config dict for the primary backend+ mount_config: Dict[str, Any] = dict(primary_plugin_config)++ # Add multi-write fields if backups are configured+ if backups_config is not None:+ # Serialize backups config to dict for FFI JSON passthrough+ mount_config["backups"] = _serialize_backups_config(backups_config, data_path)+ mount_config["server_encryption_enabled"] = server_encryption_enabled+ mount_config["primary_encryption_enabled"] = server_encryption_enabled++ # Serialize redirect policies+ if redirects_config is not None:+ mount_config["primary_redirects"] = [+ _serialize_redirect_policy(p) for p in redirects_config+ ]++ # Determine the plugin type name for the primary backend+ if backend == "local":+ plugin_name = "localfs"+ elif backend == "s3":+ plugin_name = "s3fs"elif backend == "memory":- config["memfs"] = {- "enabled": True,- "path": "/local",- }+ plugin_name = "memfs"+ else:+ plugin_name = backend++ config[plugin_name] = {+ "enabled": True,+ "path": "/local",+ "config": mount_config,+ }+return config+def _map_backend_to_plugin_name(backend: str) -> str:+ """Map user-facing backend name to Rust plugin name."""+ mapping = {+ "local": "localfs",+ "s3": "s3fs",+ "memory": "memfs",+ }+ return mapping.get(backend, backend)+++def _get_config_value(config: Any, key: str, default: Any = None) -> Any:+ """Read one value from a dict-like or object-like config."""+ if isinstance(config, dict):+ return config.get(key, default)+ return getattr(config, key, default)+++def _get_backend_specific_params(item: Any) -> Any:+ """Return backend-specific nested params from one backup item using the backend name key."""+ backend_params = _get_config_value(item, "backend_params")+ if backend_params is not None:+ return backend_params++ backend_type = _get_config_value(item, "backend")+ if not isinstance(backend_type, str):+ return None+ return _get_config_value(item, backend_type)+++def _serialize_s3_plugin_params(s3_config: Any) -> Dict[str, Any]:+ """Serialize user-facing S3 config into Rust s3fs plugin parameters."""+ directory_marker_mode = _get_config_value(s3_config, "directory_marker_mode")+ return {+ "bucket": _get_config_value(s3_config, "bucket"),+ "region": _get_config_value(s3_config, "region"),+ "access_key_id": _get_config_value(s3_config, "access_key"),+ "secret_access_key": _get_config_value(s3_config, "secret_key"),+ "endpoint": _get_config_value(s3_config, "endpoint"),+ "prefix": _get_config_value(s3_config, "prefix", ""),+ "disable_ssl": not _get_config_value(s3_config, "use_ssl", True),+ "use_path_style": _get_config_value(s3_config, "use_path_style", True),+ "directory_marker_mode": directory_marker_mode.value+ if hasattr(directory_marker_mode, "value")+ else directory_marker_mode,+ "disable_batch_delete": _get_config_value(s3_config, "disable_batch_delete", False),+ "normalize_encoding_chars": _get_config_value(+ s3_config, "normalize_encoding_chars", "?#%+@"+ ),+ }+++def _dump_config_object(config: Any) -> Dict[str, Any]:+ """Dump a pydantic or namespace-like config object without None values."""+ if hasattr(config, "model_dump"):+ return config.model_dump(exclude_none=True)+ if isinstance(config, dict):+ return {key: value for key, value in config.items() if value is not None}+ return {+ key: value+ for key, value in vars(config).items()+ if value is not None and not key.startswith("_")+ }+++def _serialize_s3_backup_params(+ _item: Any, backend_config: Any, _data_path: Path+) -> Dict[str, Any]:+ """Serialize one S3 backup item into the Rust s3fs parameter shape."""+ if backend_config is None:+ return {}+ return _serialize_s3_plugin_params(backend_config)+++def _serialize_local_backup_params(+ item: Any, backend_config: Any, data_path: Path+) -> Dict[str, Any]:+ """Serialize one local backup item and fill the default workspace local_dir."""+ local_dir = (+ _get_config_value(backend_config, "local_dir") if backend_config is not None else None+ )+ if local_dir is None:+ local_dir = data_path / "viking" / "_backups" / _get_config_value(item, "name")+ local_dir_path = Path(local_dir).expanduser()+ return {"local_dir": str(local_dir_path)}+++def _serialize_generic_backup_params(+ _item: Any, backend_config: Any, _data_path: Path+) -> Dict[str, Any]:+ """Serialize one backup item with the generic config dumper."""+ if backend_config is None:+ return {}+ return _dump_config_object(backend_config)+++def _backup_param_serializers() -> Dict[str, Callable[[Any, Any, Path], Dict[str, Any]]]:+ """Return the serializer registry keyed by user-facing backup backend type."""+ return {+ "s3": _serialize_s3_backup_params,+ "local": _serialize_local_backup_params,+ }+++def _serialize_backup_params(item: Any, data_path: Path) -> Dict[str, Any]:+ """Serialize backend-specific params for one backup item via the serializer registry."""+ backend_type = _get_config_value(item, "backend")+ backend_config = _get_backend_specific_params(item)+ serializer = _backup_param_serializers().get(backend_type, _serialize_generic_backup_params)+ return serializer(item, backend_config, data_path)+++def _normalize_backup_item(item: Any, data_path: Path) -> Dict[str, Any]:+ """Normalize one raw backup item for Rust while preserving unknown future fields."""+ item_dict = _dump_config_object(item)+ backend_type = _get_config_value(item, "backend")+ item_dict["backend"] = _map_backend_to_plugin_name(backend_type)+ item_dict.pop("backend_params", None)+ if isinstance(backend_type, str):+ item_dict.pop(backend_type, None)++ params = _serialize_backup_params(item, data_path)+ if params:+ item_dict["params"] = params+ else:+ item_dict.pop("params", None)++ return item_dict+++def _serialize_backups_config(backups_config: Any, data_path: Path) -> Dict[str, Any]:+ """Serialize raw multi-write backups config with top-level passthrough and normalized items."""+ result = _dump_config_object(backups_config)+ result.setdefault("sync_type", "async")+ result["items"] = [+ _normalize_backup_item(item, data_path)+ for item in _get_config_value(backups_config, "items", [])+ ]+ return result+++def _serialize_redirect_policy(policy: Any) -> Dict[str, Any]:+ """Serialize one raw redirect/exclude policy object to a dict."""+ return _dump_config_object(policy)++def create_agfs_client(config: RagfsBindingConfig) -> Any:"""Create a RAGFS client based on the provided configuration.@@ -223,7 +385,12 @@ def mount_agfs_backend(agfs: Any, config: RagfsBindingConfig | Any) -> None:vikingfs_path.mkdir(parents=True, exist_ok=True)# 1. Mount standard plugins- config = _generate_plugin_config(agfs_config, data_path)+ server_encryption_enabled = (+ config.encryption_enabled() if isinstance(config, RagfsBindingConfig) else False+ )+ config = _generate_plugin_config(+ agfs_config, data_path, server_encryption_enabled=server_encryption_enabled+ )for plugin_name, plugin_config in config.items():mount_path = plugin_config["path"]@@ -232,6 +399,12 @@ def mount_agfs_backend(agfs: Any, config: RagfsBindingConfig | Any) -> None:local_dir = plugin_config["config"]["local_dir"]os.makedirs(local_dir, exist_ok=True)logger.debug("[RAGFSUtils] Ensured localfs storage directory exists")+ for backup_item in plugin_config.get("config", {}).get("backups", {}).get("items", []):+ if backup_item.get("backend") != "localfs":+ continue+ backup_local_dir = backup_item.get("params", {}).get("local_dir")+ if backup_local_dir:+ os.makedirs(backup_local_dir, exist_ok=True)# Ensure queuefs db_path parent directory exists before mountingif plugin_name == "queuefs" and "db_path" in plugin_config.get("config", {}):db_path = plugin_config["config"]["db_path"]@@ -242,7 +415,12 @@ def mount_agfs_backend(agfs: Any, config: RagfsBindingConfig | Any) -> None:except Exception:passtry:- agfs.mount(plugin_name, mount_path, plugin_config.get("config", {}))+ cfg = plugin_config.get("config", {})+ logger.debug(+ f"[RAGFSUtils] Mounting {plugin_name} at {mount_path} with config keys: {list(cfg.keys()) if isinstance(cfg, dict) else type(cfg)}"+ )+ agfs.mount(plugin_name, mount_path, cfg)logger.debug(f"[RAGFSUtils] Successfully mounted {plugin_name}")- except Exception:- logger.error(f"[RAGFSUtils] Failed to mount {plugin_name}")+ except Exception as e:+ logger.error(f"[RAGFSUtils] Failed to mount {plugin_name}: {e}")+ raisediff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.pyindex 39617addf..d7652dab8 100644--- a/openviking_cli/utils/config/agfs_config.py+++ b/openviking_cli/utils/config/agfs_config.py@@ -1,7 +1,9 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0+from __future__ import annotations+from enum import Enum-from typing import Any, Optional+from typing import Any, List, Optionalfrom pydantic import BaseModel, Field, model_validator@@ -148,6 +150,11 @@ class QueueFSConfig(BaseModel):class AGFSConfig(BaseModel):"""Configuration for RAGFS (Rust-based AGFS)."""+ name: str = Field(+ default="primary",+ description="Logical backend name, globally unique across primary and all backups",+ )+path: Optional[str] = Field(default=None,description="[Deprecated in favor of `storage.workspace`] RAGFS data storage path. This will be ignored if `storage.workspace` is set.",@@ -224,6 +231,14 @@ class AGFSConfig(BaseModel):# RAGFS will act as a gateway to the specified S3 bucket.s3: S3Config = Field(default_factory=lambda: S3Config(), description="S3 backend configuration")+ # Multi-write configuration+ backups: Optional[dict[str, Any]] = Field(+ default=None, description="Multi-write backups configuration. None = single backend mode."+ )+ redirects: Optional[List[dict[str, Any]]] = Field(+ default=None, description="Primary redirect policies."+ )+model_config = {"extra": "forbid"}@model_validator(mode="after")@@ -271,4 +286,9 @@ class AGFSConfig(BaseModel):"db_path/queue_db_path will be ignored.")+ if self.redirects is not None and self.backups is None:+ raise ValueError(+ "redirects requires backups; single-backend mode does not support redirects"+ )+return selfdiff --git a/tests/integration/test_encryption_integration.py b/tests/integration/test_encryption_integration.pyindex bc3a12458..80aa11b2e 100644--- a/tests/integration/test_encryption_integration.py+++ b/tests/integration/test_encryption_integration.py@@ -307,7 +307,31 @@ class TestVikingFSEncryptionWithAccounts:except Exception:return False- def _check_all_files_encrypted(self, svc, ctx, base_uri: str, print_paths: bool = True) -> None:+ def _agfs_data_root(self, test_data_dir: Path) -> Path:+ """Return the mounted localfs root used by integration tests."""+ return test_data_dir / "viking" / "viking"++ def _backend_file_path(self, svc, ctx, test_data_dir: Path, uri: str) -> Path:+ """Map one Viking URI to the underlying backend file path."""+ agfs_path = svc.viking_fs._uri_to_path(uri, ctx=ctx)+ rel_path = agfs_path.removeprefix("/local/").lstrip("/")+ return self._agfs_data_root(test_data_dir) / rel_path++ def _backend_file_bytes(self, svc, ctx, test_data_dir: Path, uri: str) -> bytes:+ """Read the underlying backend bytes for one Viking URI."""+ file_path = self._backend_file_path(svc, ctx, test_data_dir, uri)+ assert file_path.exists(), f"Backend file missing for {uri}: {file_path}"+ return file_path.read_bytes()++ def _assert_uri_encrypted(self, svc, ctx, test_data_dir: Path, uri: str) -> bytes:+ """Assert one Viking URI is stored as ciphertext on disk."""+ raw_content = self._backend_file_bytes(svc, ctx, test_data_dir, uri)+ assert raw_content.startswith(b"OVE1"), f"File not encrypted: {uri}"+ return raw_content++ def _check_all_files_encrypted(+ self, svc, ctx, test_data_dir: Path, base_uri: str, print_paths: bool = True+ ) -> None:"""Recursively check if all files in a directory are encrypted.@@ -317,7 +341,6 @@ class TestVikingFSEncryptionWithAccounts:base_uri: Base URI to checkprint_paths: Whether to print encrypted file paths"""- agfs_client = svc._agfs_clientasync def _check_recursive(uri: str):try:@@ -331,11 +354,7 @@ class TestVikingFSEncryptionWithAccounts:if ".relations.json" in entry.get("name", ""):continuetry:- agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1"), (- f"File not encrypted: {entry_uri}"- )+ self._assert_uri_encrypted(svc, ctx, test_data_dir, entry_uri)if print_paths:print(f"[ENCRYPTED] {entry_uri}")except AssertionError:@@ -460,10 +479,8 @@ class TestVikingFSEncryptionWithAccounts:read_content = await svc.viking_fs.read_file(test_uri, ctx=ctx)assert read_content == test_content, "Read content does not match original content"- # Verify encryption by reading raw file content directly via AGFS- agfs_client = svc._agfs_client- agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)+ # Verify encryption by reading backend file content directly+ raw_content = self._backend_file_bytes(svc, ctx, data["test_data_dir"], test_uri)assert raw_content.startswith(b"OVE1"), (f"File not encrypted, raw content start: {raw_content[:10]}")@@ -548,10 +565,8 @@ This is a test skill for verifying encryption functionality.assert "Test Skill" in skill_content# Verify encryption by reading raw file content directly via AGFS- agfs_client = svc._agfs_clientskill_md_uri = f"{skill_uri}/SKILL.md"- agfs_path = svc.viking_fs._uri_to_path(skill_md_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)+ raw_content = self._backend_file_bytes(svc, ctx, data["test_data_dir"], skill_md_uri)assert raw_content.startswith(b"OVE1"), (f"Skill file not encrypted, raw content start: {raw_content[:10]}")@@ -559,7 +574,9 @@ This is a test skill for verifying encryption functionality."Skill file content is plaintext, not encrypted")- async def _check_all_files_encrypted(self, root_uri, ctx, svc, agfs_client, prefix=" "):+ async def _check_all_files_encrypted(+ self, root_uri, ctx, svc, test_data_dir: Path, prefix=" "+ ):"""Recursively check if all files are encrypted@@ -567,17 +584,16 @@ This is a test skill for verifying encryption functionality.root_uri: Root URIctx: Request contextsvc: OpenVikingService instance- agfs_client: AGFS client+ test_data_dir: Integration test data directoryprefix: Output prefix"""try:# First try to get URI path to see if it's a filetry:- agfs_path = svc.viking_fs._uri_to_path(root_uri, ctx=ctx)- if agfs_client.exists(agfs_path) and not agfs_client.isdir(agfs_path):+ stat_info = await svc.viking_fs.stat(root_uri, ctx=ctx)+ if not stat_info["isDir"]:# This is a file, check directly- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1"), f"File not encrypted: {root_uri}"+ self._assert_uri_encrypted(svc, ctx, test_data_dir, root_uri)print(f"{prefix}✓ [ENCRYPTED] {root_uri}")returnexcept Exception:@@ -589,13 +605,11 @@ This is a test skill for verifying encryption functionality.if entry["isDir"]:# Recursively check subdirectoriesawait self._check_all_files_encrypted(- entry["uri"], ctx, svc, agfs_client, prefix + " "+ entry["uri"], ctx, svc, test_data_dir, prefix + " ")else:# Check file encryption- agfs_path = svc.viking_fs._uri_to_path(entry["uri"], ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1"), f"File not encrypted: {entry['uri']}"+ self._assert_uri_encrypted(svc, ctx, test_data_dir, entry["uri"])print(f"{prefix}✓ [ENCRYPTED] {entry['uri']}")except Exception as e:print(f"{prefix}[WARNING] Error checking {root_uri}: {e}")@@ -610,6 +624,7 @@ This is a test skill for verifying encryption functionality.data = openviking_service_with_encryptionsvc = data["service"]api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]# ========== Prerequisites ==========print("\n" + "=" * 80)@@ -657,10 +672,9 @@ This is a test skill for verifying encryption functionality.ctx = RequestContext(user=test_user, role=Role.USER)root_ctx = RequestContext(user=test_user, role=Role.ROOT)- agfs_client = svc._agfs_clientaccount_root_uri = f"viking://{test_account_id}"- await self._check_all_files_encrypted(account_root_uri, ctx, svc, agfs_client)+ await self._check_all_files_encrypted(account_root_uri, ctx, svc, test_data_dir)# ========== Execute tests ==========print("\n" + "=" * 80)@@ -688,7 +702,7 @@ This is a test skill for verifying encryption functionality.# Check all files in resources directory are encryptedprint("[1.1] Check files in resources directory are encrypted")resources_dir_uri = "viking://resources"- await self._check_all_files_encrypted(resources_dir_uri, ctx, svc, agfs_client)+ await self._check_all_files_encrypted(resources_dir_uri, ctx, svc, test_data_dir)# 1.2 Execute various operations to verify returns unencryptedprint("[1.2] Verify various operations return unencrypted content")@@ -786,7 +800,7 @@ This is a test skill for verifying encryption functionality.# Check all files in the skill directory are encryptedprint("[2.1] Check files in skill directory are encrypted")- await self._check_all_files_encrypted(test_skill_dir_uri, root_ctx, svc, agfs_client)+ await self._check_all_files_encrypted(test_skill_dir_uri, root_ctx, svc, test_data_dir)# 2.2 Verify various operations return unencryptedprint("[2.2] Verify various operations return unencrypted content")@@ -881,7 +895,7 @@ This is a test skill for verifying encryption functionality.# Check all files in user directory are encryptedprint("[3.1] Check files in user directory are encrypted")user_dir_uri = f"viking://{test_account_id}/user/{test_user_id}"- await self._check_all_files_encrypted(user_dir_uri, ctx, svc, agfs_client)+ await self._check_all_files_encrypted(user_dir_uri, ctx, svc, test_data_dir)# 3.2 Verify various operations return unencryptedprint("[3.2] Verify various operations return unencrypted content")@@ -953,7 +967,7 @@ This is a test skill for verifying encryption functionality.# Check session directory files are encryptedprint("[4.1] Check files in session directory are encrypted")session_dir_uri = f"viking://{test_account_id}/session"- await self._check_all_files_encrypted(session_dir_uri, ctx, svc, agfs_client)+ await self._check_all_files_encrypted(session_dir_uri, ctx, svc, test_data_dir)# 4.2 Verify various operations return unencryptedprint("[4.2] Verify various operations return unencrypted content")@@ -1015,7 +1029,7 @@ This is a test skill for verifying encryption functionality.# Check message.jsonl file encryptionprint("[4.3] Check messages file encryption")- await self._check_all_files_encrypted(session_dir_uri, ctx, svc, agfs_client)+ await self._check_all_files_encrypted(session_dir_uri, ctx, svc, test_data_dir)# 4.4 Verify read operation returns unencryptedprint("[4.4] Re-read session to verify message")@@ -1066,8 +1080,9 @@ This is a test skill for verifying encryption functionality.print("[5.2] Check relation.json file encryption")try:relation_file_uri = f"{dir_a_uri}/.relations.json"- agfs_path = svc.viking_fs._uri_to_path(relation_file_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)+ raw_content = self._backend_file_bytes(+ svc, ctx, data["test_data_dir"], relation_file_uri+ )assert raw_content.startswith(b"OVE1"), (f"relation.json not encrypted: {relation_file_uri}")@@ -1138,9 +1153,7 @@ This is a test skill for verifying encryption functionality.assert from_line_3.rstrip("\n") == expected_from_line_3, "Read from offset failed"# Verify file is encrypted on disk- agfs_client = svc._agfs_client- agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)+ raw_content = self._backend_file_bytes(svc, ctx, data["test_data_dir"], test_uri)assert raw_content.startswith(b"OVE1"), "File should be encrypted"print("\n" + "=" * 80)@@ -1186,9 +1199,7 @@ This is a test skill for verifying encryption functionality.assert from_byte_10.rstrip(b"\n") == expected_from_byte_10, "Read from offset failed"# Verify file is encrypted on disk- agfs_client = svc._agfs_client- agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)+ raw_content = self._backend_file_bytes(svc, ctx, data["test_data_dir"], test_uri)assert raw_content.startswith(b"OVE1"), "File should be encrypted"print("\n" + "=" * 80)@@ -1292,8 +1303,6 @@ class TestAddResourceWithSemanticProcessing:test_user = UserIdentifier(test_account_id, test_user_id)ctx = RequestContext(user=test_user, role=Role.USER)- agfs_client = svc._agfs_client-print("[3] Create test resource files (including directory structure)")test_dir = tmp_path / "test_abstract_dir"test_dir.mkdir()@@ -1347,8 +1356,9 @@ This is a file in subdir2.if ".relations.json" in entry_name:continuetry:- agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)+ raw_content = self._backend_file_bytes(+ svc, ctx, data["test_data_dir"], entry_uri+ )assert raw_content.startswith(b"OVE1"), (f"File not encrypted: {entry_uri}")@@ -1407,8 +1417,9 @@ This is a file in subdir2.if entry.get("isDir"):try:abstract_md_uri = f"{entry['uri']}/.abstract.md"- agfs_path = svc.viking_fs._uri_to_path(abstract_md_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)+ raw_content = self._backend_file_bytes(+ svc, ctx, data["test_data_dir"], abstract_md_uri+ )assert raw_content.startswith(b"OVE1"), (f".abstract.md not encrypted: {abstract_md_uri}")diff --git a/tests/integration/test_vault_encryption_integration.py b/tests/integration/test_vault_encryption_integration.pyindex c3476ed5c..c080ce933 100644--- a/tests/integration/test_vault_encryption_integration.py+++ b/tests/integration/test_vault_encryption_integration.py@@ -389,8 +389,24 @@ class TestVikingFSEncryptionWithVault:except Exception:return False+ def _agfs_data_root(self, test_data_dir: Path) -> Path:+ """Return the mounted localfs root used by integration tests."""+ return test_data_dir / "viking" / "viking"++ def _backend_file_path(self, svc, ctx, test_data_dir: Path, uri: str) -> Path:+ """Map one Viking URI to the underlying backend file path."""+ agfs_path = svc.viking_fs._uri_to_path(uri, ctx=ctx)+ rel_path = agfs_path.removeprefix("/local/").lstrip("/")+ return self._agfs_data_root(test_data_dir) / rel_path++ def _assert_uri_encrypted(self, svc, ctx, test_data_dir: Path, uri: str) -> None:+ """Assert one Viking URI is stored as ciphertext on disk."""+ file_path = self._backend_file_path(svc, ctx, test_data_dir, uri)+ assert file_path.exists(), f"Backend file missing for {uri}: {file_path}"+ assert self._is_file_encrypted(file_path), f"File {uri} is not encrypted"+async def _check_all_files_encrypted(- self, svc, ctx, base_uri: str, print_paths: bool = True+ self, svc, ctx, test_data_dir: Path, base_uri: str, print_paths: bool = True) -> None:"""Recursively check if all files in directory are encrypted.@@ -401,7 +417,6 @@ class TestVikingFSEncryptionWithVault:base_uri: Base URI to checkprint_paths: Whether to print encrypted file paths"""- agfs_client = svc._agfs_clientasync def _check_recursive(uri: str):try:@@ -411,10 +426,7 @@ class TestVikingFSEncryptionWithVault:if entry["isDir"]:await _check_recursive(entry_uri)else:- # Use VikingFS's _uri_to_path method to get file path- agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1"), f"File {entry_uri} is not encrypted"+ self._assert_uri_encrypted(svc, ctx, test_data_dir, entry_uri)if print_paths:print(f"✓ File is encrypted: {entry_uri}")except Exception as e:@@ -433,6 +445,7 @@ class TestVikingFSEncryptionWithVault:data = openviking_service_with_vault_encryptionsvc = data["service"]api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]# Create test accountaccount_id = "test-account-resources"@@ -456,10 +469,7 @@ class TestVikingFSEncryptionWithVault:assert read_content == test_content# Verify file is encrypted- agfs_client = svc._agfs_client- agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1")+ self._assert_uri_encrypted(svc, ctx, test_data_dir, test_uri)# Test various operationsresources_dir_uri = f"viking://{account_id}/resources"@@ -499,6 +509,7 @@ class TestVikingFSEncryptionWithVault:data = openviking_service_with_vault_encryptionsvc = data["service"]api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]# Create test accountaccount_id = "test-account-skills"@@ -535,10 +546,7 @@ This is a test skill for verifying Vault encryption.assert "Test Skill" in read_content# Verify file is encrypted- agfs_client = svc._agfs_client- agfs_path = svc.viking_fs._uri_to_path(skill_md_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1")+ self._assert_uri_encrypted(svc, ctx, test_data_dir, skill_md_uri)# Test various operationsagent_dir_uri = f"viking://{account_id}/agent"@@ -562,6 +570,7 @@ This is a test skill for verifying Vault encryption.data = openviking_service_with_vault_encryptionsvc = data["service"]api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]# Create test accountaccount_id = "test-account-memories"@@ -592,10 +601,7 @@ This is a test skill for verifying Vault encryption.assert "User Preferences" in read_content# Verify file is encrypted- agfs_client = svc._agfs_client- agfs_path = svc.viking_fs._uri_to_path(memory_file_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1")+ self._assert_uri_encrypted(svc, ctx, test_data_dir, memory_file_uri)@pytest.mark.asyncioasync def test_session_operations_with_encryption(@@ -605,6 +611,7 @@ This is a test skill for verifying Vault encryption.data = openviking_service_with_vault_encryptionsvc = data["service"]api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]# Create test accountaccount_id = "test-account-sessions"@@ -623,7 +630,9 @@ This is a test skill for verifying Vault encryption.# Check if session directory files are encryptedsession_dir_uri = f"viking://{account_id}/session"- await self._check_all_files_encrypted(svc, ctx, session_dir_uri, print_paths=False)+ await self._check_all_files_encrypted(+ svc, ctx, test_data_dir, session_dir_uri, print_paths=False+ )@pytest.mark.asyncioasync def test_complete_encryption_workflow_with_vault(@@ -642,7 +651,7 @@ This is a test skill for verifying Vault encryption.svc = openviking_service_with_vault_encryption["service"]api_key_manager = openviking_service_with_vault_encryption["api_key_manager"]- openviking_service_with_vault_encryption["test_data_dir"]+ test_data_dir = openviking_service_with_vault_encryption["test_data_dir"]# 1. Precondition: Create random account and userrandom_account_id = f"test-account-{uuid.uuid4()}"@@ -698,7 +707,9 @@ description: Test skill for complete workflow# Check if all files are encryptedaccount_root_uri = f"viking://{random_account_id}"- await self._check_all_files_encrypted(svc, ctx, account_root_uri, print_paths=True)+ await self._check_all_files_encrypted(+ svc, ctx, test_data_dir, account_root_uri, print_paths=True+ )# 4. Postcondition: Delete accountprint("\n=== Cleanup ===")diff --git a/tests/integration/test_volcengine_kms_encryption_integration.py b/tests/integration/test_volcengine_kms_encryption_integration.pyindex 048d069f7..51b2f7654 100644--- a/tests/integration/test_volcengine_kms_encryption_integration.py+++ b/tests/integration/test_volcengine_kms_encryption_integration.py@@ -395,11 +395,26 @@ class TestVikingFSEncryptionWithVolcengineKMS:except Exception:return False+ def _agfs_data_root(self, test_data_dir: Path) -> Path:+ """Return the mounted localfs root used by integration tests."""+ return test_data_dir / "viking" / "viking"++ def _backend_file_path(self, svc, ctx, test_data_dir: Path, uri: str) -> Path:+ """Map one Viking URI to the underlying backend file path."""+ agfs_path = svc.viking_fs._uri_to_path(uri, ctx=ctx)+ rel_path = agfs_path.removeprefix("/local/").lstrip("/")+ return self._agfs_data_root(test_data_dir) / rel_path++ def _assert_uri_encrypted(self, svc, ctx, test_data_dir: Path, uri: str) -> None:+ """Assert one Viking URI is stored as ciphertext on disk."""+ file_path = self._backend_file_path(svc, ctx, test_data_dir, uri)+ assert file_path.exists(), f"Backend file missing for {uri}: {file_path}"+ assert self._is_file_encrypted(file_path), f"File {uri} is not encrypted"+async def _check_all_files_encrypted(- self, svc, ctx, base_uri: str, print_paths: bool = True+ self, svc, ctx, test_data_dir: Path, base_uri: str, print_paths: bool = True) -> None:"""Recursively check if all files in directory are encrypted"""- agfs_client = svc._agfs_clientasync def _check_recursive(uri: str):try:@@ -409,10 +424,7 @@ class TestVikingFSEncryptionWithVolcengineKMS:if entry["isDir"]:await _check_recursive(entry_uri)else:- # Use VikingFS's _uri_to_path method to get file path- agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1"), f"File {entry_uri} is not encrypted"+ self._assert_uri_encrypted(svc, ctx, test_data_dir, entry_uri)if print_paths:print(f"✓ File is encrypted: {entry_uri}")except Exception as e:@@ -480,7 +492,9 @@ class TestVikingFSEncryptionWithVolcengineKMS:assert resources_dir.exists()- await self._check_all_files_encrypted(svc, ctx, f"viking:///{account_id}/resources")+ await self._check_all_files_encrypted(+ svc, ctx, test_data_dir, f"viking:///{account_id}/resources"+ )finally:import os@@ -495,6 +509,7 @@ class TestVikingFSEncryptionWithVolcengineKMS:data = openviking_service_with_volcengine_encryptionsvc = data["service"]api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]# Create test accountaccount_id = "test-account-resources"@@ -518,10 +533,7 @@ class TestVikingFSEncryptionWithVolcengineKMS:assert read_content == test_content# Verify file is encrypted- agfs_client = svc._agfs_client- agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1")+ self._assert_uri_encrypted(svc, ctx, test_data_dir, test_uri)# Test various operationsresources_dir_uri = f"viking://{account_id}/resources"@@ -563,6 +575,7 @@ class TestVikingFSEncryptionWithVolcengineKMS:data = openviking_service_with_volcengine_encryptionsvc = data["service"]api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]# Create test accountaccount_id = "test-account-skills"@@ -599,10 +612,7 @@ This is a test skill for verifying Volcengine KMS encryption.assert "Test Skill" in read_content# Verify file is encrypted- agfs_client = svc._agfs_client- agfs_path = svc.viking_fs._uri_to_path(skill_md_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1")+ self._assert_uri_encrypted(svc, ctx, test_data_dir, skill_md_uri)# Test various operationsagent_dir_uri = f"viking://{account_id}/agent"@@ -626,6 +636,7 @@ This is a test skill for verifying Volcengine KMS encryption.data = openviking_service_with_volcengine_encryptionsvc = data["service"]api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]# Create test accountaccount_id = "test-account-memories"@@ -656,10 +667,7 @@ This is a test skill for verifying Volcengine KMS encryption.assert "User Preferences" in read_content# Verify file is encrypted- agfs_client = svc._agfs_client- agfs_path = svc.viking_fs._uri_to_path(memory_file_uri, ctx=ctx)- raw_content = agfs_client.read_raw(agfs_path)- assert raw_content.startswith(b"OVE1")+ self._assert_uri_encrypted(svc, ctx, test_data_dir, memory_file_uri)@pytest.mark.asyncioasync def test_session_operations_with_encryption(@@ -669,6 +677,7 @@ This is a test skill for verifying Volcengine KMS encryption.data = openviking_service_with_volcengine_encryptionsvc = data["service"]api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]# Create test accountaccount_id = "test-account-sessions"@@ -687,7 +696,9 @@ This is a test skill for verifying Volcengine KMS encryption.# Check if session directory files are encryptedsession_dir_uri = f"viking://{account_id}/session"- await self._check_all_files_encrypted(svc, ctx, session_dir_uri, print_paths=False)+ await self._check_all_files_encrypted(+ svc, ctx, test_data_dir, session_dir_uri, print_paths=False+ )@pytest.mark.asyncioasync def test_complete_encryption_workflow(@@ -697,6 +708,7 @@ This is a test skill for verifying Volcengine KMS encryption.data = openviking_service_with_volcengine_encryptionsvc = data["service"]api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]# Generate random account nameimport secrets@@ -765,7 +777,9 @@ description: Test skill for complete workflow# 5. Verify all files are encryptedprint("\n5. Verifying all files are encrypted...")account_root_uri = f"viking://{test_account_id}"- await self._check_all_files_encrypted(svc, ctx, account_root_uri, print_paths=False)+ await self._check_all_files_encrypted(+ svc, ctx, test_data_dir, account_root_uri, print_paths=False+ )print("✓ All files are encrypted")print("\n✓ Complete Volcengine KMS encryption workflow test passed!")diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.pyindex 5f50f1672..60fdd7b26 100644--- a/tests/misc/test_config_validation.py+++ b/tests/misc/test_config_validation.py@@ -102,142 +102,269 @@ def test_agfs_s3_normalize_encoding_chars_is_forwarded_to_ragfs_plugin_config():print(f" Fail: {e}")-def test_agfs_queuefs_defaults_to_sqlite_backend():- config = AGFSConfig(path="/tmp/ov-test", backend="local")+ ("queuefs", "expected"),+ [+ (+ None,+ {+ "mode": "shared",+ "backend": "sqlite",+ "recover_stale_sec": 0,+ "busy_timeout_ms": 5000,+ },+ ),+ (+ {"mode": "worker", "backend": "memory"},+ {+ "mode": "worker",+ "backend": "memory",+ "recover_stale_sec": 0,+ "busy_timeout_ms": 5000,+ },+ ),+ ],+)+def test_agfs_queuefs_validation_accepts_supported_shapes(queuefs, expected):+ config_kwargs = {"path": "/tmp/ov-test", "backend": "local"}+ if queuefs is not None:+ config_kwargs["queuefs"] = queuefs++ config = AGFSConfig(**config_kwargs)++ assert config.queuefs.mode == expected["mode"]+ assert config.queuefs.backend == expected["backend"]+ assert config.queuefs.recover_stale_sec == expected["recover_stale_sec"]+ assert config.queuefs.busy_timeout_ms == expected["busy_timeout_ms"]+++ ("queuefs", "match"),+ [+ ({"mode": "process"}, "queuefs mode"),+ ({"backend": "bogus"}, "queuefs"),+ ({"busy_timeout_ms": -1}, "busy_timeout_ms"),+ ({"recover_stale_sec": -1}, "recover_stale_sec"),+ ],+)+def test_agfs_queuefs_validation_rejects_invalid_shapes(queuefs, match):+ with pytest.raises(ValueError, match=match):+ AGFSConfig(path="/tmp/ov-test", backend="local", queuefs=queuefs)+++ ("queuefs", "queue_db_path", "expected"),+ [+ (+ {"backend": "memory"},+ None,+ {"backend": "memory", "db_path": None},+ ),+ (+ {"backend": "sqlite", "db_path": "/tmp/new-queue.db"},+ "/tmp/legacy-queue.db",+ {"backend": "sqlite", "db_path": str(Path("/tmp/new-queue.db").resolve())},+ ),+ (+ None,+ "/tmp/legacy-queue.db",+ {"backend": "sqlite", "db_path": str(Path("/tmp/legacy-queue.db").resolve())},+ ),+ (+ None,+ None,+ {"backend": "sqlite", "db_path": "/tmp/ov-test/_system/queue/queue.db"},+ ),+ (+ {"backend": "memory", "db_path": "/tmp/new-queue.db"},+ "/tmp/legacy-queue.db",+ {"backend": "memory", "db_path": None},+ ),+ ],+)+def test_generate_plugin_config_materializes_queuefs_paths(queuefs, queue_db_path, expected):+ config_kwargs = {+ "path": "/tmp/ov-test",+ "backend": "local",+ "queue_db_path": queue_db_path,+ }+ if queuefs is not None:+ config_kwargs["queuefs"] = queuefs++ config = AGFSConfig(**config_kwargs)+ plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))- assert config.queuefs.mode == "shared"- assert config.queuefs.backend == "sqlite"- assert config.queuefs.recover_stale_sec == 0- assert config.queuefs.busy_timeout_ms == 5000+ queuefs_config = plugins["queuefs"]["config"]+ assert queuefs_config["backend"] == expected["backend"]+ if expected["db_path"] is None:+ assert "db_path" not in queuefs_config+ else:+ assert queuefs_config["db_path"] == expected["db_path"]-def test_agfs_queuefs_accepts_memory_backend():+def test_generate_plugin_config_forwards_queuefs_runtime_options():config = AGFSConfig(path="/tmp/ov-test",backend="local",- queuefs={"mode": "worker", "backend": "memory"},+ queuefs={+ "backend": "sqlite3",+ "recover_stale_sec": 17,+ "busy_timeout_ms": 1234,+ },)- assert config.queuefs.mode == "worker"- assert config.queuefs.backend == "memory"---def test_agfs_queuefs_rejects_legacy_process_mode():- with pytest.raises(ValueError, match="queuefs mode"):- AGFSConfig(- path="/tmp/ov-test",- backend="local",- queuefs={"mode": "process"},- )-+ plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))-def test_agfs_queuefs_rejects_invalid_backend():- with pytest.raises(ValueError, match="queuefs"):- AGFSConfig(- path="/tmp/ov-test",- backend="local",- queuefs={"backend": "bogus"},- )+ assert plugins["queuefs"]["config"]["backend"] == "sqlite3"+ assert plugins["queuefs"]["config"]["recover_stale_sec"] == 17+ assert plugins["queuefs"]["config"]["busy_timeout_ms"] == 1234-def test_agfs_queuefs_rejects_negative_timeouts():- with pytest.raises(ValueError, match="busy_timeout_ms"):+def test_agfs_redirects_require_backups():+ """Single-backend mode must reject redirect policies during config validation."""+ with pytest.raises(ValueError, match="redirects requires backups"):AGFSConfig(path="/tmp/ov-test",backend="local",- queuefs={"busy_timeout_ms": -1},+ redirects=[+ {+ "type": "FileExtensionPolicy",+ "extensions": ["(md)"],+ "target": ["s3-backup"],+ }+ ],)- with pytest.raises(ValueError, match="recover_stale_sec"):- AGFSConfig(- path="/tmp/ov-test",- backend="local",- queuefs={"recover_stale_sec": -1},- )+def test_generate_plugin_config_rejects_redirects_without_backups(tmp_path):+ """Runtime plugin config generation must also reject redirect-only configs."""+ config = type(+ "RedirectOnlyConfig",+ (),+ {+ "backend": "local",+ "s3": None,+ "backups": None,+ "redirects": [+ type(+ "RedirectPolicy",+ (),+ {+ "type": "FileExtensionPolicy",+ "extensions": ["(md)"],+ "target": ["s3-backup"],+ },+ )()+ ],+ "queuefs": type(+ "QueueConfig",+ (),+ {+ "mode": "shared",+ "backend": "sqlite",+ "db_path": None,+ "recover_stale_sec": 0,+ "busy_timeout_ms": 5000,+ },+ )(),+ "queue_db_path": None,+ },+ )()-def test_generate_plugin_config_uses_queuefs_memory_backend_without_db_path():- config = AGFSConfig(- path="/tmp/ov-test",- backend="local",- queuefs={"backend": "memory"},- )-- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))-- assert plugins["queuefs"]["config"]["backend"] == "memory"- assert "db_path" not in plugins["queuefs"]["config"]---def test_generate_plugin_config_uses_new_queuefs_db_path_over_legacy_field():- config = AGFSConfig(- path="/tmp/ov-test",- backend="local",- queue_db_path="/tmp/legacy-queue.db",- queuefs={"backend": "sqlite", "db_path": "/tmp/new-queue.db"},- )-- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))-- assert plugins["queuefs"]["config"]["backend"] == "sqlite"- assert plugins["queuefs"]["config"]["db_path"] == str(Path("/tmp/new-queue.db").resolve())---def test_generate_plugin_config_supports_legacy_queue_db_path():- config = AGFSConfig(- path="/tmp/ov-test",- backend="local",- queue_db_path="/tmp/legacy-queue.db",- )-- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))-- assert plugins["queuefs"]["config"]["backend"] == "sqlite"- assert plugins["queuefs"]["config"]["db_path"] == str(Path("/tmp/legacy-queue.db").resolve())+ with pytest.raises(ValueError, match="redirects requires backups"):+ _generate_plugin_config(config, tmp_path)-def test_generate_plugin_config_uses_workspace_default_queue_db_path():+def test_generate_plugin_config_passes_multiwrite_encryption_flag(tmp_path):+ """Multi-write mount config must reflect the real server encryption state."""config = AGFSConfig(- path="/tmp/ov-test",+ path=str(tmp_path),backend="local",+ backups={+ "items": [+ {+ "name": "mem-backup",+ "backend": "memory",+ }+ ]+ },)- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))+ plugins = _generate_plugin_config(config, tmp_path, server_encryption_enabled=True)- assert plugins["queuefs"]["config"]["backend"] == "sqlite"- assert plugins["queuefs"]["config"]["db_path"] == "/tmp/ov-test/_system/queue/queue.db"+ mount_config = plugins["localfs"]["config"]+ assert mount_config["server_encryption_enabled"] is True+ assert mount_config["primary_encryption_enabled"] is True-def test_generate_plugin_config_forwards_queuefs_runtime_options():+def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path):+ """Plugin config generation should normalize backup params while preserving top-level multi-write fields."""+ explicit_backup_dir = tmp_path / "backup-local-no-mkdir"config = AGFSConfig(- path="/tmp/ov-test",+ path=str(tmp_path),backend="local",- queuefs={- "backend": "sqlite3",- "recover_stale_sec": 17,- "busy_timeout_ms": 1234,+ backups={+ "retry_interval_ms": 1234,+ "retry_backoff_base_ms": 55,+ "items": [+ {+ "name": "local-explicit",+ "backend": "local",+ "local": {"local_dir": str(explicit_backup_dir)},+ },+ {+ "name": "local-default",+ "backend": "local",+ },+ {+ "name": "s3-backup",+ "backend": "s3",+ "s3": {+ "bucket": "backup-bucket",+ "region": "cn-beijing",+ "access_key": "test-access-key",+ "secret_key": "test-secret-key",+ "endpoint": "https://tos.example.com",+ "prefix": "backup-prefix",+ "use_ssl": False,+ "use_path_style": False,+ "normalize_encoding_chars": "#?",+ },+ },+ ],},)- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))+ plugins = _generate_plugin_config(config, tmp_path)- assert plugins["queuefs"]["config"]["backend"] == "sqlite3"- assert plugins["queuefs"]["config"]["recover_stale_sec"] == 17- assert plugins["queuefs"]["config"]["busy_timeout_ms"] == 1234+ mount_backups = plugins["localfs"]["config"]["backups"]+ assert mount_backups["retry_interval_ms"] == 1234+ assert mount_backups["retry_backoff_base_ms"] == 55+ explicit_local, default_local, s3_backup = mount_backups["items"]+ assert explicit_local["backend"] == "localfs"+ assert explicit_local["params"]["local_dir"] == str(explicit_backup_dir)+ assert not explicit_backup_dir.exists()-def test_generate_plugin_config_ignores_db_paths_in_memory_mode():- config = AGFSConfig(- path="/tmp/ov-test",- backend="local",- queue_db_path="/tmp/legacy-queue.db",- queuefs={"backend": "memory", "db_path": "/tmp/new-queue.db"},+ assert default_local["backend"] == "localfs"+ assert default_local["params"]["local_dir"] == str(+ tmp_path / "viking" / "_backups" / "local-default")- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))-- assert plugins["queuefs"]["config"]["backend"] == "memory"- assert "db_path" not in plugins["queuefs"]["config"]+ assert s3_backup["backend"] == "s3fs"+ assert s3_backup["params"] == {+ "bucket": "backup-bucket",+ "region": "cn-beijing",+ "access_key_id": "test-access-key",+ "secret_access_key": "test-secret-key",+ "endpoint": "https://tos.example.com",+ "prefix": "backup-prefix",+ "disable_ssl": True,+ "use_path_style": False,+ "directory_marker_mode": None,+ "disable_batch_delete": False,+ "normalize_encoding_chars": "#?",+ }class _FakeMountClient:@@ -251,6 +378,11 @@ class _FakeMountClient:return None+class _FailingMountClient(_FakeMountClient):+ def mount(self, plugin_name, mount_path, config):+ raise RuntimeError(f"mount failed: {plugin_name}:{mount_path}")++class _FakeBindingClient:def __init__(self, *, config):self.config = config@@ -298,6 +430,15 @@ def test_mount_agfs_backend_creates_queue_sqlite_dirs_for_sqlite_backend(tmp_patassert queuefs_mount[2]["db_path"] == str(queue_db_path.resolve())+def test_mount_agfs_backend_raises_mount_error(tmp_path):+ """Mount failures must fail fast instead of being delayed to later filesystem calls."""+ config = AGFSConfig(path=str(tmp_path), backend="local")+ client = _FailingMountClient()++ with pytest.raises(RuntimeError, match="mount failed"):+ mount_agfs_backend(client, config)++def test_ragfs_binding_config_builds_single_binding_dict_for_local_backend(tmp_path):agfs_config = AGFSConfig(path=str(tmp_path), backend="local")diff --git a/tests/server/test_api_key_manager.py b/tests/server/test_api_key_manager.pyindex f8820aa6a..982c4fc57 100644--- a/tests/server/test_api_key_manager.py+++ b/tests/server/test_api_key_manager.py@@ -8,6 +8,7 @@ import uuidimport pytestimport pytest_asyncio+from openviking.pyagfs.exceptions import AGFSNotFoundErrorfrom openviking.server.api_keys import APIKeyManagerfrom openviking.server.identity import Rolefrom openviking.service.core import OpenVikingService@@ -91,6 +92,30 @@ async def test_create_duplicate_account_raises(manager: APIKeyManager):await manager.create_account(acct, "bob")+async def test_create_account_rolls_back_when_user_persistence_fails(+ manager: APIKeyManager, monkeypatch: pytest.MonkeyPatch+):+ """create_account should not leave partial in-memory state after a write failure."""+ acct = _uid()+ original_save_users_json = manager._legacy._save_users_json++ async def _fail_save_users_json(account_id: str) -> None:+ if account_id == acct:+ raise AGFSNotFoundError(account_id)+ await original_save_users_json(account_id)++ monkeypatch.setattr(manager._legacy, "_save_users_json", _fail_save_users_json)++ with pytest.raises(AGFSNotFoundError):+ await manager.create_account(acct, "alice")++ assert acct not in manager._legacy._accounts++ monkeypatch.setattr(manager._legacy, "_save_users_json", original_save_users_json)+ retry_key = await manager.create_account(acct, "alice")+ assert manager.resolve(retry_key).account_id == acct++async def test_delete_account(manager: APIKeyManager):"""Deleting account should invalidate all its user keys."""acct = _uid()diff --git a/tests/server/test_api_webdav.py b/tests/server/test_api_webdav.pyindex b44b03638..8a10a822d 100644--- a/tests/server/test_api_webdav.py+++ b/tests/server/test_api_webdav.py@@ -7,6 +7,9 @@ import xml.etree.ElementTree as ETimport httpx+from openviking.server.routers.webdav import _ensure_exposed_path, _exposed_child_entries+from openviking_cli.exceptions import NotFoundError+def _dav_display_names(xml_bytes: bytes) -> list[str]:root = ET.fromstring(xml_bytes)@@ -52,6 +55,9 @@ async def test_webdav_propfind_hides_reserved_files_but_keeps_user_dotdirs(clienassert ".abstract.md" not in namesassert ".overview.md" not in namesassert ".relations.json" not in names+ assert ".path.ovlock" not in names+ assert ".sync_log.json" not in names+ assert ".redirect.json" not in namesasync def test_webdav_rejects_direct_access_to_reserved_semantic_files(client_with_resource):@@ -62,6 +68,36 @@ async def test_webdav_rejects_direct_access_to_reserved_semantic_files(client_wiassert resp.status_code == 404+async def test_webdav_rejects_direct_access_to_multiwrite_internal_files(client_with_resource):+ client, uri = client_with_resource+ webdav_path = _webdav_path_from_uri(uri)++ for hidden_name in (".path.ovlock", ".sync_log.json", ".redirect.json"):+ resp = await client.get(f"/webdav/resources/{webdav_path}/{hidden_name}")+ assert resp.status_code == 404+++def test_webdav_unit_rejects_direct_access_to_multiwrite_internal_files():+ for hidden_name in (".path.ovlock", ".sync_log.json", ".redirect.json"):+ try:+ _ensure_exposed_path(f"workspace/{hidden_name}")+ except NotFoundError:+ continue+ raise AssertionError(f"{hidden_name} should be rejected")+++def test_webdav_unit_filters_multiwrite_internal_files_from_children():+ entries = [+ {"name": ".obsidian"},+ {"name": ".path.ovlock"},+ {"name": ".sync_log.json"},+ {"name": ".redirect.json"},+ {"name": "notes.md"},+ ]+ filtered = _exposed_child_entries(entries)+ assert [entry["name"] for entry in filtered] == [".obsidian", "notes.md"]++async def test_webdav_put_create_get_and_replace_text_file(client: httpx.AsyncClient):mkcol_resp = await client.request("MKCOL", "/webdav/resources/scratch")assert mkcol_resp.status_code == 201diff --git a/tests/server/test_auth.py b/tests/server/test_auth.pyindex b54556329..b5c2a866e 100644--- a/tests/server/test_auth.py+++ b/tests/server/test_auth.py@@ -337,6 +337,35 @@ async def test_auth_on_multiple_endpoints(auth_client: httpx.AsyncClient):assert tenant_resp.status_code == 403assert tenant_resp.json()["error"]["code"] == "PERMISSION_DENIED"++async def test_admin_sync_route_accepts_root_key(auth_client: httpx.AsyncClient, auth_service):+ """ROOT keys should be allowed to call the system sync admin route."""+ calls: list[str] = []++ async def _fake_system_sync_status(uri: str, ctx):+ calls.append(uri)+ return {"path": uri, "entry_count": 1}++ auth_service.fs.system_sync_status = _fake_system_sync_status++ resp = await auth_client.get(+ "/api/v1/system/sync/viking://resources",+ headers={"X-API-Key": ROOT_KEY},+ )+ assert resp.status_code == 200+ assert resp.json()["result"] == {"path": "viking://resources", "entry_count": 1}+ assert calls == ["viking://resources"]+++async def test_admin_sync_route_rejects_user_key(auth_client: httpx.AsyncClient, user_key: str):+ """Regular user keys must not access the system sync admin route."""+ resp = await auth_client.get(+ "/api/v1/system/sync/viking://resources",+ headers={"X-API-Key": user_key},+ )+ assert resp.status_code == 403+ assert resp.json()["error"]["code"] == "PERMISSION_DENIED"+tenant_resp = await auth_client.get("/api/v1/fs/ls?uri=viking://",headers={diff --git a/tests/server/test_server_health.py b/tests/server/test_server_health.pyindex 763bee2a9..f7f1fac3d 100644--- a/tests/server/test_server_health.py+++ b/tests/server/test_server_health.py@@ -5,11 +5,12 @@import asyncioimport time+from types import SimpleNamespaceimport httpximport pytest-from openviking.server.app import _on_deferred_init_done, create_app+from openviking.server.app import _initialize_runtime_state, _on_deferred_init_done, create_appfrom openviking.server.config import ServerConfig@@ -79,6 +80,84 @@ async def test_system_status(client: httpx.AsyncClient):assert body["result"]["initialized"] is True+async def test_backend_sync_status_endpoint(client: httpx.AsyncClient, service):+ calls: list[str] = []++ async def _fake_system_sync_status(uri: str, ctx):+ calls.append(uri)+ assert ctx is not None+ return {"path": uri, "entry_count": 1}++ service.fs.system_sync_status = _fake_system_sync_status++ resp = await client.post(+ "/api/v1/system/backend/sync-status",+ json={"uri": "viking://resources"},+ )+ assert resp.status_code == 200+ body = resp.json()+ assert body["status"] == "ok"+ assert body["result"] == {"path": "viking://resources", "entry_count": 1}+ assert calls == ["viking://resources"]+++async def test_backend_sync_retry_endpoint(client: httpx.AsyncClient, service):+ calls: list[str] = []++ async def _fake_system_sync_retry(uri: str, ctx):+ calls.append(uri)+ assert ctx is not None+ return {"path": uri, "retried": 2, "failed": 0}++ service.fs.system_sync_retry = _fake_system_sync_retry++ resp = await client.post(+ "/api/v1/system/backend/sync-retry",+ json={"uri": "viking://resources"},+ )+ assert resp.status_code == 200+ body = resp.json()+ assert body["status"] == "ok"+ assert body["result"] == {"path": "viking://resources", "retried": 2, "failed": 0}+ assert calls == ["viking://resources"]+++async def test_admin_sync_status_route(client: httpx.AsyncClient, service):+ calls: list[str] = []++ async def _fake_system_sync_status(uri: str, ctx):+ calls.append(uri)+ assert ctx is not None+ return {"path": uri, "entry_count": 3}++ service.fs.system_sync_status = _fake_system_sync_status++ resp = await client.get("/api/v1/system/sync/viking://resources")+ assert resp.status_code == 200+ body = resp.json()+ assert body["status"] == "ok"+ assert body["result"] == {"path": "viking://resources", "entry_count": 3}+ assert calls == ["viking://resources"]+++async def test_admin_sync_retry_route(client: httpx.AsyncClient, service):+ calls: list[str] = []++ async def _fake_system_sync_retry(uri: str, ctx):+ calls.append(uri)+ assert ctx is not None+ return {"path": uri, "retried": 4, "failed": 1}++ service.fs.system_sync_retry = _fake_system_sync_retry++ resp = await client.post("/api/v1/system/sync/viking://resources/retry")+ assert resp.status_code == 200+ body = resp.json()+ assert body["status"] == "ok"+ assert body["result"] == {"path": "viking://resources", "retried": 4, "failed": 1}+ assert calls == ["viking://resources"]++async def test_process_time_header(client: httpx.AsyncClient):resp = await client.get("/health")assert "x-process-time" in resp.headers@@ -156,9 +235,14 @@ async def test_ready_returns_200_after_initialized(monkeypatch):"""Ready returns 200 when service is fully initialized and subsystems are healthy."""class MockVikingFS:+ """Mock VikingFS for readiness checks."""+async def ls(self, path, ctx=None):return []+ async def system_sync_status(self, uri, ctx=None):+ return {"path": uri, "entry_count": 0}+def _get_vector_store(self):class MockVectorStore:async def health_check(self):@@ -184,6 +268,9 @@ async def test_ready_returns_200_after_initialized(monkeypatch):assert resp.status_code == 200body = resp.json()assert body["status"] == "ready"+ assert body["checks"]["agfs"]["status"] == "ok"+ assert body["checks"]["agfs"]["checks"]["filesystem"] == "ok"+ assert body["checks"]["agfs"]["checks"]["multiwrite_sync"] == "ok"async def test_slow_init_does_not_block_health(monkeypatch):@@ -206,3 +293,37 @@ async def test_slow_init_does_not_block_health(monkeypatch):assert resp.status_code == 200# Health responds instantly since it's stateless (no service dependency)assert elapsed < 0.5+++async def test_initialize_runtime_state_loads_api_key_manager(monkeypatch):+ """API key auth must finish manager loading before the app is considered ready."""++ class MockService:+ def __init__(self):+ self._initialized = False+ self.viking_fs = object()++ async def initialize(self):+ self._initialized = True++ class FakeAPIKeyManager:+ def __init__(self, root_key, viking_fs, api_key_hashing_enabled):+ self.root_key = root_key+ self.viking_fs = viking_fs+ self.api_key_hashing_enabled = api_key_hashing_enabled+ self.loaded = False++ async def load(self):+ self.loaded = True++ monkeypatch.setattr("openviking.server.app.APIKeyManager", FakeAPIKeyManager)++ app = SimpleNamespace(state=SimpleNamespace(api_key_manager=None))+ service = MockService()+ config = ServerConfig(root_api_key="root-key-for-test")++ await _initialize_runtime_state(app, service, config)++ assert service._initialized is True+ assert app.state.api_key_manager is not None+ assert app.state.api_key_manager.loaded is Truediff --git a/tests/storage/test_viking_fs_tree.py b/tests/storage/test_viking_fs_tree.pyindex 11cac777d..ce8b86051 100644--- a/tests/storage/test_viking_fs_tree.py+++ b/tests/storage/test_viking_fs_tree.py@@ -147,10 +147,12 @@ def test_is_name_visible_at_account_root(fs, name, parent_path, expected):("_system", "/local/test_account/resources", False),("tasks", "/local/test_account/resources/bar", False),(".path.ovlock", "/local/test_account/resources", False),+ (".sync_log.json", "/local/test_account/resources", False),+ (".redirect.json", "/local/test_account/resources", False),],)def test_is_name_visible_at_non_root(fs, name, parent_path, expected):- """PY-FLT-004: Non-root _INTERNAL_NAMES blacklist."""+ """PY-FLT-004: Non-root internal-name blacklist."""assert fs._is_name_visible_at_path(name, parent_path) == expected@@ -201,6 +203,18 @@ def test_is_tree_entry_visible_path_ovlock_filtered(monkeypatch, fs):assert fs._is_tree_entry_visible(entry, "/local/test_account", _default_ctx()) is False+def test_is_tree_entry_visible_multiwrite_meta_filtered(monkeypatch, fs):+ """PY-FLT-012: multi-write metadata files are filtered."""+ patch_visibility(monkeypatch, fs, is_accessible=True)+ for hidden_name in (".sync_log.json", ".redirect.json"):+ entry = make_entry(+ f"/local/test_account/resources/{hidden_name}",+ hidden_name,+ is_dir=False,+ )+ assert fs._is_tree_entry_visible(entry, "/local/test_account", _default_ctx()) is False++def test_is_tree_entry_visible_default_ctx(monkeypatch, fs):"""PY-FLT-010: ctx=None uses default context."""patch_visibility(monkeypatch, fs, is_accessible=True)