Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feature(storage): support multi write storage (#2466)

baojun-zhang7ee1481e1dc1
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs
index 288db767a..70fdd09a9 100644
--- a/crates/ov_cli/src/client.rs
+++ b/crates/ov_cli/src/client.rs
@@ -234,6 +234,20 @@ impl HttpClient {
self.post("/api/v1/system/consistency", &body).await
}
+ pub async fn backend_sync_status(&self, uri: &str) -> Result<serde_json::Value> {
+ let body = serde_json::json!({
+ "uri": uri,
+ });
+ self.post("/api/v1/system/backend/sync-status", &body).await
+ }
+
+ pub async fn backend_sync_retry(&self, uri: &str) -> Result<serde_json::Value> {
+ let body = serde_json::json!({
+ "uri": uri,
+ });
+ self.post("/api/v1/system/backend/sync-retry", &body).await
+ }
+
/// Download file as raw bytes
pub async fn get_bytes(&self, uri: &str) -> Result<Vec<u8>> {
let url = format!("{}/api/v1/content/download", self.base.base_url);
diff --git a/crates/ov_cli/src/commands/system.rs b/crates/ov_cli/src/commands/system.rs
index 037cfae04..5ad24fff8 100644
--- a/crates/ov_cli/src/commands/system.rs
+++ b/crates/ov_cli/src/commands/system.rs
@@ -81,6 +81,28 @@ pub async fn consistency(
Ok(())
}
+pub async fn backend_sync_status(
+ client: &HttpClient,
+ uri: &str,
+ output_format: OutputFormat,
+ compact: bool,
+) -> Result<()> {
+ let response: serde_json::Value = client.backend_sync_status(uri).await?;
+ output_success(&response, output_format, compact);
+ Ok(())
+}
+
+pub async fn backend_sync_retry(
+ client: &HttpClient,
+ uri: &str,
+ output_format: OutputFormat,
+ compact: bool,
+) -> Result<()> {
+ let response: serde_json::Value = client.backend_sync_retry(uri).await?;
+ output_success(&response, output_format, compact);
+ Ok(())
+}
+
fn output_consistency_table(response: &serde_json::Value, compact: bool) {
let summary = json!({
"ok": response.get("ok").and_then(|v| v.as_bool()).unwrap_or(false),
diff --git a/crates/ov_cli/src/error_ui.rs b/crates/ov_cli/src/error_ui.rs
index 97260c046..ec5752f0a 100644
--- a/crates/ov_cli/src/error_ui.rs
+++ b/crates/ov_cli/src/error_ui.rs
@@ -437,6 +437,12 @@ fn system_help_command(program: &str, tokens: &[&str]) -> String {
Some("init-key") => format!("{program} system crypto init-key --help"),
_ => format!("{program} system crypto --help"),
},
+ Some("backend") => match tokens.get(3).copied() {
+ Some("sync-status" | "sync-retry") => {
+ format!("{program} system backend {} --help", tokens[3])
+ }
+ _ => format!("{program} system backend --help"),
+ },
Some("wait" | "status" | "health" | "consistency") => {
format!("{program} system {} --help", tokens[2])
}
diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rs
index e89e7e17a..f1e9d1039 100644
--- a/crates/ov_cli/src/handlers.rs
+++ b/crates/ov_cli/src/handlers.rs
@@ -224,7 +224,7 @@ pub async fn handle_restore(
.await
}
-use crate::SystemCommands;
+use crate::{SystemBackendCommands, SystemCommands};
pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> {
let client = ctx.get_client();
@@ -249,6 +249,16 @@ pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> {
commands::system::consistency(&client, &uri, ctx.output_format, ctx.compact).await
}
SystemCommands::Crypto { action } => commands::crypto::handle_crypto(action).await,
+ SystemCommands::Backend { action } => match action {
+ SystemBackendCommands::SyncStatus { uri } => {
+ commands::system::backend_sync_status(&client, &uri, ctx.output_format, ctx.compact)
+ .await
+ }
+ SystemBackendCommands::SyncRetry { uri } => {
+ commands::system::backend_sync_retry(&client, &uri, ctx.output_format, ctx.compact)
+ .await
+ }
+ },
}
}
diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rs
index 8b78eb31c..eb420daba 100644
--- a/crates/ov_cli/src/help_ui.rs
+++ b/crates/ov_cli/src/help_ui.rs
@@ -2096,7 +2096,7 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[
},
CommandHelpSpec {
path: &["system"],
- purpose: "Run server utility, health, consistency, and crypto commands.",
+ purpose: "Run server utility, health, consistency, backend sync, and crypto commands.",
usage: "ov system <subcommand>",
examples: &[
HelpItem {
@@ -2107,6 +2107,10 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[
label: "ov system consistency viking://projects/acme",
description: "Check filesystem/vector consistency.",
},
+ HelpItem {
+ label: "ov system backend sync-status viking://projects/acme",
+ description: "Inspect multi-write backend sync lag.",
+ },
],
arguments: &[],
common_options: &[],
@@ -2120,6 +2124,10 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[
label: "consistency <uri>",
description: "Check subtree consistency.",
},
+ HelpItem {
+ label: "backend <subcommand>",
+ description: "Inspect or repair multi-write backend sync state.",
+ },
HelpItem {
label: "crypto <subcommand>",
description: "Key management commands.",
@@ -2130,6 +2138,38 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[
description: "Use the standard status view.",
}],
},
+ CommandHelpSpec {
+ path: &["system", "backend"],
+ purpose: "Inspect and repair multi-write backend sync state.",
+ usage: "ov system backend <subcommand>",
+ examples: &[
+ HelpItem {
+ label: "ov system backend sync-status viking://resources",
+ description: "Show pending and acknowledged backend sync state.",
+ },
+ HelpItem {
+ label: "ov system backend sync-retry viking://resources",
+ description: "Retry lagging backend sync targets.",
+ },
+ ],
+ arguments: &[],
+ common_options: &[],
+ advanced_options: &[],
+ subcommands: &[
+ HelpItem {
+ label: "sync-status <uri>",
+ description: "Show multi-write backend sync status for a subtree.",
+ },
+ HelpItem {
+ label: "sync-retry <uri>",
+ description: "Retry pending multi-write backend sync work.",
+ },
+ ],
+ next_steps: &[HelpItem {
+ label: "ov system backend sync-status <uri>",
+ description: "Inspect the subtree again after retry.",
+ }],
+ },
CommandHelpSpec {
path: &["reindex"],
purpose: "Reindex semantic/vector artifacts for a URI.",
diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs
index e84c4d218..a24d7536c 100644
--- a/crates/ov_cli/src/main.rs
+++ b/crates/ov_cli/src/main.rs
@@ -745,6 +745,27 @@ enum SystemCommands {
#[command(subcommand)]
action: commands::crypto::CryptoCommands,
},
+ /// Backend sync inspection and repair commands
+ Backend {
+ #[command(subcommand)]
+ action: SystemBackendCommands,
+ },
+}
+
+#[derive(Subcommand)]
+enum SystemBackendCommands {
+ /// Show multi-write backend sync status for a URI subtree
+ #[command(name = "sync-status")]
+ SyncStatus {
+ /// Viking URI to inspect
+ uri: String,
+ },
+ /// Retry pending multi-write backend sync work for a URI subtree
+ #[command(name = "sync-retry")]
+ SyncRetry {
+ /// Viking URI to repair
+ uri: String,
+ },
}
#[derive(Subcommand)]
@@ -1507,6 +1528,10 @@ fn known_task_command_requires_config(tokens: &[String]) -> bool {
fn known_system_command_requires_config(tokens: &[String]) -> bool {
match tokens.get(1).map(String::as_str) {
Some("wait" | "status" | "health" | "consistency") => true,
+ Some("backend") => matches!(
+ tokens.get(2).map(String::as_str),
+ None | Some("sync-status" | "sync-retry")
+ ),
Some("crypto") => matches!(tokens.get(2).map(String::as_str), None | Some("init-key")),
_ => false,
}
@@ -2327,7 +2352,7 @@ mod tests {
preprocess_privacy_args,
};
use crate::config::{Config, DEFAULT_CUSTOM_URL};
- use crate::handlers;
+ use crate::{SystemBackendCommands, SystemCommands, handlers};
use crate::output::OutputFormat;
use clap::{CommandFactory, Parser};
use std::ffi::OsString;
@@ -2372,6 +2397,23 @@ mod tests {
}
}
+ #[test]
+ fn cli_parses_system_backend_sync_status() {
+ let cli = Cli::try_parse_from(["ov", "system", "backend", "sync-status", "viking://a"])
+ .expect("system backend sync-status should parse");
+
+ match cli.command {
+ Commands::System { action } => match action {
+ SystemCommands::Backend { action } => match action {
+ SystemBackendCommands::SyncStatus { uri } => assert_eq!(uri, "viking://a"),
+ _ => panic!("expected backend sync-status command"),
+ },
+ _ => panic!("expected system backend command"),
+ },
+ _ => panic!("expected system command"),
+ }
+ }
+
#[test]
fn server_commands_require_existing_cli_config() {
let cli = Cli::try_parse_from(["ov", "ls"]).expect("ls should parse");
@@ -2510,6 +2552,7 @@ mod tests {
&["ov", "config", "validate"],
&["ov", "config", "show"],
&["ov", "system", "consistency"],
+ &["ov", "system", "backend", "sync-status"],
&["ov", "admin", "list-accounts"],
] {
assert!(
@@ -2588,6 +2631,9 @@ mod tests {
&["ov", "system", "status"],
&["ov", "system", "health"],
&["ov", "system", "consistency"],
+ &["ov", "system", "backend"],
+ &["ov", "system", "backend", "sync-status"],
+ &["ov", "system", "backend", "sync-retry"],
&["ov", "system", "crypto"],
&["ov", "system", "crypto", "init-key"],
&["ov", "session", "new"],
@@ -2970,6 +3016,10 @@ mod tests {
vec!["ov", "system", "consistency", "help"],
"ov system consistency --help",
),
+ (
+ vec!["ov", "system", "backend", "sync-status", "help"],
+ "ov system backend sync-status --help",
+ ),
] {
let misuse = plain_help_misuse(&os_args(&args))
.unwrap_or_else(|| panic!("{args:?} should be plain-help misuse"));
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs
index 02beed13e..5f580485c 100644
--- a/crates/ragfs-python/src/lib.rs
+++ b/crates/ragfs-python/src/lib.rs
@@ -12,9 +12,10 @@ use std::future::Future;
use std::sync::Arc;
use std::time::UNIX_EPOCH;
+use ragfs::core::builder::EncryptionConfig;
use ragfs::core::{
- build_default_stack, ConfigValue, EncryptionConfig, FileInfo, FileSystem, FilesystemStats,
- FsContext, FsContextInner, FsOperation, GrepResult, MountableFS, OperationStats, PluginConfig,
+ build_default_stack, ConfigValue, FileInfo, FileSystem, FilesystemStats, FsContext,
+ FsContextInner, FsOperation, GrepResult, MountableFS, OperationStats, PluginConfig,
RagfsConfig, TreeEntry, WriteFlag, FS_CTX,
};
@@ -72,6 +73,8 @@ fn to_py_err(e: ragfs::core::Error) -> PyErr {
ragfs::core::Error::Serialization(_) => new_py_err("AGFSSerializationError", msg),
ragfs::core::Error::Network(_) => new_py_err("AGFSNetworkError", msg),
ragfs::core::Error::Timeout(_) => new_py_err("AGFSTimeoutError", msg),
+ ragfs::core::Error::SyncWriteQuorum { .. } => new_py_err("AGFSInternalError", msg),
+ ragfs::core::Error::ContextMissing(_) => new_py_err("AGFSInternalError", msg),
ragfs::core::Error::Internal(_) => new_py_err("AGFSInternalError", msg),
}
}
@@ -232,25 +235,86 @@ fn days_to_ymd(days: u64) -> (u64, u64, u64) {
(y, m, d)
}
-/// Convert a Python dict to HashMap<String, ConfigValue>
+/// Convert a Python dict to HashMap<String, ConfigValue>, with recursive nesting support.
fn py_dict_to_config(dict: &Bound<'_, PyDict>) -> PyResult<HashMap<String, ConfigValue>> {
let mut params = HashMap::new();
for (k, v) in dict.iter() {
let key: String = k.extract()?;
- let value = if let Ok(s) = v.extract::<String>() {
- ConfigValue::String(s)
- } else if let Ok(b) = v.extract::<bool>() {
- ConfigValue::Bool(b)
- } else if let Ok(i) = v.extract::<i64>() {
- ConfigValue::Int(i)
- } else {
- ConfigValue::String(v.str()?.to_string())
- };
+ let value = py_any_to_config_value(&v)?;
params.insert(key, value);
}
Ok(params)
}
+/// Convert a single Python value to ConfigValue, recursing into dicts/lists.
+fn py_any_to_config_value(v: &Bound<'_, PyAny>) -> PyResult<ConfigValue> {
+ if let Ok(s) = v.extract::<String>() {
+ Ok(ConfigValue::String(s))
+ } else if let Ok(b) = v.extract::<bool>() {
+ Ok(ConfigValue::Bool(b))
+ } else if let Ok(i) = v.extract::<i64>() {
+ Ok(ConfigValue::Int(i))
+ } else if let Ok(list) = v.downcast::<PyList>() {
+ let mut strings = Vec::new();
+ let mut has_nested = false;
+ for item in list.iter() {
+ if let Ok(s) = item.extract::<String>() {
+ strings.push(s);
+ } else {
+ has_nested = true;
+ break;
+ }
+ }
+ if has_nested || strings.is_empty() {
+ // Convert to serde_json::Value for complex lists
+ let json_val = py_to_json_value(v)?;
+ Ok(ConfigValue::Json(json_val))
+ } else {
+ Ok(ConfigValue::StringList(strings))
+ }
+ } else if v.is_instance_of::<PyDict>() {
+ let json_val = py_to_json_value(v)?;
+ Ok(ConfigValue::Json(json_val))
+ } else {
+ // Fallback: try string representation
+ Ok(ConfigValue::String(v.str()?.to_string()))
+ }
+}
+
+/// Convert a Python value to serde_json::Value recursively.
+fn py_to_json_value(v: &Bound<'_, PyAny>) -> PyResult<serde_json::Value> {
+ if v.is_none() {
+ Ok(serde_json::Value::Null)
+ } else if let Ok(b) = v.extract::<bool>() {
+ Ok(serde_json::Value::Bool(b))
+ } else if let Ok(i) = v.extract::<i64>() {
+ Ok(serde_json::Value::Number(i.into()))
+ } else if let Ok(f) = v.extract::<f64>() {
+ if let Some(n) = serde_json::Number::from_f64(f) {
+ Ok(serde_json::Value::Number(n))
+ } else {
+ Ok(serde_json::Value::String(f.to_string()))
+ }
+ } else if let Ok(s) = v.extract::<String>() {
+ Ok(serde_json::Value::String(s))
+ } else if let Ok(list) = v.downcast::<PyList>() {
+ let mut arr = Vec::new();
+ for item in list.iter() {
+ arr.push(py_to_json_value(&item)?);
+ }
+ Ok(serde_json::Value::Array(arr))
+ } else if let Ok(dict) = v.downcast::<PyDict>() {
+ let mut map = serde_json::Map::new();
+ for (k, val) in dict.iter() {
+ let key: String = k.extract()?;
+ map.insert(key, py_to_json_value(&val)?);
+ }
+ Ok(serde_json::Value::Object(map))
+ } else {
+ Ok(serde_json::Value::String(v.str()?.to_string()))
+ }
+}
+
/// Build an immutable `FsContext` from an optional Python `{"account_id": str}` dict.
///
/// A context is always constructed (even when the dict is absent or lacks `account_id`), so the
@@ -462,42 +526,6 @@ impl RAGFSBindingClient {
self.read(py, path, offset, size, stream, ctx)
}
- /// Read raw bytes straight from the mount layer, bypassing the encryption layer.
- ///
- /// Used by cp / persist whole-blob copies (ciphertext is moved verbatim, never decrypted).
- #[pyo3(signature = (path, offset=0, size=-1))]
- fn read_raw(
- &self,
- py: Python<'_>,
- path: String,
- offset: i64,
- size: i64,
- ) -> PyResult<Py<PyAny>> {
- let mountable = self.mountable.clone();
- let off = if offset < 0 { 0u64 } else { offset as u64 };
- let sz = if size < 0 { 0u64 } else { size as u64 };
- let data = py_detach_blocking(py, move || {
- self.rt
- .block_on(async move { mountable.read(&path, off, sz).await })
- })
- .map_err(to_py_err)?;
- Python::attach(|py| Ok(PyBytes::new(py, &data).into()))
- }
-
- /// Write raw bytes straight to the mount layer, bypassing the encryption layer.
- ///
- /// Counterpart to `read_raw` for cp / persist whole-blob copies.
- fn write_raw(&self, py: Python<'_>, path: String, data: Vec<u8>) -> PyResult<String> {
- let mountable = self.mountable.clone();
- let len = data.len();
- py_detach_blocking(py, move || {
- self.rt
- .block_on(async move { mountable.write(&path, &data, 0, WriteFlag::Create).await })
- })
- .map_err(to_py_err)?;
- Ok(format!("Written {} bytes", len))
- }
-
/// Write data to file.
///
/// Args:
@@ -660,6 +688,28 @@ impl RAGFSBindingClient {
Ok(m)
}
+ /// Attempt a same-mount verbatim copy and report whether the fast-path was used.
+ #[pyo3(signature = (src_path, dst_path, ctx=None))]
+ fn copy_within_mount(
+ &self,
+ py: Python<'_>,
+ src_path: String,
+ dst_path: String,
+ ctx: Option<HashMap<String, String>>,
+ ) -> PyResult<HashMap<String, bool>> {
+ let fs_ctx = build_fs_context(ctx);
+ let mountable = self.mountable.clone();
+ let performed = self
+ .run_scoped(py, fs_ctx, move || async move {
+ mountable.copy_within_mount(&src_path, &dst_path).await
+ })
+ .map_err(to_py_err)?;
+
+ let mut result = HashMap::new();
+ result.insert("performed".to_string(), performed);
+ Ok(result)
+ }
+
/// Change file permissions.
#[pyo3(signature = (path, mode, ctx=None))]
fn chmod(
@@ -737,7 +787,11 @@ impl RAGFSBindingClient {
/// Args:
/// fstype: Filesystem type (e.g., "memfs", "sqlfs", "kvfs", "queuefs")
/// path: Mount path
- /// config: Plugin configuration as dict
+ /// config: Plugin configuration as dict. For multi-write, include:
+ /// - "backups": nested dict matching BackendsConfig schema
+ /// - "server_encryption_enabled": bool
+ /// - "primary_encryption_enabled": bool
+ /// - "primary_redirects": list of redirect policy dicts
#[pyo3(signature = (fstype, path, config=None))]
fn mount(
&self,
@@ -750,12 +804,8 @@ impl RAGFSBindingClient {
Some(dict) => py_dict_to_config(dict)?,
None => HashMap::new(),
};
-
- let plugin_config = PluginConfig {
- name: fstype.clone(),
- mount_path: path.clone(),
- params,
- };
+ let plugin_config = PluginConfig::from_raw_parts(fstype.clone(), path.clone(), params)
+ .map_err(to_py_err)?;
let mountable = self.mountable.clone();
py_detach_blocking(py, move || {
@@ -927,6 +977,58 @@ impl RAGFSBindingClient {
})
}
+ /// Query multi-write sync status under a file or directory path.
+ ///
+ /// Args:
+ /// path: Absolute AGFS path rooted at the mounted namespace
+ /// ctx: Optional FsContext dict (e.g. {"account_id": ...})
+ ///
+ /// Returns:
+ /// A JSON-like dict describing pending and acknowledged backup sync state.
+ #[pyo3(signature = (path, ctx=None))]
+ fn system_sync_status(
+ &self,
+ py: Python<'_>,
+ path: String,
+ ctx: Option<HashMap<String, String>>,
+ ) -> PyResult<Py<PyAny>> {
+ let fs_ctx = build_fs_context(ctx);
+ let mountable = self.mountable.clone();
+ let result = self
+ .run_scoped(py, fs_ctx, move || async move {
+ mountable.system_sync_status(&path).await
+ })
+ .map_err(to_py_err)?;
+
+ Python::attach(|py| serde_json_to_py(py, &result))
+ }
+
+ /// Manually retry lagging multi-write backup sync operations under a path.
+ ///
+ /// Args:
+ /// path: Absolute AGFS path rooted at the mounted namespace
+ /// ctx: Optional FsContext dict (e.g. {"account_id": ...})
+ ///
+ /// Returns:
+ /// A JSON-like dict summarizing retry results per target backend.
+ #[pyo3(signature = (path, ctx=None))]
+ fn system_sync_retry(
+ &self,
+ py: Python<'_>,
+ path: String,
+ ctx: Option<HashMap<String, String>>,
+ ) -> PyResult<Py<PyAny>> {
+ let fs_ctx = build_fs_context(ctx);
+ let mountable = self.mountable.clone();
+ let result = self
+ .run_scoped(py, fs_ctx, move || async move {
+ mountable.system_sync_retry(&path).await
+ })
+ .map_err(to_py_err)?;
+
+ Python::attach(|py| serde_json_to_py(py, &result))
+ }
+
/// Calculate file digest (not yet implemented in ragfs).
#[pyo3(signature = (path, algorithm="xxh3"))]
fn digest(&self, path: String, algorithm: &str) -> PyResult<HashMap<String, String>> {
@@ -1025,7 +1127,9 @@ mod tests {
Python::attach(|py| {
let ty = py.get_type::<RAGFSBindingClient>();
let kwargs = PyDict::new(py);
- kwargs.set_item("config_path", "/tmp/legacy-ov.conf").unwrap();
+ kwargs
+ .set_item("config_path", "/tmp/legacy-ov.conf")
+ .unwrap();
let instance = ty.call((), Some(&kwargs));
assert!(instance.is_ok());
});
diff --git a/crates/ragfs/src/core/builder.rs b/crates/ragfs/src/core/builder.rs
index 88ae2c063..da0c3e7c0 100644
--- a/crates/ragfs/src/core/builder.rs
+++ b/crates/ragfs/src/core/builder.rs
@@ -1,12 +1,14 @@
//! Standard RAGFS stack builder.
//!
//! Centralizes "register all built-in plugins + assemble the wrapper stack" so the binding has a
-//! single construction path. Whether the encryption layer is present is decided *here, at
-//! construction time*, by whether `RagfsConfig::encryption` is set (see §2.4/§2.6 of the design).
+//! single construction path. Encryption is now applied per-backend inside `MountableFS::mount()`
+//! (single backend) or `MountableFS::build_multi_write_fs()` (multi-write), so the global
+//! `EncryptionWrappedFS` wrapper is removed from this builder.
+//!
+//! The top layer is always `StatsWrappedFS` so end-to-end timing (including crypto) is captured.
use std::sync::Arc;
-use super::encryption_wrapper::EncryptionWrappedFS;
use super::filesystem::FileSystem;
use super::mountable::MountableFS;
use super::stats_wrapper::StatsWrappedFS;
@@ -23,7 +25,7 @@ use crate::plugins::{
/// `build_default_stack`'s signature.
#[derive(Default)]
pub struct RagfsConfig {
- /// Encryption section: `None` → plaintext stack; `Some` → wrap an `EncryptionWrappedFS`.
+ /// Encryption section: `None` → plaintext stack; `Some` → per-backend encryption wrapping.
pub encryption: Option<EncryptionConfig>,
}
@@ -39,29 +41,29 @@ pub struct EncryptionConfig {
pub struct RagfsStack {
/// Mount manager (mount/unmount/list/stats/register_plugin live here).
pub mountable: Arc<MountableFS>,
- /// Data entry point: `Stats(Encryption(Mountable))` or `Stats(Mountable)`.
+ /// Data entry point: `Stats(Mountable)` — encryption is per-backend inside mount.
pub top: Arc<dyn FileSystem>,
}
/// Build the standard RAGFS stack.
///
-/// `config.encryption == None` → no encryption layer (plaintext); `Some` → insert
-/// `EncryptionWrappedFS` in the middle. The top is always `StatsWrappedFS` so end-to-end timing
-/// (including crypto) is captured.
+/// Encryption config is forwarded to `MountableFS` so it can wrap individual backends
+/// with `EncryptionWrappedFS` during `mount()`. The top is always `StatsWrappedFS`.
pub async fn build_default_stack(config: RagfsConfig) -> RagfsStack {
let mountable = Arc::new(MountableFS::new());
register_builtin_plugins(&mountable).await;
- let data_fs: Arc<dyn FileSystem> = match config.encryption {
- Some(enc) => Arc::new(EncryptionWrappedFS::new(
- mountable.clone() as Arc<dyn FileSystem>,
- enc.root_key,
- enc.provider_type,
- )),
- None => mountable.clone() as Arc<dyn FileSystem>,
- };
+ // Forward encryption config to MountableFS for per-backend wrapping.
+ if let Some(enc) = &config.encryption {
+ mountable
+ .set_encryption_config(Some(enc.root_key), Some(enc.provider_type))
+ .await;
+ }
- let top: Arc<dyn FileSystem> = Arc::new(StatsWrappedFS::with_arc(data_fs));
+ // MountableFS is the data entry point; encryption is applied inside mount().
+ let top: Arc<dyn FileSystem> = Arc::new(StatsWrappedFS::with_arc(
+ mountable.clone() as Arc<dyn FileSystem>
+ ));
RagfsStack { mountable, top }
}
@@ -81,9 +83,11 @@ pub async fn register_builtin_plugins(fs: &MountableFS) {
mod tests {
use super::*;
use crate::core::context::{FsContextInner, FS_CTX};
- use crate::core::{PluginConfig, WriteFlag};
+ use crate::core::{ConfigValue, PluginConfig, Result, WriteFlag};
use crate::crypto;
use std::collections::HashMap;
+ use std::fs;
+ use tempfile::TempDir;
fn enc_config() -> RagfsConfig {
RagfsConfig {
@@ -94,18 +98,36 @@ mod tests {
}
}
+ /// Create a plugin config for stack builder tests.
+ fn plugin_config(
+ name: &str,
+ mount_path: &str,
+ params: HashMap<String, ConfigValue>,
+ ) -> PluginConfig {
+ PluginConfig::single_backend(name, mount_path, params)
+ }
+
async fn mount_mem(stack: &RagfsStack) {
stack
.mountable
- .mount(PluginConfig {
- name: "memfs".to_string(),
- mount_path: "/mem".to_string(),
- params: HashMap::new(),
- })
+ .mount(plugin_config("memfs", "/mem", HashMap::new()))
.await
.unwrap();
}
+ /// Mount a LocalFS backend backed by a temporary directory.
+ async fn mount_local(stack: &RagfsStack, mount_path: &str, local_dir: &str) -> Result<()> {
+ let mut params = HashMap::new();
+ params.insert(
+ "local_dir".to_string(),
+ ConfigValue::String(local_dir.to_string()),
+ );
+ stack
+ .mountable
+ .mount(plugin_config("localfs", mount_path, params))
+ .await
+ }
+
#[tokio::test]
async fn encrypted_stack_encrypts_on_disk() {
let stack = build_default_stack(enc_config()).await;
@@ -122,8 +144,8 @@ mod tests {
})
.await;
- // Underlying mountable holds ciphertext.
- let raw = stack.mountable.read("/mem/f", 0, 0).await.unwrap();
+ // Read raw bytes from mountable (bypasses encryption layer) to verify ciphertext.
+ let raw = stack.mountable.read_raw("/mem/f", 0, 0).await.unwrap();
assert!(crypto::is_encrypted(&raw));
}
@@ -138,7 +160,7 @@ mod tests {
.write("/mem/f", b"hello", 0, WriteFlag::Create)
.await
.unwrap();
- let raw = stack.mountable.read("/mem/f", 0, 0).await.unwrap();
+ let raw = stack.mountable.read_raw("/mem/f", 0, 0).await.unwrap();
assert_eq!(raw, b"hello", "plaintext stack stores raw bytes");
}
@@ -147,11 +169,7 @@ mod tests {
let stack = build_default_stack(enc_config()).await;
stack
.mountable
- .mount(PluginConfig {
- name: "queuefs".to_string(),
- mount_path: "/queue".to_string(),
- params: HashMap::new(),
- })
+ .mount(plugin_config("queuefs", "/queue", HashMap::new()))
.await
.unwrap();
@@ -172,4 +190,44 @@ mod tests {
})
.await;
}
+
+ #[tokio::test]
+ async fn encrypted_mount_writes_backend_shape_guard() {
+ let dir = TempDir::new().unwrap();
+ let stack = build_default_stack(enc_config()).await;
+
+ mount_local(&stack, "/local", dir.path().to_str().unwrap())
+ .await
+ .unwrap();
+
+ let manifest = fs::read(dir.path().join("backend_meta.json")).unwrap();
+ assert!(crypto::is_encrypted(&manifest));
+ let account_key = crypto::hkdf_sha256(&[4u8; 32], b"_system");
+ let (_provider, enc_key, key_iv, data_iv, ciphertext) =
+ crypto::parse_envelope(&manifest).unwrap();
+ let file_key =
+ crypto::aes_gcm_decrypt(&account_key, key_iv.try_into().unwrap(), enc_key).unwrap();
+ let file_key: [u8; 32] = file_key.as_slice().try_into().unwrap();
+ let plaintext =
+ crypto::aes_gcm_decrypt(&file_key, data_iv.try_into().unwrap(), ciphertext).unwrap();
+ assert!(plaintext.is_empty());
+ }
+
+ #[tokio::test]
+ async fn encrypted_mount_rejects_legacy_plaintext_backend_without_manifest() {
+ let dir = TempDir::new().unwrap();
+ fs::write(dir.path().join("legacy.txt"), b"plaintext").unwrap();
+
+ let stack = build_default_stack(enc_config()).await;
+ let err = mount_local(&stack, "/local", dir.path().to_str().unwrap())
+ .await
+ .unwrap_err();
+
+ match err {
+ crate::core::errors::Error::Config(message) => {
+ assert!(message.contains("backend storage shape mismatch"));
+ }
+ other => panic!("unexpected error: {other:?}"),
+ }
+ }
}
diff --git a/crates/ragfs/src/core/encryption_wrapper.rs b/crates/ragfs/src/core/encryption_wrapper.rs
index 1ac0b7265..18f02c44d 100644
--- a/crates/ragfs/src/core/encryption_wrapper.rs
+++ b/crates/ragfs/src/core/encryption_wrapper.rs
@@ -14,6 +14,7 @@ use std::sync::{Arc, RwLock};
use async_trait::async_trait;
use crate::crypto;
+use crate::shape::SHAPE_MANIFEST_PATH;
use super::context::FsContextView;
use super::errors::{Error, Result};
@@ -35,6 +36,11 @@ pub struct EncryptionWrappedFS {
}
impl EncryptionWrappedFS {
+ /// Get the wrapped filesystem for specialized delegation.
+ pub(crate) fn inner_fs(&self) -> &Arc<dyn FileSystem> {
+ &self.inner
+ }
+
/// Construct an encryption layer over `inner`. Built only when a root key is configured.
pub fn new(inner: Arc<dyn FileSystem>, root_key: [u8; 32], provider_type: u8) -> Self {
Self {
@@ -94,6 +100,11 @@ impl EncryptionWrappedFS {
|| path.starts_with("/serverinfo/")
}
+ /// Return true when one path points at the backend-shape manifest.
+ fn is_shape_manifest_path(path: &str) -> bool {
+ path == SHAPE_MANIFEST_PATH || path.ends_with(SHAPE_MANIFEST_PATH)
+ }
+
/// Derive the encryption account domain from a filesystem path.
///
/// `/local/{account_id}/...` paths are tenant-scoped and use that account id. Control or
@@ -256,7 +267,11 @@ impl FileSystem for EncryptionWrappedFS {
}
async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {
- self.inner.read_dir(path).await
+ let entries = self.inner.read_dir(path).await?;
+ Ok(entries
+ .into_iter()
+ .filter(|entry| entry.name != SHAPE_MANIFEST_PATH.trim_start_matches('/'))
+ .collect())
}
async fn stat(&self, path: &str) -> Result<FileInfo> {
@@ -282,9 +297,14 @@ impl FileSystem for EncryptionWrappedFS {
level_limit: Option<usize>,
) -> Result<Vec<TreeEntry>> {
// Metadata only — no content read, so delegate to preserve plugin-native tree optimizations.
- self.inner
+ let entries = self
+ .inner
.tree_directory(path, show_hidden, node_limit, level_limit)
- .await
+ .await?;
+ Ok(entries
+ .into_iter()
+ .filter(|entry| !Self::is_shape_manifest_path(&entry.path))
+ .collect())
}
async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {
@@ -305,30 +325,21 @@ mod tests {
use crate::core::PluginConfig;
use crate::plugins::MemFSPlugin;
+ /// Build a memfs plugin config for encryption wrapper tests.
+ fn memfs_config(mount_path: &str) -> PluginConfig {
+ PluginConfig::single_backend("memfs", mount_path, HashMap::new())
+ }
+
+ /// Build a memfs-backed stack mounted at the default path.
async fn memfs_stack() -> Arc<MountableFS> {
- let m = Arc::new(MountableFS::new());
- m.register_plugin(MemFSPlugin).await;
- m.mount(PluginConfig {
- name: "memfs".to_string(),
- mount_path: "/mem".to_string(),
- params: HashMap::new(),
- })
- .await
- .unwrap();
- m
+ memfs_stack_at("/mem").await
}
/// Build a memfs-backed stack mounted at the provided path for path-sensitive tests.
async fn memfs_stack_at(mount_path: &str) -> Arc<MountableFS> {
let m = Arc::new(MountableFS::new());
m.register_plugin(MemFSPlugin).await;
- m.mount(PluginConfig {
- name: "memfs".to_string(),
- mount_path: mount_path.to_string(),
- params: HashMap::new(),
- })
- .await
- .unwrap();
+ m.mount(memfs_config(mount_path)).await.unwrap();
m
}
@@ -483,7 +494,9 @@ mod tests {
enc.write("/mem/a.txt", b"first", 0, WriteFlag::Create)
.await
.unwrap();
- let appended = enc.write("/mem/a.txt", b"second", 0, WriteFlag::Append).await;
+ let appended = enc
+ .write("/mem/a.txt", b"second", 0, WriteFlag::Append)
+ .await;
assert!(appended.is_err());
})
.await;
diff --git a/crates/ragfs/src/core/errors.rs b/crates/ragfs/src/core/errors.rs
index b2f802842..f6ff185f6 100644
--- a/crates/ragfs/src/core/errors.rs
+++ b/crates/ragfs/src/core/errors.rs
@@ -4,12 +4,23 @@
//! We use `thiserror` for structured error definitions to ensure type safety
//! and clear error messages.
-use std::io;
use serde_json;
+use std::io;
/// Result type alias for RAGFS operations
pub type Result<T> = std::result::Result<T, Error>;
+/// Structured failure detail for one backup target in synchronous multi-write fanout.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SyncWriteFailureDetail {
+ /// Logical backup backend name.
+ pub backend: String,
+ /// Stable error kind for programmatic inspection.
+ pub kind: String,
+ /// Human-readable error message including the original error display text.
+ pub message: String,
+}
+
/// Main error type for RAGFS operations
#[derive(Debug, thiserror::Error)]
pub enum Error {
@@ -77,6 +88,25 @@ pub enum Error {
#[error("operation timed out: {0}")]
Timeout(String),
+ /// Multi-write synchronous fanout failed to reach the required acknowledgement quorum
+ #[error(
+ "sync write quorum failed: {succeeded}/{attempted} backups succeeded (required {required}); failures: {failures:?}"
+ )]
+ SyncWriteQuorum {
+ /// Number of backup targets that completed successfully.
+ succeeded: usize,
+ /// Minimum number of backup acknowledgements required for success.
+ required: usize,
+ /// Total number of backup targets that were attempted.
+ attempted: usize,
+ /// Structured failure details for each unsuccessful backup target.
+ failures: Vec<SyncWriteFailureDetail>,
+ },
+
+ /// Required filesystem context is missing from the current task
+ #[error("filesystem context missing: {0}")]
+ ContextMissing(String),
+
/// Internal error
#[error("internal error: {0}")]
Internal(String),
@@ -124,6 +154,41 @@ impl Error {
Self::Internal(msg.into())
}
+ /// Create a ContextMissing error
+ pub fn context_missing(msg: impl Into<String>) -> Self {
+ Self::ContextMissing(msg.into())
+ }
+
+ /// Create a Timeout error
+ pub fn timeout(msg: impl Into<String>) -> Self {
+ Self::Timeout(msg.into())
+ }
+
+ /// Return a stable error kind name for structured reporting.
+ pub fn kind_name(&self) -> &'static str {
+ match self {
+ Self::NotFound(_) => "not_found",
+ Self::AlreadyExists(_) => "already_exists",
+ Self::PermissionDenied(_) => "permission_denied",
+ Self::InvalidPath(_) => "invalid_path",
+ Self::NotADirectory(_) => "not_a_directory",
+ Self::IsADirectory(_) => "is_a_directory",
+ Self::DirectoryNotEmpty(_) => "directory_not_empty",
+ Self::InvalidOperation(_) => "invalid_operation",
+ Self::Io(_) => "io",
+ Self::Plugin(_) => "plugin",
+ Self::Config(_) => "config",
+ Self::MountPointNotFound(_) => "mount_point_not_found",
+ Self::MountPointExists(_) => "mount_point_exists",
+ Self::Serialization(_) => "serialization",
+ Self::Network(_) => "network",
+ Self::Timeout(_) => "timeout",
+ Self::SyncWriteQuorum { .. } => "sync_write_quorum",
+ Self::ContextMissing(_) => "context_missing",
+ Self::Internal(_) => "internal",
+ }
+ }
+
/// Create an InvalidOperation error
pub fn invalid_operation(msg: impl Into<String>) -> Self {
Self::InvalidOperation(msg.into())
diff --git a/crates/ragfs/src/core/filesystem.rs b/crates/ragfs/src/core/filesystem.rs
index be60d2b3e..121315d7c 100644
--- a/crates/ragfs/src/core/filesystem.rs
+++ b/crates/ragfs/src/core/filesystem.rs
@@ -6,6 +6,7 @@
use async_trait::async_trait;
use regex::Regex;
+use std::any::Any;
use super::errors::{Error, Result};
use super::types::{FileInfo, GrepResult, TreeEntry, WriteFlag};
@@ -93,7 +94,7 @@ pub(crate) fn compile_grep_regex(pattern: &str, case_insensitive: bool) -> Resul
/// All filesystem plugins must implement this trait to provide file operations.
/// All methods are async to support I/O-bound operations efficiently.
#[async_trait]
-pub trait FileSystem: Send + Sync {
+pub trait FileSystem: Send + Sync + Any {
/// Create an empty file at the specified path
///
/// # Arguments
@@ -786,11 +787,38 @@ mod tests {
entries.iter().map(|e| e.rel_path.clone()).collect()
}
+ /// Run the default tree traversal for the common `/root` test case.
+ async fn root_tree(
+ fs: &TreeFS,
+ show_hidden: bool,
+ node_limit: Option<usize>,
+ level_limit: Option<usize>,
+ ) -> Vec<TreeEntry> {
+ fs.tree_directory("/root", show_hidden, node_limit, level_limit)
+ .await
+ .unwrap()
+ }
+
+ /// Assert tree entry names in order.
+ fn assert_tree_names(entries: &[TreeEntry], expected: &[&str]) {
+ assert_eq!(tree_names(entries), strings(expected));
+ }
+
+ /// Assert tree entry relative paths in order.
+ fn assert_tree_rel_paths(entries: &[TreeEntry], expected: &[&str]) {
+ assert_eq!(tree_rel_paths(entries), strings(expected));
+ }
+
+ /// Convert string slices to owned strings for assertions.
+ fn strings(values: &[&str]) -> Vec<String> {
+ values.iter().map(|value| value.to_string()).collect()
+ }
+
#[tokio::test]
async fn test_tree_empty_dir() {
let fs = TreeFS::default().with_dir_entries("/root", vec![]);
- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+ let entries = root_tree(&fs, false, None, None).await;
assert!(entries.is_empty(), "empty dir returns empty vec");
}
@@ -802,11 +830,11 @@ mod tests {
vec![("a.txt", false), ("b.txt", false), ("c.txt", false)],
);
- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+ let entries = root_tree(&fs, false, None, None).await;
assert_eq!(entries.len(), 3);
- assert_eq!(tree_names(&entries), vec!["a.txt", "b.txt", "c.txt"]);
- assert_eq!(tree_rel_paths(&entries), vec!["a.txt", "b.txt", "c.txt"]);
+ assert_tree_names(&entries, &["a.txt", "b.txt", "c.txt"]);
+ assert_tree_rel_paths(&entries, &["a.txt", "b.txt", "c.txt"]);
}
#[tokio::test]
@@ -815,11 +843,11 @@ mod tests {
.with_dir_entries("/root", vec![("a.txt", false), ("sub", true)])
.with_dir_entries("/root/sub", vec![("b.txt", false)]);
- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+ let entries = root_tree(&fs, false, None, None).await;
assert_eq!(entries.len(), 3);
- assert_eq!(tree_names(&entries), vec!["a.txt", "sub", "b.txt"]);
- assert_eq!(tree_rel_paths(&entries), vec!["a.txt", "sub", "sub/b.txt"]);
+ assert_tree_names(&entries, &["a.txt", "sub", "b.txt"]);
+ assert_tree_rel_paths(&entries, &["a.txt", "sub", "sub/b.txt"]);
assert_eq!(
tree_paths(&entries),
vec!["/root/a.txt", "/root/sub", "/root/sub/b.txt"]
@@ -844,23 +872,17 @@ mod tests {
],
);
- let entries = fs
- .tree_directory("/root", false, Some(3), None)
- .await
- .unwrap();
+ let entries = root_tree(&fs, false, Some(3), None).await;
assert_eq!(entries.len(), 3);
- assert_eq!(tree_names(&entries), vec!["a.txt", "b.txt", "c.txt"]);
+ assert_tree_names(&entries, &["a.txt", "b.txt", "c.txt"]);
}
#[tokio::test]
async fn test_tree_node_limit_zero() {
let fs = TreeFS::default().with_dir_entries("/root", vec![("a.txt", false)]);
- let entries = fs
- .tree_directory("/root", false, Some(0), None)
- .await
- .unwrap();
+ let entries = root_tree(&fs, false, Some(0), None).await;
assert!(entries.is_empty(), "node_limit=0 returns empty");
}
@@ -871,10 +893,7 @@ mod tests {
.with_dir_entries("/root", vec![("a.txt", false), ("sub", true)])
.with_dir_entries("/root/sub", vec![("b.txt", false)]);
- let entries = fs
- .tree_directory("/root", false, None, Some(0))
- .await
- .unwrap();
+ let entries = root_tree(&fs, false, None, Some(0)).await;
assert!(
entries.is_empty(),
@@ -888,13 +907,10 @@ mod tests {
.with_dir_entries("/root", vec![("a.txt", false), ("sub", true)])
.with_dir_entries("/root/sub", vec![("b.txt", false)]);
- let entries = fs
- .tree_directory("/root", false, None, Some(1))
- .await
- .unwrap();
+ let entries = root_tree(&fs, false, None, Some(1)).await;
assert_eq!(entries.len(), 2);
- assert_eq!(tree_names(&entries), vec!["a.txt", "sub"]);
+ assert_tree_names(&entries, &["a.txt", "sub"]);
}
#[tokio::test]
@@ -904,7 +920,7 @@ mod tests {
.with_dir_entries("/root/sub", vec![("b.txt", false), ("deep", true)])
.with_dir_entries("/root/sub/deep", vec![("c.txt", false)]);
- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+ let entries = root_tree(&fs, false, None, None).await;
assert_eq!(entries.len(), 5);
assert!(tree_paths(&entries).contains(&"/root/sub/deep/c.txt".to_string()));
@@ -917,7 +933,7 @@ mod tests {
vec![("a.txt", false), (".hidden", false), ("visible", true)],
);
- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+ let entries = root_tree(&fs, false, None, None).await;
let names = tree_names(&entries);
assert!(names.contains(&"a.txt".to_string()));
@@ -931,10 +947,10 @@ mod tests {
.with_dir_entries("/root", vec![(".hidden_dir", true)])
.with_dir_entries("/root/.hidden_dir", vec![("secret.txt", false)]);
- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+ let entries = root_tree(&fs, false, None, None).await;
assert_eq!(entries.len(), 2);
- assert_eq!(tree_names(&entries), vec![".hidden_dir", "secret.txt"]);
+ assert_tree_names(&entries, &[".hidden_dir", "secret.txt"]);
}
#[tokio::test]
@@ -944,7 +960,7 @@ mod tests {
vec![("a.txt", false), (".hidden", false), (".hidden_dir", true)],
);
- let entries = fs.tree_directory("/root", true, None, None).await.unwrap();
+ let entries = root_tree(&fs, true, None, None).await;
let names = tree_names(&entries);
assert!(names.contains(&"a.txt".to_string()));
@@ -1002,7 +1018,7 @@ mod tests {
.with_dir_entries("/root/sub", vec![("b.txt", false)])
.with_dir_entries("/root/.hidden_dir", vec![("secret.txt", false)]);
- let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+ let entries = root_tree(&fs, false, None, None).await;
assert_eq!(entries.len(), 5);
let names = tree_names(&entries);
diff --git a/crates/ragfs/src/core/mod.rs b/crates/ragfs/src/core/mod.rs
index cd51f2c0a..b7469ce8b 100644
--- a/crates/ragfs/src/core/mod.rs
+++ b/crates/ragfs/src/core/mod.rs
@@ -13,24 +13,30 @@ pub mod encryption_wrapper;
pub mod errors;
pub mod filesystem;
pub mod mountable;
+pub mod multibackend_wrapper;
pub mod plugin;
pub mod stats;
pub mod stats_wrapper;
pub mod types;
// Re-export commonly used types
+pub use crate::multibackend::{FsContextResolver, MetaStateStore};
pub use builder::{
- build_default_stack, register_builtin_plugins, EncryptionConfig, RagfsConfig, RagfsStack,
+ build_default_stack, register_builtin_plugins, EncryptionConfig as BuilderEncryptionConfig,
+ RagfsConfig, RagfsStack,
};
pub use context::{FsContext, FsContextInner, FsContextView, FS_CTX};
pub use encryption_wrapper::EncryptionWrappedFS;
pub use errors::{Error, Result};
pub use filesystem::FileSystem;
pub use mountable::MountableFS;
+pub use multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS, SyncMode};
pub use plugin::{HealthStatus, PluginRegistry, ServicePlugin};
pub use stats::{FilesystemStats, FsOperation, OperationStats, OperationTimer, StatsCollector};
pub use stats_wrapper::StatsWrappedFS;
pub use types::{
- ConfigParameter, ConfigValue, FileInfo, GrepMatch, GrepResult, PluginConfig, TreeEntry,
- WriteFlag,
+ BackendItemConfig, BackendRole, BackendSyncState, BackendsConfig, ConfigParameter, ConfigValue,
+ EncryptionConfig, FileInfo, GrepMatch, GrepResult, OperationItemConfig, PluginConfig,
+ RedirectEntry, RedirectMeta, RedirectPolicy, SyncLogEntry, SyncLogMeta, SyncOp, SyncType,
+ TreeEntry, WriteFlag,
};
diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs
index 1262365f8..a1a922f69 100644
--- a/crates/ragfs/src/core/mountable.rs
+++ b/crates/ragfs/src/core/mountable.rs
@@ -6,16 +6,27 @@
use async_trait::async_trait;
use radix_trie::{Trie, TrieCommon};
+use serde_json::Value;
use std::collections::HashMap;
use std::sync::Arc;
use tokio::sync::RwLock;
+use tracing::warn;
+use crate::multibackend::factory::build_multi_write_fs;
+use crate::multibackend::types::MultiBackendBuildContext;
+use crate::shape::validate::ensure_backend_shape;
+
+/// Python-side transaction path-lock marker — hidden in all listing/grep operations.
+const PATH_LOCK_FILE: &str = ".path.ovlock";
+
+use super::encryption_wrapper::EncryptionWrappedFS;
use super::errors::{Error, Result};
use super::filesystem::FileSystem;
+use super::multibackend_wrapper::MultiWriteWrappedFS;
use super::plugin::ServicePlugin;
use super::stats::{FilesystemStats, StatsCollector};
use super::stats_wrapper::StatsWrappedFS;
-use super::types::{FileInfo, GrepResult, PluginConfig, TreeEntry, WriteFlag};
+use super::types::{BackendsConfig, FileInfo, GrepResult, PluginConfig, TreeEntry, WriteFlag};
/// Information about a mounted filesystem
#[derive(Clone)]
@@ -26,6 +37,10 @@ struct MountInfo {
/// The filesystem instance (wrapped with statistics)
fs: Arc<dyn FileSystem>,
+ /// The raw plugin backend before encryption wrapping (for read_raw/write_raw).
+ /// None for multi-write mounts.
+ raw_fs: Option<Arc<dyn FileSystem>>,
+
/// The statistics collector for this mount
stats: Arc<StatsCollector>,
@@ -33,6 +48,8 @@ struct MountInfo {
plugin_name: String,
}
+const DEFAULT_RAW_COPY_CHUNK_SIZE: usize = 8 * 1024 * 1024;
+
/// MountableFS routes filesystem operations to mounted plugins
///
/// This is the core component that allows multiple filesystem implementations
@@ -44,17 +61,92 @@ pub struct MountableFS {
/// Plugin registry for creating new filesystem instances
registry: Arc<RwLock<HashMap<String, Arc<dyn ServicePlugin>>>>,
+
+ /// Global encryption config (root_key + provider_type), shared across all mounts.
+ /// When set, single-backend mounts are wrapped with EncryptionWrappedFS;
+ /// multi-write mounts use it for per-backend encryption decisions.
+ encryption_root_key: RwLock<Option<[u8; 32]>>,
+ encryption_provider_type: RwLock<Option<u8>>,
}
impl MountableFS {
+ /// Return the raw backend for one mounted path when raw access is supported.
+ fn raw_backend_for_mount<'a>(
+ mount_info: &'a MountInfo,
+ op_name: &str,
+ ) -> Result<&'a Arc<dyn FileSystem>> {
+ if Self::as_multiwrite(&mount_info.fs).is_some() {
+ return Err(Error::invalid_operation(format!(
+ "{} is not supported for multi-write mounts",
+ op_name
+ )));
+ }
+ mount_info.raw_fs.as_ref().ok_or_else(|| {
+ Error::internal(format!(
+ "raw backend is unavailable for mount '{}'",
+ mount_info.path
+ ))
+ })
+ }
+
+ /// Try to unwrap a mounted filesystem stack to the underlying multi-write wrapper.
+ fn as_multiwrite(fs: &Arc<dyn FileSystem>) -> Option<&MultiWriteWrappedFS> {
+ let any = fs.as_ref() as &dyn std::any::Any;
+ if let Some(mw) = any.downcast_ref::<MultiWriteWrappedFS>() {
+ return Some(mw);
+ }
+ if let Some(stats) = any.downcast_ref::<StatsWrappedFS>() {
+ return Self::as_multiwrite(stats.inner_fs());
+ }
+ if let Some(enc) = any.downcast_ref::<EncryptionWrappedFS>() {
+ return Self::as_multiwrite(enc.inner_fs());
+ }
+ None
+ }
+
+ /// Query multi-write sync status for a mounted path.
+ pub async fn system_sync_status(&self, path: &str) -> Result<Value> {
+ let (mount_info, rel_path) = self.find_mount(path).await?;
+ let multiwrite = Self::as_multiwrite(&mount_info.fs)
+ .ok_or_else(|| Error::invalid_operation("mounted filesystem is not multi-write"))?;
+ multiwrite.system_sync_status(&rel_path).await
+ }
+
+ /// Manually retry pending multi-write sync operations under a mounted path.
+ pub async fn system_sync_retry(&self, path: &str) -> Result<Value> {
+ let (mount_info, rel_path) = self.find_mount(path).await?;
+ let multiwrite = Self::as_multiwrite(&mount_info.fs)
+ .ok_or_else(|| Error::invalid_operation("mounted filesystem is not multi-write"))?;
+ multiwrite.system_sync_retry(&rel_path).await
+ }
+
/// Create a new MountableFS
pub fn new() -> Self {
Self {
mounts: Arc::new(RwLock::new(Trie::new())),
registry: Arc::new(RwLock::new(HashMap::new())),
+ encryption_root_key: RwLock::new(None),
+ encryption_provider_type: RwLock::new(None),
}
}
+ /// Set the global encryption configuration for per-mount encryption wrapping.
+ pub async fn set_encryption_config(
+ &self,
+ root_key: Option<[u8; 32]>,
+ provider_type: Option<u8>,
+ ) {
+ *self.encryption_root_key.write().await = root_key;
+ *self.encryption_provider_type.write().await = provider_type;
+ }
+
+ /// Get the encryption root key and provider type.
+ async fn get_encryption_config(&self) -> (Option<[u8; 32]>, Option<u8>) {
+ let rk = *self.encryption_root_key.read().await;
+ let pt = *self.encryption_provider_type.read().await;
+ (rk, pt)
+ }
+
/// Register a plugin
///
/// # Arguments
@@ -99,17 +191,59 @@ impl MountableFS {
// Validate configuration
plugin.validate(&config).await?;
- // Initialize filesystem
- let fs = plugin.initialize(config.clone()).await?;
+ // Branch: multi-write vs single backend
+ let (inner_fs, raw_fs): (Arc<dyn FileSystem>, Option<Arc<dyn FileSystem>>) = match &config
+ .backups
+ {
+ None => {
+ // Single backend: initialize plugin, optionally wrap with encryption.
+ // Control plugins (queuefs, serverinfofs) are never encrypted.
+ let raw = plugin.initialize(config.clone()).await?;
+ let raw_arc: Arc<dyn FileSystem> = Arc::from(raw);
+ let is_control_plugin = matches!(config.name.as_str(), "queuefs" | "serverinfofs");
+ let (enc_root_key, enc_provider_type) = self.get_encryption_config().await;
+ if !is_control_plugin {
+ ensure_backend_shape(
+ &raw_arc,
+ &config.name,
+ enc_root_key.is_some() && enc_provider_type.is_some(),
+ enc_provider_type,
+ enc_root_key,
+ )
+ .await?;
+ }
+ let inner: Arc<dyn FileSystem> = if is_control_plugin {
+ raw_arc.clone()
+ } else {
+ match (enc_root_key, enc_provider_type) {
+ (Some(rk), Some(pt)) => {
+ Arc::new(EncryptionWrappedFS::new(raw_arc.clone(), rk, pt))
+ }
+ _ => raw_arc.clone(),
+ }
+ };
+ (inner, Some(raw_arc))
+ }
+ Some(bc) => {
+ // Multi-write: build MultiWriteWrappedFS
+ warn!(
+ "multi-write metadata locking is process-local only; multi-instance shared-primary mounts are not safe"
+ );
+ let mw = self.build_multi_write_fs(&config, bc).await?;
+ let arc: Arc<dyn FileSystem> = Arc::new(mw);
+ (arc, None)
+ }
+ };
// Wrap with statistics
- let wrapped_fs = StatsWrappedFS::new(fs);
+ let wrapped_fs = StatsWrappedFS::with_arc(inner_fs);
let stats_collector = wrapped_fs.stats_collector();
// Add to mounts
let mount_info = MountInfo {
path: normalized_path.clone(),
fs: Arc::from(Box::new(wrapped_fs) as Box<dyn FileSystem>),
+ raw_fs,
stats: stats_collector,
plugin_name: config.name.clone(),
};
@@ -120,6 +254,31 @@ impl MountableFS {
Ok(())
}
+ /// Build a MultiWriteWrappedFS from primary + backup configurations.
+ ///
+ /// Steps:
+ /// 1. Initialize primary backend via registry, optionally wrap with EncryptionWrappedFS
+ /// 2. Initialize each backup backend via registry, optionally wrap with EncryptionWrappedFS
+ /// 3. Validate: unique names, primary has no operations, redirect targets exist
+ /// 4. Assemble MultiWriteWrappedFS
+ async fn build_multi_write_fs(
+ &self,
+ config: &PluginConfig,
+ bc: &BackendsConfig,
+ ) -> Result<MultiWriteWrappedFS> {
+ let (enc_root_key, enc_provider_type) = self.get_encryption_config().await;
+ build_multi_write_fs(
+ &self.registry,
+ config,
+ bc,
+ MultiBackendBuildContext {
+ enc_root_key,
+ enc_provider_type,
+ },
+ )
+ .await
+ }
+
/// Unmount a filesystem at the specified path
///
/// # Arguments
@@ -129,6 +288,17 @@ impl MountableFS {
/// * `Error::MountPointNotFound` - If no filesystem is mounted at this path
pub async fn unmount(&self, path: &str) -> Result<()> {
let normalized_path = normalize_path(path);
+ let mount_info = {
+ let mounts = self.mounts.read().await;
+ mounts
+ .get(&normalized_path)
+ .cloned()
+ .ok_or_else(|| Error::MountPointNotFound(normalized_path.clone()))?
+ };
+
+ if let Some(multiwrite) = Self::as_multiwrite(&mount_info.fs) {
+ multiwrite.shutdown().await?;
+ }
let mut mounts = self.mounts.write().await;
if mounts.remove(&normalized_path).is_none() {
@@ -184,6 +354,95 @@ impl MountableFS {
result
}
+ /// Read raw bytes from the underlying plugin backend, bypassing the encryption layer.
+ ///
+ /// Used by tests to verify ciphertext on disk and by cp/persist for verbatim blob copies.
+ pub async fn read_raw(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {
+ let (mount_info, rel_path) = self.find_mount(path).await?;
+ let raw = Self::raw_backend_for_mount(&mount_info, "read_raw")?;
+ raw.read(&rel_path, offset, size).await
+ }
+
+ /// Write raw bytes to the underlying plugin backend, bypassing the encryption layer.
+ ///
+ /// Counterpart to `read_raw` for cp/persist verbatim blob copies.
+ pub async fn write_raw(&self, path: &str, data: &[u8], flags: WriteFlag) -> Result<u64> {
+ let (mount_info, rel_path) = self.find_mount(path).await?;
+ let raw = Self::raw_backend_for_mount(&mount_info, "write_raw")?;
+ raw.write(&rel_path, data, 0, flags).await
+ }
+
+ /// Copy one file within the same mounted filesystem while preserving raw bytes when possible.
+ pub async fn copy_within_mount(&self, src_path: &str, dst_path: &str) -> Result<bool> {
+ let (src_mount, src_rel_path) = self.find_mount(src_path).await?;
+ let (dst_mount, dst_rel_path) = self.find_mount(dst_path).await?;
+
+ if src_mount.path != dst_mount.path {
+ return Ok(false);
+ }
+
+ if normalize_path(&src_rel_path) == normalize_path(&dst_rel_path) {
+ return Ok(true);
+ }
+
+ if let Some(multiwrite) = Self::as_multiwrite(&src_mount.fs) {
+ return multiwrite
+ .copy_within_primary(&src_rel_path, &dst_rel_path)
+ .await;
+ }
+
+ let Some(raw_backend) = src_mount.raw_fs.clone() else {
+ return Ok(false);
+ };
+
+ Self::copy_raw_within_mount(raw_backend, &src_rel_path, &dst_rel_path).await?;
+ Ok(true)
+ }
+
+ /// Copy one file within one raw backend in bounded-size chunks.
+ async fn copy_raw_within_mount(
+ raw_backend: Arc<dyn FileSystem>,
+ src_rel_path: &str,
+ dst_rel_path: &str,
+ ) -> Result<()> {
+ let source_info = raw_backend.stat(src_rel_path).await?;
+ if source_info.is_dir {
+ return Err(Error::IsADirectory(src_rel_path.to_string()));
+ }
+
+ raw_backend.ensure_parent_dirs(dst_rel_path, 0o755).await?;
+
+ if source_info.size == 0 {
+ raw_backend
+ .write(dst_rel_path, &[], 0, WriteFlag::Create)
+ .await?;
+ return Ok(());
+ }
+
+ let mut offset = 0u64;
+ while offset < source_info.size {
+ let chunk_len = (source_info.size - offset).min(DEFAULT_RAW_COPY_CHUNK_SIZE as u64);
+ let chunk = raw_backend.read(src_rel_path, offset, chunk_len).await?;
+ let flag = if offset == 0 {
+ WriteFlag::Create
+ } else {
+ WriteFlag::None
+ };
+ raw_backend
+ .write(dst_rel_path, &chunk, offset, flag)
+ .await?;
+ offset = offset.saturating_add(chunk.len() as u64);
+ if chunk.is_empty() {
+ return Err(Error::internal(format!(
+ "raw backend returned empty chunk while copying '{}' -> '{}'",
+ src_rel_path, dst_rel_path
+ )));
+ }
+ }
+
+ Ok(())
+ }
+
/// Find the mount point for a given path
///
/// # Arguments
@@ -292,7 +551,9 @@ impl FileSystem for MountableFS {
async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {
let (mount_info, rel_path) = self.find_mount(path).await?;
- mount_info.fs.read_dir(&rel_path).await
+ let mut entries = mount_info.fs.read_dir(&rel_path).await?;
+ entries.retain(|e| e.name != PATH_LOCK_FILE);
+ Ok(entries)
}
async fn stat(&self, path: &str) -> Result<FileInfo> {
@@ -360,7 +621,7 @@ impl FileSystem for MountableFS {
},
};
- mount_info
+ let mut result = mount_info
.fs
.grep(
&rel_path,
@@ -371,7 +632,17 @@ impl FileSystem for MountableFS {
exclude_rel.as_deref(),
level_limit,
)
- .await
+ .await?;
+
+ // Filter out path-lock markers.
+ result.matches.retain(|m| {
+ m.file
+ .rsplit('/')
+ .next()
+ .map_or(true, |name| name != PATH_LOCK_FILE)
+ });
+ result.count = result.matches.len();
+ Ok(result)
}
async fn tree_directory(
@@ -404,6 +675,14 @@ impl FileSystem for MountableFS {
}
}
+ // Filter out path-lock markers.
+ entries.retain(|e| {
+ e.path
+ .rsplit('/')
+ .next()
+ .map_or(true, |name| name != PATH_LOCK_FILE)
+ });
+
Ok(entries)
}
}
@@ -411,8 +690,51 @@ impl FileSystem for MountableFS {
#[cfg(test)]
mod tests {
use super::*;
+ use crate::core::BackendItemConfig;
+ use crate::core::RedirectPolicy;
+ use crate::shape::SHAPE_MANIFEST_PATH;
+ use serde_json::Value;
use std::collections::HashMap;
+ /// Helper to create a PluginConfig for tests with new fields defaulted.
+ fn test_config(name: &str, mount_path: &str) -> PluginConfig {
+ PluginConfig::single_backend(name, mount_path, HashMap::new())
+ }
+
+ /// Helper to create a simple multi-write PluginConfig for tests.
+ fn multiwrite_test_config(
+ primary_backend: &str,
+ backup_backend: &str,
+ mount_path: &str,
+ ) -> PluginConfig {
+ PluginConfig {
+ name: primary_backend.to_string(),
+ mount_path: mount_path.to_string(),
+ params: HashMap::new(),
+ backups: Some(BackendsConfig {
+ sync_type: "async".to_string(),
+ write_ack_count: None,
+ write_ack_timeout_ms: None,
+ write_concurrency: None,
+ retry_interval_ms: None,
+ retry_backoff_base_ms: None,
+ retry_max_retries_per_round: None,
+ retry_quarantine_after_failures: None,
+ read_probe_cache_ttl_ms: None,
+ items: vec![BackendItemConfig {
+ name: "backup1".to_string(),
+ backend: backup_backend.to_string(),
+ params: Value::Null,
+ timeout: None,
+ encryption: None,
+ operations: None,
+ excludes: None,
+ }],
+ }),
+ ..PluginConfig::default()
+ }
+ }
+
// Mock filesystem for testing
struct MockFS {
name: String,
@@ -453,7 +775,13 @@ mod tests {
Ok(())
}
- async fn read(&self, _path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {
+ async fn read(&self, path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {
+ if path == SHAPE_MANIFEST_PATH
+ || path.ends_with(".redirect.json")
+ || path.ends_with(".sync_log.json")
+ {
+ return Err(Error::not_found(path));
+ }
Ok(self.name.as_bytes().to_vec())
}
@@ -562,6 +890,24 @@ mod tests {
}
}
+ /// Create a MountableFS with one registered and mounted mock plugin.
+ async fn mounted_mock(name: &str, mount_path: &str) -> MountableFS {
+ let mfs = MountableFS::new();
+ mfs.register_plugin(MockPlugin::new(name)).await;
+ mfs.mount(test_config(name, mount_path)).await.unwrap();
+ mfs
+ }
+
+ /// Create a MountableFS with two mounted mock plugins.
+ async fn mounted_two_mocks() -> MountableFS {
+ let mfs = MountableFS::new();
+ for (name, mount_path) in [("mock1", "/fs1"), ("mock2", "/fs2")] {
+ mfs.register_plugin(MockPlugin::new(name)).await;
+ mfs.mount(test_config(name, mount_path)).await.unwrap();
+ }
+ mfs
+ }
+
#[test]
fn test_normalize_path() {
assert_eq!(normalize_path("/test"), "/test");
@@ -586,11 +932,7 @@ mod tests {
mfs.register_plugin(MockPlugin::new("mock")).await;
// Mount filesystem
- let config = PluginConfig {
- name: "mock".to_string(),
- mount_path: "/mock".to_string(),
- params: HashMap::new(),
- };
+ let config = test_config("mock", "/mock");
assert!(mfs.mount(config).await.is_ok());
@@ -613,11 +955,7 @@ mod tests {
let mfs = MountableFS::new();
mfs.register_plugin(MockPlugin::new("mock")).await;
- let config = PluginConfig {
- name: "mock".to_string(),
- mount_path: "/mock".to_string(),
- params: HashMap::new(),
- };
+ let config = test_config("mock", "/mock");
// First mount should succeed
assert!(mfs.mount(config.clone()).await.is_ok());
@@ -639,16 +977,7 @@ mod tests {
#[tokio::test]
async fn test_filesystem_operations() {
- let mfs = MountableFS::new();
- mfs.register_plugin(MockPlugin::new("mock")).await;
-
- let config = PluginConfig {
- name: "mock".to_string(),
- mount_path: "/mock".to_string(),
- params: HashMap::new(),
- };
-
- mfs.mount(config).await.unwrap();
+ let mfs = mounted_mock("mock", "/mock").await;
// Test read operation
let data = mfs.read("/mock/test.txt", 0, 0).await.unwrap();
@@ -668,25 +997,7 @@ mod tests {
#[tokio::test]
async fn test_path_routing() {
- let mfs = MountableFS::new();
- mfs.register_plugin(MockPlugin::new("mock1")).await;
- mfs.register_plugin(MockPlugin::new("mock2")).await;
-
- // Mount two filesystems
- let config1 = PluginConfig {
- name: "mock1".to_string(),
- mount_path: "/fs1".to_string(),
- params: HashMap::new(),
- };
-
- let config2 = PluginConfig {
- name: "mock2".to_string(),
- mount_path: "/fs2".to_string(),
- params: HashMap::new(),
- };
-
- mfs.mount(config1).await.unwrap();
- mfs.mount(config2).await.unwrap();
+ let mfs = mounted_two_mocks().await;
// Test routing to different filesystems
let data1 = mfs.read("/fs1/file.txt", 0, 0).await.unwrap();
@@ -698,24 +1009,7 @@ mod tests {
#[tokio::test]
async fn test_rename_across_mounts_error() {
- let mfs = MountableFS::new();
- mfs.register_plugin(MockPlugin::new("mock1")).await;
- mfs.register_plugin(MockPlugin::new("mock2")).await;
-
- let config1 = PluginConfig {
- name: "mock1".to_string(),
- mount_path: "/fs1".to_string(),
- params: HashMap::new(),
- };
-
- let config2 = PluginConfig {
- name: "mock2".to_string(),
- mount_path: "/fs2".to_string(),
- params: HashMap::new(),
- };
-
- mfs.mount(config1).await.unwrap();
- mfs.mount(config2).await.unwrap();
+ let mfs = mounted_two_mocks().await;
// Try to rename across different mounts - should fail
let result = mfs.rename("/fs1/file.txt", "/fs2/file.txt").await;
@@ -727,16 +1021,7 @@ mod tests {
async fn test_concurrent_operations() {
use tokio::task;
- let mfs = Arc::new(MountableFS::new());
- mfs.register_plugin(MockPlugin::new("mock")).await;
-
- let config = PluginConfig {
- name: "mock".to_string(),
- mount_path: "/mock".to_string(),
- params: HashMap::new(),
- };
-
- mfs.mount(config).await.unwrap();
+ let mfs = Arc::new(mounted_mock("mock", "/mock").await);
// Spawn multiple concurrent read operations
let mut handles = vec![];
@@ -774,11 +1059,7 @@ mod tests {
for i in 0..5 {
let mfs_clone = Arc::clone(&mfs);
let handle = task::spawn(async move {
- let config = PluginConfig {
- name: format!("mock{}", i),
- mount_path: format!("/mock{}", i),
- params: HashMap::new(),
- };
+ let config = test_config(&format!("mock{}", i), &format!("/mock{}", i));
mfs_clone.mount(config).await
});
handles.push(handle);
@@ -816,15 +1097,7 @@ mod tests {
#[tokio::test]
async fn test_grep_routes_to_plugin() {
- let mfs = MountableFS::new();
- mfs.register_plugin(MockPlugin::new("mock")).await;
-
- let config = PluginConfig {
- name: "mock".to_string(),
- mount_path: "/mock".to_string(),
- params: HashMap::new(),
- };
- mfs.mount(config).await.unwrap();
+ let mfs = mounted_mock("mock", "/mock").await;
let result = mfs
.grep("/mock/a.txt", "foo", false, false, None, None, None)
@@ -842,12 +1115,10 @@ mod tests {
TreeEntry {
path: path.to_string(),
rel_path: rel_path.to_string(),
- info: FileInfo {
- name: name.to_string(),
- size: if is_dir { 0 } else { 100 },
- mode: if is_dir { 0o755 } else { 0o644 },
- mod_time: std::time::UNIX_EPOCH,
- is_dir,
+ info: if is_dir {
+ FileInfo::new_dir(name.to_string(), 0o755)
+ } else {
+ FileInfo::new_file(name.to_string(), 100, 0o644)
},
extra: HashMap::new(),
}
@@ -875,13 +1146,7 @@ mod tests {
)],
);
mfs.register_plugin(plugin).await;
- mfs.mount(PluginConfig {
- name: "rewrite".to_string(),
- mount_path: "/rewrite".to_string(),
- params: HashMap::new(),
- })
- .await
- .unwrap();
+ mfs.mount(test_config("rewrite", "/rewrite")).await.unwrap();
let result = mfs
.tree_directory("/rewrite/sub", false, None, None)
@@ -900,13 +1165,9 @@ mod tests {
vec![make_tree_entry("/a/b/c.txt", "a/b/c.txt", "c.txt", false)],
);
mfs.register_plugin(plugin).await;
- mfs.mount(PluginConfig {
- name: "relpath".to_string(),
- mount_path: "/local/test_account".to_string(),
- params: HashMap::new(),
- })
- .await
- .unwrap();
+ mfs.mount(test_config("relpath", "/local/test_account"))
+ .await
+ .unwrap();
let result = mfs
.tree_directory("/local/test_account/a", false, None, None)
@@ -916,4 +1177,226 @@ mod tests {
assert_eq!(result[0].path, "/local/test_account/a/b/c.txt");
assert_eq!(result[0].rel_path, "a/b/c.txt");
}
+
+ #[tokio::test]
+ async fn test_build_multi_write_fs_rejects_primary_encryption_disable() {
+ let mfs = MountableFS::new();
+ mfs.register_plugin(MockPlugin::new("primary")).await;
+ mfs.register_plugin(MockPlugin::new("backupfs")).await;
+ mfs.set_encryption_config(Some([7u8; 32]), Some(1)).await;
+
+ let backups = BackendsConfig {
+ sync_type: "async".to_string(),
+ write_ack_count: None,
+ write_ack_timeout_ms: None,
+ write_concurrency: None,
+ retry_interval_ms: None,
+ retry_backoff_base_ms: None,
+ retry_max_retries_per_round: None,
+ retry_quarantine_after_failures: None,
+ read_probe_cache_ttl_ms: None,
+ items: vec![BackendItemConfig {
+ name: "backup1".to_string(),
+ backend: "backupfs".to_string(),
+ params: serde_json::Value::Null,
+ timeout: None,
+ encryption: None,
+ operations: None,
+ excludes: None,
+ }],
+ };
+ let config = PluginConfig {
+ name: "primary".to_string(),
+ mount_path: "/local".to_string(),
+ params: HashMap::new(),
+ backups: Some(backups.clone()),
+ server_encryption_enabled: true,
+ ..PluginConfig::default()
+ };
+
+ let result = mfs.build_multi_write_fs(&config, &backups).await;
+ assert!(result.is_err());
+ assert!(matches!(result.err(), Some(Error::Config(_))));
+ }
+
+ #[tokio::test]
+ async fn test_build_multi_write_fs_rejects_exclude_target_field() {
+ let mfs = MountableFS::new();
+ mfs.register_plugin(MockPlugin::new("primary")).await;
+ mfs.register_plugin(MockPlugin::new("backupfs")).await;
+
+ let backups = BackendsConfig {
+ sync_type: "async".to_string(),
+ write_ack_count: None,
+ write_ack_timeout_ms: None,
+ write_concurrency: None,
+ retry_interval_ms: None,
+ retry_backoff_base_ms: None,
+ retry_max_retries_per_round: None,
+ retry_quarantine_after_failures: None,
+ read_probe_cache_ttl_ms: None,
+ items: vec![BackendItemConfig {
+ name: "backup1".to_string(),
+ backend: "backupfs".to_string(),
+ params: serde_json::Value::Null,
+ timeout: None,
+ encryption: None,
+ operations: None,
+ excludes: Some(vec![RedirectPolicy::FileExtensionPolicy {
+ extensions: vec!["\\.tmp$".to_string()],
+ target: Some(vec!["should-not-exist".to_string()]),
+ }]),
+ }],
+ };
+ let config = PluginConfig {
+ name: "primary".to_string(),
+ mount_path: "/local".to_string(),
+ params: HashMap::new(),
+ backups: Some(backups.clone()),
+ ..PluginConfig::default()
+ };
+
+ let result = mfs.build_multi_write_fs(&config, &backups).await;
+ assert!(result.is_err());
+ assert!(matches!(result.err(), Some(Error::Config(_))));
+ }
+
+ #[tokio::test]
+ async fn test_build_multi_write_fs_rejects_reserved_primary_backup_name() {
+ let mfs = MountableFS::new();
+ mfs.register_plugin(MockPlugin::new("primary")).await;
+ mfs.register_plugin(MockPlugin::new("backupfs")).await;
+
+ let backups = BackendsConfig {
+ sync_type: "async".to_string(),
+ write_ack_count: None,
+ write_ack_timeout_ms: None,
+ write_concurrency: None,
+ retry_interval_ms: None,
+ retry_backoff_base_ms: None,
+ retry_max_retries_per_round: None,
+ retry_quarantine_after_failures: None,
+ read_probe_cache_ttl_ms: None,
+ items: vec![BackendItemConfig {
+ name: "primary".to_string(),
+ backend: "backupfs".to_string(),
+ params: serde_json::Value::Null,
+ timeout: None,
+ encryption: None,
+ operations: None,
+ excludes: None,
+ }],
+ };
+ let config = PluginConfig {
+ name: "primary".to_string(),
+ mount_path: "/local".to_string(),
+ params: HashMap::new(),
+ backups: Some(backups.clone()),
+ ..PluginConfig::default()
+ };
+
+ let result = mfs.build_multi_write_fs(&config, &backups).await;
+ assert!(result.is_err());
+ assert!(matches!(result.err(), Some(Error::Config(_))));
+ }
+
+ #[tokio::test]
+ async fn test_single_backend_mount_wraps_stats_outside_encryption() {
+ let mfs = MountableFS::new();
+ mfs.register_plugin(MockPlugin::new("mock")).await;
+ mfs.set_encryption_config(Some([9u8; 32]), Some(1)).await;
+
+ mfs.mount(test_config("mock", "/mock")).await.unwrap();
+
+ let mounts = mfs.mounts.read().await;
+ let mount_info = mounts.get("/mock").expect("mounted entry should exist");
+ let wrapped = (mount_info.fs.as_ref() as &dyn std::any::Any)
+ .downcast_ref::<StatsWrappedFS>()
+ .expect("stats wrapper should be outermost");
+ assert!(
+ (wrapped.inner_fs().as_ref() as &dyn std::any::Any)
+ .downcast_ref::<EncryptionWrappedFS>()
+ .is_some(),
+ "single-backend encrypted mount should place encryption under stats"
+ );
+ }
+
+ #[tokio::test]
+ async fn test_multiwrite_mount_wraps_stats_outside_multiwrite() {
+ let mfs = MountableFS::new();
+ mfs.register_plugin(MockPlugin::new("primary")).await;
+ mfs.register_plugin(MockPlugin::new("backupfs")).await;
+
+ mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))
+ .await
+ .unwrap();
+
+ let mounts = mfs.mounts.read().await;
+ let mount_info = mounts.get("/local").expect("mounted entry should exist");
+ let wrapped = (mount_info.fs.as_ref() as &dyn std::any::Any)
+ .downcast_ref::<StatsWrappedFS>()
+ .expect("stats wrapper should be outermost");
+ assert!(
+ (wrapped.inner_fs().as_ref() as &dyn std::any::Any)
+ .downcast_ref::<MultiWriteWrappedFS>()
+ .is_some(),
+ "multi-write mount should place MultiWriteWrappedFS directly under stats"
+ );
+ }
+
+ #[tokio::test]
+ async fn test_multiwrite_raw_access_is_rejected() {
+ let mfs = MountableFS::new();
+ mfs.register_plugin(MockPlugin::new("primary")).await;
+ mfs.register_plugin(MockPlugin::new("backupfs")).await;
+
+ mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))
+ .await
+ .unwrap();
+
+ let read_err = mfs.read_raw("/local/data.txt", 0, 0).await.unwrap_err();
+ assert!(matches!(read_err, Error::InvalidOperation(_)));
+
+ let write_err = mfs
+ .write_raw("/local/data.txt", b"raw", WriteFlag::Create)
+ .await
+ .unwrap_err();
+ assert!(matches!(write_err, Error::InvalidOperation(_)));
+ }
+
+ #[tokio::test]
+ async fn test_mountable_multiwrite_admin_smoke() {
+ let mfs = MountableFS::new();
+ mfs.register_plugin(MockPlugin::new("primary")).await;
+ mfs.register_plugin(MockPlugin::new("backupfs")).await;
+
+ mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))
+ .await
+ .unwrap();
+
+ let status = crate::core::FS_CTX
+ .scope(
+ Arc::new(crate::core::context::FsContextInner::new(
+ "acct".to_string(),
+ )),
+ async { mfs.system_sync_status("/local").await.unwrap() },
+ )
+ .await;
+ assert_eq!(status["path"], "/");
+ assert_eq!(status["entry_count"], 0);
+ assert_eq!(status["pending_target_count"], 0);
+ assert_eq!(status["capabilities"]["multi_instance_safe"], false);
+
+ let retry = crate::core::FS_CTX
+ .scope(
+ Arc::new(crate::core::context::FsContextInner::new(
+ "acct".to_string(),
+ )),
+ async { mfs.system_sync_retry("/local").await.unwrap() },
+ )
+ .await;
+ assert_eq!(retry["path"], "/");
+ assert_eq!(retry["retried"], 0);
+ assert_eq!(retry["failed"], 0);
+ }
}
diff --git a/crates/ragfs/src/core/multibackend_wrapper.rs b/crates/ragfs/src/core/multibackend_wrapper.rs
new file mode 100644
index 000000000..932d24fa8
--- /dev/null
+++ b/crates/ragfs/src/core/multibackend_wrapper.rs
@@ -0,0 +1,1673 @@
+//! Multi-write wrapper — routes operations across primary and backup backends.
+//!
+//! Implements `MultiWriteWrappedFS` which handles:
+//! - Write fanout to primary + backup backends (sync/async)
+//! - Read routing with priority-based fallback chain
+//! - Redirect policy evaluation
+//! - Exclude policy filtering
+//! - `.redirect.json` / `.sync_log.json` metadata management
+
+use std::collections::{HashMap, HashSet};
+use std::future::Future;
+use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use regex::Regex;
+use serde_json::Value;
+use tokio::sync::{Mutex, Notify};
+
+use super::context::{FsContext, FS_CTX};
+use super::errors::{Error, Result};
+use super::filesystem::{normalize_prefix_path, relative_match_file, FileSystem};
+use super::types::{
+ BackendRole, BackendSyncState, FileInfo, GrepResult, OperationItemConfig, RedirectEntry,
+ RedirectPolicy, SyncLogEntry, SyncOp, SyncType, TreeEntry, WriteFlag,
+};
+use crate::multibackend::meta::{
+ current_required_ctx, file_name, parent_dir, DefaultFsContextResolver, FsContextResolver,
+ MetaStateStore, PathSerializer, MULTIWRITE_INTERNAL_NAMES,
+};
+
+mod retry;
+mod routing;
+#[cfg(test)]
+mod tests;
+
+/// Default chunk size used when copying file state from primary to backup.
+const DEFAULT_COPY_CHUNK_SIZE: usize = 8 * 1024 * 1024;
+/// Default retry loop interval.
+const DEFAULT_RETRY_INTERVAL: Duration = Duration::from_secs(30);
+/// Default retry backoff base in milliseconds.
+const DEFAULT_RETRY_BACKOFF_BASE_MS: u64 = 1000;
+/// Default maximum retries per file per retry round.
+const DEFAULT_MAX_RETRIES_PER_ROUND: usize = 3;
+/// Default failure threshold before a target is quarantined.
+const DEFAULT_QUARANTINE_AFTER_FAILURES: u32 = 9;
+/// Default wait timeout when shutting down background tasks.
+const DEFAULT_SHUTDOWN_WAIT: Duration = Duration::from_secs(5);
+
+#[derive(Debug)]
+struct SyncFanoutTaskResult {
+ backend: String,
+ result: Result<()>,
+}
+
+#[derive(Clone)]
+enum BackupWriteOp {
+ Replay(SyncOp),
+ WriteFile {
+ data: Arc<Vec<u8>>,
+ offset: u64,
+ flags: WriteFlag,
+ },
+ EnsureParentDirs {
+ mode: u32,
+ },
+}
+
+impl BackupWriteOp {
+ /// Apply this backup-side operation under the provided request context.
+ async fn apply(
+ &self,
+ inner: &Inner,
+ backup: Arc<dyn FileSystem>,
+ path: &str,
+ ctx: &FsContext,
+ ) -> Result<()> {
+ match self {
+ Self::Replay(op) => {
+ op.replay(inner.primary().backend.clone(), backup, path, ctx)
+ .await
+ }
+ Self::WriteFile {
+ data,
+ offset,
+ flags,
+ } => {
+ let data = Arc::clone(data);
+ let offset = *offset;
+ let flags = *flags;
+ FS_CTX
+ .scope(ctx.clone(), async move {
+ backup.ensure_parent_dirs(path, 0o755).await?;
+ backup.write(path, data.as_slice(), offset, flags).await?;
+ Ok(())
+ })
+ .await
+ }
+ Self::EnsureParentDirs { mode } => {
+ let mode = *mode;
+ FS_CTX
+ .scope(ctx.clone(), async move {
+ backup.ensure_parent_dirs(path, mode).await
+ })
+ .await
+ }
+ }
+ }
+}
+
+/// Cloned target data passed to fanout strategies without borrowing `Inner`.
+#[derive(Clone)]
+struct FanoutTarget {
+ name: String,
+ backend: Arc<dyn FileSystem>,
+}
+
+/// Effective sync work item with resolved target backend names.
+pub(crate) struct SyncWorkEntry {
+ pub(crate) file_path: String,
+ pub(crate) entry: SyncLogEntry,
+ pub(crate) targets: Vec<String>,
+}
+
+#[derive(Clone, Copy)]
+enum ReadRouteSource {
+ Backup,
+ Primary,
+ Redirect,
+ Miss,
+}
+
+/// Builder-style sync mode configuration.
+pub enum SyncMode {
+ /// Synchronous fanout requiring backup acknowledgement.
+ Sync {
+ /// Minimum backup acknowledgements required for a successful write.
+ ack_count: usize,
+ /// Maximum time to wait for backup acknowledgements, in milliseconds.
+ timeout_ms: u64,
+ },
+ /// Asynchronous fanout with background retry.
+ Async,
+}
+
+/// A backend entry within the multi-write wrapper.
+pub struct BackendEntry {
+ /// Logical name (globally unique)
+ pub name: String,
+ /// Role: Primary or Backup
+ pub role: BackendRole,
+ /// The backend filesystem handle (may be encrypted)
+ pub backend: Arc<dyn FileSystem>,
+ /// Optional raw backend handle used by primary verbatim copy fast-paths.
+ pub raw_backend: Option<Arc<dyn FileSystem>>,
+ /// Operations this backend participates in (only for Backup)
+ pub operations: Vec<OperationItemConfig>,
+ /// Exclude policies (only for Backup)
+ pub excludes: Vec<RedirectPolicy>,
+}
+
+impl BackendEntry {
+ /// Check if this backend participates in read operations.
+ fn participates_in_read(&self) -> bool {
+ self.operations.iter().any(|op| op.operation == "read")
+ }
+
+ /// Check if this backend participates in write operations.
+ /// Backups default to write-enabled when operations is empty.
+ fn participates_in_write(&self) -> bool {
+ if self.operations.is_empty() {
+ true
+ } else {
+ self.operations.iter().any(|op| op.operation == "write")
+ }
+ }
+
+ /// Get read priority (lower = higher priority). Returns None if not read-enabled.
+ fn read_priority(&self) -> Option<u32> {
+ self.operations
+ .iter()
+ .find(|op| op.operation == "read")
+ .map(|op| op.priority)
+ }
+
+ /// Convert this backend entry into a fanout target.
+ fn fanout_target(&self) -> FanoutTarget {
+ FanoutTarget {
+ name: self.name.clone(),
+ backend: self.backend.clone(),
+ }
+ }
+}
+
+/// File policy trait — shared by redirects and excludes.
+pub trait FilePolicy {
+ /// Check if this policy matches the given file.
+ fn matches(&self, path: &str, size: u64) -> bool;
+}
+
+impl FilePolicy for RedirectPolicy {
+ fn matches(&self, path: &str, size: u64) -> bool {
+ match self {
+ RedirectPolicy::FileOverSizePolicy { max_size_mb, .. } => {
+ let max_bytes = max_size_mb * 1024 * 1024;
+ size > max_bytes
+ }
+ RedirectPolicy::FileExtensionPolicy { extensions, .. } => {
+ let name = file_name(path);
+ extensions.iter().any(|ext_pattern| {
+ if let Ok(re) = Regex::new(ext_pattern) {
+ re.is_match(name)
+ } else {
+ name.ends_with(ext_pattern.as_str())
+ }
+ })
+ }
+ }
+ }
+}
+
+impl SyncOp {
+ /// Replay this operation on one backup backend using the original request semantics.
+ async fn replay(
+ &self,
+ primary: Arc<dyn FileSystem>,
+ backup: Arc<dyn FileSystem>,
+ file_path: &str,
+ ctx: &FsContext,
+ ) -> Result<()> {
+ match self {
+ SyncOp::SyncFile { size } => {
+ let size = *size;
+ copy_current_primary_state(primary, backup, file_path, size, ctx).await
+ }
+ SyncOp::Create => {
+ FS_CTX
+ .scope(ctx.clone(), async { backup.create(file_path).await })
+ .await
+ }
+ SyncOp::Mkdir { mode } => {
+ let mode = *mode;
+ FS_CTX
+ .scope(ctx.clone(), async { backup.mkdir(file_path, mode).await })
+ .await
+ }
+ SyncOp::Remove => {
+ match FS_CTX
+ .scope(ctx.clone(), async { backup.remove(file_path).await })
+ .await
+ {
+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),
+ Err(e) => Err(e),
+ }
+ }
+ SyncOp::RemoveAll => {
+ match FS_CTX
+ .scope(ctx.clone(), async { backup.remove_all(file_path).await })
+ .await
+ {
+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),
+ Err(e) => Err(e),
+ }
+ }
+ SyncOp::Rename { to } => {
+ let to = to.clone();
+ FS_CTX
+ .scope(ctx.clone(), async {
+ backup.ensure_parent_dirs(&to, 0o755).await?;
+ backup.rename(file_path, &to).await
+ })
+ .await
+ }
+ SyncOp::Chmod { mode } => {
+ let mode = *mode;
+ FS_CTX
+ .scope(ctx.clone(), async { backup.chmod(file_path, mode).await })
+ .await
+ }
+ }
+ }
+}
+
+/// Copy one file between two filesystem handles in bounded-size chunks.
+async fn copy_file_state(
+ source: Arc<dyn FileSystem>,
+ source_path: &str,
+ destination: Arc<dyn FileSystem>,
+ destination_path: &str,
+ size: u64,
+ ctx: &FsContext,
+) -> Result<()> {
+ FS_CTX
+ .scope(ctx.clone(), async {
+ destination
+ .ensure_parent_dirs(destination_path, 0o755)
+ .await?;
+ if size == 0 {
+ if destination.exists(destination_path).await {
+ return destination.truncate(destination_path, 0).await;
+ }
+ return destination.create(destination_path).await;
+ }
+
+ let mut offset = 0u64;
+ while offset < size {
+ let chunk_len = (size - offset).min(DEFAULT_COPY_CHUNK_SIZE as u64);
+ let chunk = source.read(source_path, offset, chunk_len).await?;
+ let flag = if offset == 0 {
+ WriteFlag::Create
+ } else {
+ WriteFlag::None
+ };
+ destination
+ .write(destination_path, &chunk, offset, flag)
+ .await?;
+ offset = offset.saturating_add(chunk.len() as u64);
+ if chunk.is_empty() {
+ return Err(Error::internal(format!(
+ "source returned empty chunk while copying '{}' -> '{}'",
+ source_path, destination_path
+ )));
+ }
+ }
+ Ok(())
+ })
+ .await
+}
+
+/// Copy the current file state from primary to backup in bounded-size chunks.
+async fn copy_current_primary_state(
+ primary: Arc<dyn FileSystem>,
+ backup: Arc<dyn FileSystem>,
+ file_path: &str,
+ size: u64,
+ ctx: &FsContext,
+) -> Result<()> {
+ copy_file_state(primary, file_path, backup, file_path, size, ctx).await
+}
+
+/// Copy one file within the primary raw backend without decrypting and re-encrypting bytes.
+async fn copy_raw_primary_state(
+ primary_raw: Arc<dyn FileSystem>,
+ source_path: &str,
+ destination_path: &str,
+ ctx: &FsContext,
+) -> Result<()> {
+ let raw_size = FS_CTX
+ .scope(ctx.clone(), async {
+ let source_info = primary_raw.stat(source_path).await?;
+ if source_info.is_dir {
+ return Err(Error::IsADirectory(source_path.to_string()));
+ }
+ Ok::<u64, Error>(source_info.size)
+ })
+ .await?;
+
+ copy_file_state(
+ primary_raw.clone(),
+ source_path,
+ primary_raw,
+ destination_path,
+ raw_size,
+ ctx,
+ )
+ .await
+}
+
+/// Inner state shared via Arc for async spawn and retry_loop.
+pub(crate) struct Inner {
+ /// All backend entries (primary at index 0)
+ backends: Vec<BackendEntry>,
+ /// Index of the primary backend
+ primary_idx: usize,
+ /// Sync type: Async or Sync.
+ sync_type: SyncType,
+ /// Minimum backup ack count for sync mode
+ write_ack_count: usize,
+ /// Timeout for waiting backup ack in sync mode (ms)
+ write_ack_timeout_ms: u64,
+ /// Semaphore for async write concurrency control
+ write_sem: Option<Arc<tokio::sync::Semaphore>>,
+ /// Primary redirect policies
+ redirects: Vec<RedirectPolicy>,
+ /// Metadata store (encrypted via primary_backend)
+ pub(crate) meta_store: MetaStateStore,
+ /// Per-path serialization queues
+ path_queues: PathSerializer,
+ /// Directories that currently have outstanding retry work.
+ pending_dirs: Mutex<HashSet<String>>,
+ /// Retry loop interval.
+ retry_interval: Duration,
+ /// Base retry backoff in milliseconds.
+ pub(crate) retry_backoff_base_ms: u64,
+ /// Maximum retry attempts for one target in one round.
+ pub(crate) max_retry_per_round: usize,
+ /// Failure threshold before quarantining one target.
+ quarantine_after_failures: u32,
+ /// Number of background tasks currently in flight.
+ background_tasks: AtomicUsize,
+ /// Notifier fired when background task count reaches zero.
+ idle_notify: Notify,
+ /// Read route hit metrics.
+ read_backup_hits: AtomicU64,
+ read_primary_hits: AtomicU64,
+ read_redirect_hits: AtomicU64,
+ read_misses: AtomicU64,
+ /// Cancellation flag for the background retry loop.
+ retry_cancelled: AtomicBool,
+ /// Wake-up signal used to stop retry_loop promptly on drop.
+ retry_shutdown: Notify,
+}
+
+/// Multi-write wrapped filesystem.
+pub struct MultiWriteWrappedFS {
+ pub(crate) inner: Arc<Inner>,
+}
+
+/// Builder for `MultiWriteWrappedFS`.
+pub struct MultiWriteWrappedFSBuilder {
+ primary_backend: Arc<dyn FileSystem>,
+ primary_raw_backend: Option<Arc<dyn FileSystem>>,
+ backup_entries: Vec<BackendEntry>,
+ redirects: Vec<RedirectPolicy>,
+ sync_mode: SyncMode,
+ write_concurrency: Option<usize>,
+ retry_interval: Duration,
+ retry_backoff_base_ms: u64,
+ max_retry_per_round: usize,
+ quarantine_after_failures: u32,
+ ctx_resolver: Arc<dyn FsContextResolver>,
+}
+
+impl MultiWriteWrappedFSBuilder {
+ /// Attach the raw primary backend so the wrapper can do verbatim primary copies.
+ pub fn with_primary_raw_backend(mut self, primary_raw_backend: Arc<dyn FileSystem>) -> Self {
+ self.primary_raw_backend = Some(primary_raw_backend);
+ self
+ }
+
+ /// Set backup backend entries on the builder.
+ pub fn with_backups(mut self, backup_entries: Vec<BackendEntry>) -> Self {
+ self.backup_entries = backup_entries;
+ self
+ }
+
+ /// Set redirect policies for the primary backend.
+ pub fn with_redirects(mut self, redirects: Vec<RedirectPolicy>) -> Self {
+ self.redirects = redirects;
+ self
+ }
+
+ /// Select the sync mode used by write fanout.
+ pub fn sync_mode(mut self, sync_mode: SyncMode) -> Self {
+ self.sync_mode = sync_mode;
+ self
+ }
+
+ /// Set the maximum number of concurrent async backup writes.
+ pub fn write_concurrency(mut self, write_concurrency: Option<usize>) -> Self {
+ self.write_concurrency = write_concurrency;
+ self
+ }
+
+ /// Configure retry loop interval.
+ pub fn retry_interval(mut self, retry_interval: Duration) -> Self {
+ self.retry_interval = retry_interval;
+ self
+ }
+
+ /// Configure retry backoff base duration in milliseconds.
+ pub fn retry_backoff_base_ms(mut self, retry_backoff_base_ms: u64) -> Self {
+ self.retry_backoff_base_ms = retry_backoff_base_ms;
+ self
+ }
+
+ /// Configure the maximum number of retries per round.
+ pub fn max_retry_per_round(mut self, max_retry_per_round: usize) -> Self {
+ self.max_retry_per_round = max_retry_per_round.max(1);
+ self
+ }
+
+ /// Configure quarantine threshold for one path/backup pair.
+ pub fn quarantine_after_failures(mut self, quarantine_after_failures: u32) -> Self {
+ self.quarantine_after_failures = quarantine_after_failures.max(1);
+ self
+ }
+
+ /// Accept the legacy read-route cache TTL option as a no-op.
+ pub fn read_route_cache_ttl(self, _read_route_cache_ttl: Duration) -> Self {
+ self
+ }
+
+ /// Configure the context resolver used by retry and admin background paths.
+ pub fn ctx_resolver(mut self, ctx_resolver: Arc<dyn FsContextResolver>) -> Self {
+ self.ctx_resolver = ctx_resolver;
+ self
+ }
+
+ /// Build the multi-write wrapper and start the retry loop when needed.
+ pub fn build(self) -> Result<MultiWriteWrappedFS> {
+ let mut backends = Vec::new();
+ backends.push(BackendEntry {
+ name: "primary".to_string(),
+ role: BackendRole::Primary,
+ backend: self.primary_backend.clone(),
+ raw_backend: self.primary_raw_backend.clone(),
+ operations: Vec::new(),
+ excludes: Vec::new(),
+ });
+ backends.extend(self.backup_entries);
+
+ let (sync_type, write_ack_count, write_ack_timeout_ms) = match self.sync_mode {
+ SyncMode::Sync {
+ ack_count,
+ timeout_ms,
+ } => (SyncType::Sync, ack_count, timeout_ms),
+ SyncMode::Async => (SyncType::Async, usize::MAX, 0),
+ };
+
+ let write_sem = self
+ .write_concurrency
+ .filter(|&n| n > 0)
+ .map(|n| Arc::new(tokio::sync::Semaphore::new(n)));
+
+ let meta_store = MetaStateStore::new(self.primary_backend, self.ctx_resolver);
+
+ let inner = Arc::new(Inner {
+ backends,
+ primary_idx: 0,
+ sync_type,
+ write_ack_count,
+ write_ack_timeout_ms,
+ write_sem,
+ redirects: self.redirects,
+ meta_store,
+ path_queues: PathSerializer::new(),
+ pending_dirs: Mutex::new(HashSet::new()),
+ retry_interval: self.retry_interval,
+ retry_backoff_base_ms: self.retry_backoff_base_ms,
+ max_retry_per_round: self.max_retry_per_round,
+ quarantine_after_failures: self.quarantine_after_failures,
+ background_tasks: AtomicUsize::new(0),
+ idle_notify: Notify::new(),
+ read_backup_hits: AtomicU64::new(0),
+ read_primary_hits: AtomicU64::new(0),
+ read_redirect_hits: AtomicU64::new(0),
+ read_misses: AtomicU64::new(0),
+ retry_cancelled: AtomicBool::new(false),
+ retry_shutdown: Notify::new(),
+ });
+
+ // Start retry_loop if there are write-enabled backups.
+ if inner.write_backups().next().is_some() {
+ inner.background_task_started();
+ tokio::spawn(Inner::retry_loop(Arc::clone(&inner)));
+ }
+
+ Ok(MultiWriteWrappedFS { inner })
+ }
+}
+
+impl MultiWriteWrappedFS {
+ /// Start building a multi-write wrapper from a primary backend.
+ pub fn builder(primary_backend: Arc<dyn FileSystem>) -> MultiWriteWrappedFSBuilder {
+ MultiWriteWrappedFSBuilder {
+ primary_backend,
+ primary_raw_backend: None,
+ backup_entries: Vec::new(),
+ redirects: Vec::new(),
+ sync_mode: SyncMode::Async,
+ write_concurrency: None,
+ retry_interval: DEFAULT_RETRY_INTERVAL,
+ retry_backoff_base_ms: DEFAULT_RETRY_BACKOFF_BASE_MS,
+ max_retry_per_round: DEFAULT_MAX_RETRIES_PER_ROUND,
+ quarantine_after_failures: DEFAULT_QUARANTINE_AFTER_FAILURES,
+ ctx_resolver: Arc::new(DefaultFsContextResolver),
+ }
+ }
+}
+
+impl Inner {
+ /// Build the per-path/per-backend queue key used by both fanout and retry.
+ fn backup_queue_key(path: &str, backup_name: &str) -> String {
+ format!("{}\0{}", path, backup_name)
+ }
+
+ /// Iterate over write-enabled backup entries.
+ fn write_backups(&self) -> impl Iterator<Item = &BackendEntry> {
+ self.backends[self.primary_idx + 1..]
+ .iter()
+ .filter(|be| be.participates_in_write())
+ }
+
+ /// Resolve write-enabled backup targets after applying exclude policies.
+ fn write_targets(&self, path: &str, size: u64) -> Vec<FanoutTarget> {
+ self.write_backups()
+ .filter(|be| !self.is_excluded(be, path, size))
+ .map(BackendEntry::fanout_target)
+ .collect()
+ }
+
+ /// Resolve explicitly named backup targets.
+ fn named_targets(&self, target_names: &[String]) -> Vec<FanoutTarget> {
+ target_names
+ .iter()
+ .filter_map(|name| self.backup_by_name(name))
+ .map(BackendEntry::fanout_target)
+ .collect()
+ }
+
+ /// Execute one backup write synchronously so redirect visibility only appears
+ /// after at least one target has durably materialized the file contents.
+ async fn write_first_target(
+ inner: &Arc<Inner>,
+ path: &str,
+ target: &FanoutTarget,
+ ctx: &FsContext,
+ op: BackupWriteOp,
+ ) -> Result<()> {
+ let queue_key = Self::backup_queue_key(path, &target.name);
+ let fs = target.backend.clone();
+ let ack_ctx = ctx.clone();
+
+ inner
+ .path_queues
+ .with_path_lock(&queue_key, || async move {
+ op.apply(inner, fs, path, ctx).await
+ })
+ .await?;
+
+ inner
+ .update_backup_acked_seq(path, &target.name, &ack_ctx)
+ .await
+ }
+
+ /// Resolve effective target backend names for sync/retry work.
+ pub(crate) fn target_backend_names(
+ &self,
+ redirect_meta: &super::types::RedirectMeta,
+ file_name: &str,
+ file_path: &str,
+ sync_entry: &SyncLogEntry,
+ ) -> Vec<String> {
+ if let Some(redir) = redirect_meta.entries.get(file_name) {
+ return redir.targets.clone();
+ }
+ let policy_size = self.retry_policy_size(sync_entry);
+ self.write_backups()
+ .filter(|be| !self.is_excluded(be, file_path, policy_size))
+ .map(|be| be.name.clone())
+ .collect()
+ }
+
+ /// Iterate over read-enabled backup entries sorted by priority.
+ fn read_backups_sorted(&self) -> Vec<&BackendEntry> {
+ let mut read_backups: Vec<&BackendEntry> = self.backends[self.primary_idx + 1..]
+ .iter()
+ .filter(|be| be.participates_in_read())
+ .collect();
+ read_backups.sort_by_key(|be| be.read_priority().unwrap_or(u32::MAX));
+ read_backups
+ }
+
+ /// Get the primary backend entry.
+ pub(crate) fn primary(&self) -> &BackendEntry {
+ &self.backends[self.primary_idx]
+ }
+
+ /// Get a backup entry by name.
+ fn backup_by_name(&self, name: &str) -> Option<&BackendEntry> {
+ self.backends.iter().find(|be| be.name == name)
+ }
+
+ /// Check if a file should be excluded from a backup.
+ fn is_excluded(&self, backup: &BackendEntry, path: &str, size: u64) -> bool {
+ backup
+ .excludes
+ .iter()
+ .any(|policy| policy.matches(path, size))
+ }
+
+ /// Check if a file matches any redirect policy.
+ fn check_redirect(&self, path: &str, size: u64) -> Option<Vec<String>> {
+ for policy in &self.redirects {
+ if policy.matches(path, size) {
+ let targets = match policy {
+ RedirectPolicy::FileOverSizePolicy { target, .. } => target.clone(),
+ RedirectPolicy::FileExtensionPolicy { target, .. } => target.clone(),
+ };
+ return targets;
+ }
+ }
+ None
+ }
+
+ /// Resolve persisted redirect targets for one already-written file path.
+ async fn redirect_targets_for_path(
+ &self,
+ path: &str,
+ ctx: &FsContext,
+ ) -> Result<Option<Vec<String>>> {
+ let dir = parent_dir(path);
+ let name = file_name(path);
+ let redirect_meta = self.meta_store.get_redirect_meta(&dir, ctx).await?;
+ Ok(redirect_meta
+ .entries
+ .get(name)
+ .map(|entry| entry.targets.clone()))
+ }
+
+ /// Generate and persist the next sequence number.
+ async fn next_seq(&self) -> Result<u64> {
+ self.meta_store.next_seq().await
+ }
+
+ /// Resolve a file size for retry-time policy decisions.
+ fn retry_policy_size(&self, sync_entry: &SyncLogEntry) -> u64 {
+ match &sync_entry.op {
+ SyncOp::SyncFile { size } => *size,
+ _ => 0,
+ }
+ }
+
+ /// Execute the primary-write, sync-log and backup-fanout pipeline.
+ async fn execute_write<R, P, PFut>(
+ inner: &Arc<Self>,
+ path: String,
+ size: u64,
+ sync_op: Option<SyncOp>,
+ backup_op: Option<BackupWriteOp>,
+ primary_fn: P,
+ ) -> Result<R>
+ where
+ P: FnOnce(Arc<dyn FileSystem>) -> PFut + Send,
+ PFut: Future<Output = Result<R>> + Send,
+ {
+ let ctx = current_required_ctx()?;
+ inner.invalidate_read_route(&path).await;
+
+ let prepared_entry = if let Some(entry) = sync_op {
+ let dir = parent_dir(&path);
+ let name = file_name(&path).to_string();
+ let seq = inner.next_seq().await?;
+ inner
+ .meta_store
+ .update_dir_meta(&dir, &ctx, {
+ let name = name.clone();
+ move |_redirect, sync_log| {
+ sync_log.entries.insert(name, SyncLogEntry::new(seq, entry));
+ Ok(())
+ }
+ })
+ .await?;
+ Some((dir, name, seq))
+ } else {
+ None
+ };
+
+ let result = match FS_CTX
+ .scope(ctx.clone(), primary_fn(inner.primary().backend.clone()))
+ .await
+ {
+ Ok(result) => result,
+ Err(err) => {
+ if let Some((dir, name, seq)) = prepared_entry.as_ref() {
+ let _ = inner
+ .meta_store
+ .update_dir_meta(dir, &ctx, {
+ let name = name.clone();
+ let seq = *seq;
+ move |_redirect, sync_log| {
+ let should_remove =
+ sync_log.entries.get(&name).is_some_and(|entry| {
+ entry.latest_seq == seq && !entry.is_primary_committed()
+ });
+ if should_remove {
+ sync_log.entries.remove(&name);
+ }
+ Ok(())
+ }
+ })
+ .await;
+ }
+ return Err(err);
+ }
+ };
+
+ if let Some((dir, name, seq)) = prepared_entry {
+ inner
+ .meta_store
+ .update_dir_meta(&dir, &ctx, move |_redirect, sync_log| {
+ let entry = sync_log.entries.get_mut(&name).ok_or_else(|| {
+ Error::internal(format!(
+ "prepared sync log entry missing while committing '{}'",
+ name
+ ))
+ })?;
+ if entry.latest_seq != seq {
+ return Err(Error::internal(format!(
+ "prepared sync log entry seq mismatch while committing '{}'",
+ name
+ )));
+ }
+ entry.mark_primary_committed();
+ Ok(())
+ })
+ .await?;
+ inner.mark_pending_dir(&dir).await;
+ }
+
+ let dir = parent_dir(&path);
+ let fanout_result = if let Some(backup_op) = backup_op {
+ Inner::fanout_write(inner, &path, size, ctx.clone(), backup_op).await
+ } else {
+ Ok(())
+ };
+ inner.refresh_pending_dir(&dir, &ctx).await?;
+ fanout_result?;
+ Ok(result)
+ }
+
+ /// Execute a write that may be redirected away from the primary backend.
+ async fn execute_write_with_redirect<P, PFut>(
+ inner: &Arc<Self>,
+ path: String,
+ size: u64,
+ sync_op: SyncOp,
+ backup_op: BackupWriteOp,
+ primary_fn: P,
+ ) -> Result<u64>
+ where
+ P: FnOnce(Arc<dyn FileSystem>) -> PFut + Send,
+ PFut: Future<Output = Result<u64>> + Send,
+ {
+ let ctx = current_required_ctx()?;
+ inner.invalidate_read_route(&path).await;
+
+ if let Some(targets) = inner.check_redirect(&path, size) {
+ let dir = parent_dir(&path);
+ let name = file_name(&path).to_string();
+ let seq = inner.next_seq().await?;
+ let resolved_targets = inner.named_targets(&targets);
+ let first_target = resolved_targets.first().cloned().ok_or_else(|| {
+ Error::internal(format!(
+ "redirect path '{}' resolved no writable targets",
+ path
+ ))
+ })?;
+ Inner::write_first_target(inner, &path, &first_target, &ctx, backup_op.clone()).await?;
+ let targets_clone = targets.clone();
+ let first_target_name = first_target.name.clone();
+ inner
+ .meta_store
+ .update_dir_meta(&dir, &ctx, move |redirect, sync_log| {
+ redirect.entries.insert(
+ name.clone(),
+ RedirectEntry {
+ targets: targets_clone.clone(),
+ },
+ );
+ let mut committed = SyncLogEntry::committed(seq, sync_op);
+ committed
+ .backends
+ .insert(first_target_name.clone(), BackendSyncState::acked(seq));
+ sync_log.entries.insert(name, committed);
+ Ok(())
+ })
+ .await?;
+
+ let remaining_targets: Vec<FanoutTarget> =
+ resolved_targets.into_iter().skip(1).collect();
+ if !remaining_targets.is_empty() {
+ inner.mark_pending_dir(&dir).await;
+ Inner::fanout_async(inner, &path, remaining_targets, &ctx, backup_op).await;
+ }
+ inner.refresh_pending_dir(&dir, &ctx).await?;
+ return Ok(size);
+ }
+
+ Inner::execute_write(
+ inner,
+ path,
+ size,
+ Some(sync_op),
+ Some(backup_op),
+ primary_fn,
+ )
+ .await
+ }
+
+ /// Fanout a write operation to all write-enabled backups.
+ /// Takes `&Arc<Inner>` so spawned tasks can clone the Arc for acked_seq updates.
+ /// `ctx` is required for encrypted backup backends and acked_seq updates.
+ async fn fanout_write(
+ inner: &Arc<Inner>,
+ path: &str,
+ size: u64,
+ ctx: FsContext,
+ op: BackupWriteOp,
+ ) -> Result<()> {
+ Inner::fanout_targets(inner, path, inner.write_targets(path, size), ctx, op).await
+ }
+
+ /// Fanout a write operation to explicitly named backup targets (used by redirect path).
+ /// Resolves names to BackendEntry references, then delegates to sync/async state machine.
+ async fn fanout_write_to_targets(
+ inner: &Arc<Inner>,
+ path: &str,
+ target_names: &[String],
+ ctx: FsContext,
+ op: BackupWriteOp,
+ ) -> Result<()> {
+ Inner::fanout_targets(inner, path, inner.named_targets(target_names), ctx, op).await
+ }
+
+ /// Fanout to already resolved targets using the configured sync mode.
+ async fn fanout_targets(
+ inner: &Arc<Inner>,
+ path: &str,
+ targets: Vec<FanoutTarget>,
+ ctx: FsContext,
+ op: BackupWriteOp,
+ ) -> Result<()> {
+ if targets.is_empty() {
+ if matches!(inner.sync_type, SyncType::Sync)
+ && inner.write_ack_count > 0
+ && inner.write_backups().next().is_some()
+ {
+ return Err(Error::SyncWriteQuorum {
+ succeeded: 0,
+ required: inner.write_ack_count,
+ attempted: 0,
+ failures: Vec::new(),
+ });
+ }
+ return Ok(());
+ }
+
+ match inner.sync_type {
+ SyncType::Sync => Inner::fanout_sync(inner, path, &targets, &ctx, op).await,
+ SyncType::Async => {
+ Inner::fanout_async(inner, path, targets, &ctx, op).await;
+ Ok(())
+ }
+ }
+ }
+
+ /// Synchronous fanout: execute writes in parallel, wait for quorum.
+ async fn fanout_sync(
+ inner: &Arc<Inner>,
+ path: &str,
+ targets: &[FanoutTarget],
+ ctx: &FsContext,
+ op: BackupWriteOp,
+ ) -> Result<()> {
+ let ack_count = inner.write_ack_count.min(targets.len());
+ let timeout = if inner.write_ack_timeout_ms > 0 {
+ Some(Duration::from_millis(inner.write_ack_timeout_ms))
+ } else {
+ None
+ };
+
+ let path_owned = path.to_string();
+ let ctx = Some(ctx.clone());
+
+ // Launch parallel tasks for all backup writes.
+ let mut handles = Vec::new();
+ for target in targets {
+ let fs = target.backend.clone();
+ let name = target.name.clone();
+ let path = path_owned.clone();
+ let inner = Arc::clone(inner);
+ let ctx = ctx.clone();
+ let op_clone = op.clone();
+ let queue_key = Self::backup_queue_key(&path, &name);
+
+ handles.push(tokio::spawn(async move {
+ // Wrap in FS_CTX.scope so encrypted backends can access account_id.
+ let exec = inner.path_queues.with_path_lock(&queue_key, || async {
+ op_clone
+ .apply(&inner, fs, &path, ctx.as_ref().unwrap())
+ .await
+ });
+
+ let result = if let Some(timeout) = timeout {
+ match tokio::time::timeout(timeout, exec).await {
+ Ok(Ok(())) => Ok(()),
+ Ok(Err(e)) => Err(e),
+ Err(_) => Err(Error::timeout(format!(
+ "backup '{}' timed out waiting for sync acknowledgement",
+ name
+ ))),
+ }
+ } else {
+ exec.await
+ };
+
+ // Update acked_seq on success.
+ if result.is_ok() {
+ if let Some(ref ctx) = ctx {
+ let _ = inner.update_backup_acked_seq(&path, &name, ctx).await;
+ }
+ }
+
+ SyncFanoutTaskResult {
+ backend: name,
+ result,
+ }
+ }));
+ }
+
+ let results = futures::future::join_all(handles).await;
+
+ let mut successes = 0usize;
+ let mut failures = Vec::new();
+
+ for result in results {
+ match result {
+ Ok(SyncFanoutTaskResult {
+ backend: _,
+ result: Ok(()),
+ }) => {
+ successes += 1;
+ }
+ Ok(SyncFanoutTaskResult {
+ backend,
+ result: Err(e),
+ }) => {
+ failures.push(super::errors::SyncWriteFailureDetail {
+ backend,
+ kind: e.kind_name().to_string(),
+ message: e.to_string(),
+ });
+ }
+ Err(e) => {
+ failures.push(super::errors::SyncWriteFailureDetail {
+ backend: "<spawn>".to_string(),
+ kind: "join_error".to_string(),
+ message: e.to_string(),
+ });
+ }
+ }
+ }
+
+ if successes >= ack_count {
+ Ok(())
+ } else {
+ Err(Error::SyncWriteQuorum {
+ succeeded: successes,
+ required: ack_count,
+ attempted: targets.len(),
+ failures,
+ })
+ }
+ }
+
+ /// Asynchronous fanout: spawn background tasks that update acked_seq on completion.
+ /// Uses per-path serialization to prevent out-of-order application on backup backends.
+ async fn fanout_async(
+ inner: &Arc<Inner>,
+ path: &str,
+ targets: Vec<FanoutTarget>,
+ ctx: &FsContext,
+ op: BackupWriteOp,
+ ) {
+ let path_owned = path.to_string();
+ let sem = inner.write_sem.clone();
+
+ for target in targets {
+ let fs = target.backend.clone();
+ let name = target.name.clone();
+ let path = path_owned.clone();
+ let ctx = ctx.clone();
+ let sem = sem.clone();
+ let inner = Arc::clone(inner);
+ let op_clone = op.clone();
+ let queue_key = Self::backup_queue_key(&path, &name);
+ inner.background_task_started();
+
+ tokio::spawn(async move {
+ {
+ // Per (path, backup) serialization preserves FIFO without blocking other backups.
+ let result = inner
+ .path_queues
+ .with_path_lock(&queue_key, || async {
+ let _permit = if let Some(ref sem) = sem {
+ sem.acquire().await.ok()
+ } else {
+ None
+ };
+
+ op_clone.apply(&inner, fs, &path, &ctx).await
+ })
+ .await;
+
+ // Update acked_seq on successful write.
+ if result.is_ok() {
+ let _ = inner.update_backup_acked_seq(&path, &name, &ctx).await;
+ }
+ }
+ inner.background_task_finished();
+ });
+ }
+ }
+
+ /// Record rename metadata and migrate redirect state when needed.
+ async fn record_rename_meta(
+ &self,
+ old_path: &str,
+ new_path: &str,
+ ctx: &FsContext,
+ ) -> Result<()> {
+ let source_dir = parent_dir(old_path);
+ let target_dir = parent_dir(new_path);
+ let old_name = file_name(old_path).to_string();
+ let new_name = file_name(new_path).to_string();
+ let seq = self.next_seq().await?;
+ let rename_op = SyncOp::Rename {
+ to: new_path.to_string(),
+ };
+
+ if source_dir == target_dir {
+ self.meta_store
+ .update_dir_meta(&source_dir, ctx, move |redirect, sync_log| {
+ sync_log
+ .entries
+ .insert(old_name.clone(), SyncLogEntry::committed(seq, rename_op));
+ if let Some(redirect_entry) = redirect.entries.remove(&old_name) {
+ redirect.entries.insert(new_name, redirect_entry);
+ }
+ Ok(())
+ })
+ .await
+ } else {
+ self.meta_store
+ .update_dual_dir_meta(
+ &source_dir,
+ &target_dir,
+ ctx,
+ move |src_redirect, src_sync_log, tgt_redirect, _tgt_sync_log| {
+ src_sync_log
+ .entries
+ .insert(old_name.clone(), SyncLogEntry::committed(seq, rename_op));
+ if let Some(redirect_entry) = src_redirect.entries.remove(&old_name) {
+ tgt_redirect.entries.insert(new_name, redirect_entry);
+ }
+ Ok(())
+ },
+ )
+ .await
+ }
+ }
+}
+
+/// Return true when `path` falls under `exclude_path` (including itself).
+fn is_excluded_grep_path(path: &str, exclude_path: Option<&str>) -> bool {
+ let Some(exclude_path) = exclude_path.map(normalize_prefix_path) else {
+ return false;
+ };
+ let normalized_path = normalize_prefix_path(path);
+ normalized_path == exclude_path
+ || normalized_path
+ .strip_prefix(&exclude_path)
+ .is_some_and(|suffix| suffix.starts_with('/'))
+}
+
+// ── FileSystem trait implementation ──
+
+impl Drop for MultiWriteWrappedFS {
+ /// Signal retry_loop to exit when the wrapper is unmounted or dropped.
+ fn drop(&mut self) {
+ self.inner.retry_cancelled.store(true, Ordering::SeqCst);
+ self.inner.retry_shutdown.notify_waiters();
+ }
+}
+
+impl MultiWriteWrappedFS {
+ /// Stop background retry work and wait for in-flight async fanout to drain.
+ pub async fn shutdown(&self) -> Result<()> {
+ self.inner.retry_cancelled.store(true, Ordering::SeqCst);
+ self.inner.retry_shutdown.notify_waiters();
+ self.inner.wait_idle(DEFAULT_SHUTDOWN_WAIT).await
+ }
+
+ /// Copy one primary-resident file to a new path while preserving physical primary bytes.
+ pub async fn copy_within_primary(&self, src_path: &str, dst_path: &str) -> Result<bool> {
+ if normalize_prefix_path(src_path) == normalize_prefix_path(dst_path) {
+ return Ok(true);
+ }
+
+ let ctx = current_required_ctx()?;
+ let inner = &self.inner;
+ let source_path = src_path.to_string();
+ let destination_path = dst_path.to_string();
+
+ let source_exists_on_primary = FS_CTX
+ .scope(ctx.clone(), async {
+ inner.primary().backend.exists(&source_path).await
+ })
+ .await;
+ if !source_exists_on_primary {
+ return Ok(false);
+ }
+
+ let source_size = FS_CTX
+ .scope(ctx.clone(), async {
+ let source_info = inner.primary().backend.stat(&source_path).await?;
+ if source_info.is_dir {
+ return Err(Error::IsADirectory(source_path.clone()));
+ }
+ Ok::<u64, Error>(source_info.size)
+ })
+ .await?;
+
+ if inner
+ .check_redirect(&destination_path, source_size)
+ .is_some()
+ {
+ return Ok(false);
+ }
+
+ let Some(primary_raw_backend) = inner.primary().raw_backend.clone() else {
+ return Ok(false);
+ };
+
+ let primary_destination = destination_path.clone();
+ Inner::execute_write(
+ inner,
+ destination_path,
+ source_size,
+ Some(SyncOp::SyncFile { size: source_size }),
+ Some(BackupWriteOp::Replay(SyncOp::SyncFile {
+ size: source_size,
+ })),
+ move |_ignored_fs| {
+ let primary_raw_backend = primary_raw_backend.clone();
+ let primary_source = source_path.clone();
+ let primary_destination = primary_destination.clone();
+ let primary_ctx = ctx.clone();
+ async move {
+ copy_raw_primary_state(
+ primary_raw_backend,
+ &primary_source,
+ &primary_destination,
+ &primary_ctx,
+ )
+ .await?;
+ Ok(())
+ }
+ },
+ )
+ .await?;
+
+ Ok(true)
+ }
+
+ /// Execute one non-redirecting write-like operation through the shared multi-write pipeline.
+ async fn execute_simple_write<R, P, PFut>(
+ &self,
+ path: &str,
+ size: u64,
+ sync_op: Option<SyncOp>,
+ primary_fn: P,
+ ) -> Result<R>
+ where
+ R: Send + 'static,
+ P: FnOnce(Arc<dyn FileSystem>, String) -> PFut + Send,
+ PFut: Future<Output = Result<R>> + Send,
+ {
+ let path_owned = path.to_string();
+ let primary_path = path_owned.clone();
+ let backup_op = sync_op.clone().map(BackupWriteOp::Replay);
+ Inner::execute_write(
+ &self.inner,
+ path_owned,
+ size,
+ sync_op,
+ backup_op,
+ move |fs| primary_fn(fs, primary_path),
+ )
+ .await
+ }
+}
+
+#[async_trait]
+impl FileSystem for MultiWriteWrappedFS {
+ async fn create(&self, path: &str) -> Result<()> {
+ self.execute_simple_write(path, 0, Some(SyncOp::Create), |fs, path| async move {
+ fs.create(&path).await
+ })
+ .await
+ }
+
+ async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {
+ self.execute_simple_write(
+ path,
+ 0,
+ Some(SyncOp::Mkdir { mode }),
+ move |fs, path| async move { fs.mkdir(&path, mode).await },
+ )
+ .await
+ }
+
+ async fn remove(&self, path: &str) -> Result<()> {
+ self.execute_simple_write(path, 0, Some(SyncOp::Remove), |fs, path| async move {
+ fs.remove(&path).await
+ })
+ .await
+ }
+
+ async fn remove_all(&self, path: &str) -> Result<()> {
+ self.execute_simple_write(path, 0, Some(SyncOp::RemoveAll), |fs, path| async move {
+ fs.remove_all(&path).await
+ })
+ .await
+ }
+
+ async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {
+ if let Some(fs) = self.inner.resolve_read_backend(path).await {
+ return fs.read(path, offset, size).await;
+ }
+ Err(Error::not_found(path))
+ }
+
+ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {
+ let inner = &self.inner;
+ let data_len = data.len() as u64;
+ let path_owned = path.to_string();
+ let backup_op = BackupWriteOp::WriteFile {
+ data: Arc::new(data.to_vec()),
+ offset,
+ flags,
+ };
+ let d = data.to_vec();
+ let primary_path = path_owned.clone();
+ Inner::execute_write_with_redirect(
+ inner,
+ path_owned,
+ data_len,
+ SyncOp::SyncFile { size: data_len },
+ backup_op,
+ move |fs: Arc<dyn FileSystem>| async move {
+ fs.write(&primary_path, &d, offset, flags).await
+ },
+ )
+ .await
+ }
+
+ async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {
+ let inner = &self.inner;
+ let mut entries = inner.primary().backend.read_dir(path).await?;
+
+ // Filter multi-write internal names.
+ entries.retain(|e| !MULTIWRITE_INTERNAL_NAMES.contains(&e.name.as_str()));
+
+ // Merge redirect entries so users can see redirected files in listings.
+ let ctx =
+ current_required_ctx().or_else(|_| inner.meta_store.ctx_resolver().resolve(path))?;
+
+ if let Ok(redirect_meta) = inner.meta_store.get_redirect_meta(path, &ctx).await {
+ for (name, redirect_entry) in &redirect_meta.entries {
+ if !entries.iter().any(|e| &e.name == name) {
+ let virtual_path = if path == "/" {
+ format!("/{}", name)
+ } else {
+ format!("{}/{}", path.trim_end_matches('/'), name)
+ };
+ entries.push(
+ inner
+ .redirect_file_info(&virtual_path, name, redirect_entry)
+ .await,
+ );
+ }
+ }
+ }
+
+ Ok(entries)
+ }
+
+ async fn stat(&self, path: &str) -> Result<FileInfo> {
+ if let Some(fs) = self.inner.resolve_read_backend(path).await {
+ return fs.stat(path).await;
+ }
+ Err(Error::not_found(path))
+ }
+
+ async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {
+ let ctx = current_required_ctx()?;
+ let inner = &self.inner;
+ let old_owned = old_path.to_string();
+ let new_owned = new_path.to_string();
+ let source_dir = parent_dir(&old_owned);
+ let redirect_targets = inner.redirect_targets_for_path(&old_owned, &ctx).await?;
+ inner.invalidate_read_route(&old_owned).await;
+ inner.invalidate_read_route(&new_owned).await;
+
+ let primary_has_old = inner.primary().backend.exists(&old_owned).await;
+ if primary_has_old {
+ FS_CTX
+ .scope(ctx.clone(), async {
+ inner.primary().backend.rename(&old_owned, &new_owned).await
+ })
+ .await?;
+ } else if redirect_targets.is_none() {
+ return Err(Error::NotFound(old_owned));
+ }
+
+ inner
+ .record_rename_meta(&old_owned, &new_owned, &ctx)
+ .await?;
+ inner.mark_pending_dir(&source_dir).await;
+
+ let fanout_path = old_owned.clone();
+ let backup_op = BackupWriteOp::Replay(SyncOp::Rename {
+ to: new_owned.clone(),
+ });
+ if let Some(targets) = redirect_targets {
+ Inner::fanout_write_to_targets(inner, &fanout_path, &targets, ctx.clone(), backup_op)
+ .await?;
+ } else {
+ Inner::fanout_write(inner, &fanout_path, 0, ctx.clone(), backup_op).await?;
+ }
+ inner.refresh_pending_dir(&source_dir, &ctx).await?;
+
+ Ok(())
+ }
+
+ async fn chmod(&self, path: &str, mode: u32) -> Result<()> {
+ self.execute_simple_write(
+ path,
+ 0,
+ Some(SyncOp::Chmod { mode }),
+ move |fs, path| async move { fs.chmod(&path, mode).await },
+ )
+ .await
+ }
+
+ async fn truncate(&self, path: &str, size: u64) -> Result<()> {
+ self.execute_simple_write(
+ path,
+ size,
+ Some(SyncOp::SyncFile { size }),
+ move |fs, path| async move { fs.truncate(&path, size).await },
+ )
+ .await
+ }
+
+ async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {
+ let path_owned = path.to_string();
+ let primary_path = path_owned.clone();
+ Inner::execute_write(
+ &self.inner,
+ path_owned,
+ 0,
+ None,
+ Some(BackupWriteOp::EnsureParentDirs { mode }),
+ move |fs| async move { fs.ensure_parent_dirs(&primary_path, mode).await },
+ )
+ .await
+ }
+
+ async fn grep(
+ &self,
+ path: &str,
+ pattern: &str,
+ recursive: bool,
+ case_insensitive: bool,
+ node_limit: Option<usize>,
+ exclude_path: Option<&str>,
+ level_limit: Option<usize>,
+ ) -> Result<GrepResult> {
+ let inner = &self.inner;
+ let path_owned = path.to_string();
+ let pattern_owned = pattern.to_string();
+ let exclude_owned = exclude_path.map(|s| s.to_string());
+
+ let mut result = inner
+ .primary()
+ .backend
+ .grep(
+ &path_owned,
+ &pattern_owned,
+ recursive,
+ case_insensitive,
+ node_limit,
+ exclude_owned.as_deref(),
+ level_limit,
+ )
+ .await?;
+
+ // Filter out multi-write internal metadata files from grep results.
+ result.matches.retain(|m| {
+ let file_name = m.file.rsplit('/').next().unwrap_or(&m.file);
+ !MULTIWRITE_INTERNAL_NAMES.contains(&file_name)
+ });
+ result.count = result.matches.len();
+
+ // For redirect files, also grep in target backends.
+ let ctx = current_required_ctx()
+ .or_else(|_| inner.meta_store.ctx_resolver().resolve(&path_owned))?;
+
+ let search_dir = if inner
+ .primary()
+ .backend
+ .stat(&path_owned)
+ .await
+ .map(|s| s.is_dir)
+ .unwrap_or(false)
+ {
+ path_owned.clone()
+ } else {
+ parent_dir(&path_owned)
+ };
+
+ if recursive && search_dir == path_owned {
+ let redirect_entries = self
+ .tree_directory(&path_owned, true, None, level_limit)
+ .await?;
+ for entry in redirect_entries {
+ if node_limit.is_some_and(|limit| result.count >= limit) {
+ break;
+ }
+ if entry.info.is_dir
+ || entry.extra.get("redirect").and_then(Value::as_bool) != Some(true)
+ {
+ continue;
+ }
+ if is_excluded_grep_path(&entry.path, exclude_owned.as_deref()) {
+ continue;
+ }
+ let Some(read_backend) = inner.resolve_read_backend(&entry.path).await else {
+ continue;
+ };
+ let rel_path = relative_match_file(&path_owned, &entry.path);
+ let target_result = match read_backend
+ .grep(
+ &entry.path,
+ &pattern_owned,
+ false,
+ case_insensitive,
+ node_limit.map(|limit| limit.saturating_sub(result.count)),
+ None,
+ None,
+ )
+ .await
+ {
+ Ok(found) => found,
+ Err(_) => continue,
+ };
+ for m in target_result.matches {
+ if node_limit.is_some_and(|limit| result.count >= limit) {
+ break;
+ }
+ result.add_match(rel_path.clone(), m.line, m.content);
+ }
+ }
+ return Ok(result);
+ }
+
+ if let Ok(redirect_meta) = inner.meta_store.get_redirect_meta(&search_dir, &ctx).await {
+ for (name, redirect_entry) in &redirect_meta.entries {
+ for target_name in &redirect_entry.targets {
+ if let Some(be) = inner.backup_by_name(target_name) {
+ let redirect_path = if search_dir == "/" {
+ format!("/{}", name)
+ } else {
+ format!("{}/{}", search_dir, name)
+ };
+ if let Ok(target_result) = be
+ .backend
+ .grep(
+ &redirect_path,
+ &pattern_owned,
+ false,
+ case_insensitive,
+ node_limit,
+ None,
+ None,
+ )
+ .await
+ {
+ let rel_path = relative_match_file(&path_owned, &redirect_path);
+ for m in target_result.matches {
+ if node_limit.is_some_and(|limit| result.count >= limit) {
+ break;
+ }
+ result.add_match(rel_path.clone(), m.line, m.content);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ Ok(result)
+ }
+
+ async fn tree_directory(
+ &self,
+ path: &str,
+ show_hidden: bool,
+ node_limit: Option<usize>,
+ level_limit: Option<usize>,
+ ) -> Result<Vec<TreeEntry>> {
+ let base = normalize_prefix_path(path);
+ let mut entries = self
+ .inner
+ .primary()
+ .backend
+ .tree_directory(path, show_hidden, node_limit, level_limit)
+ .await?;
+
+ entries.retain(|e| {
+ let name = file_name(&e.path);
+ !MULTIWRITE_INTERNAL_NAMES.contains(&name)
+ });
+
+ let ctx = current_required_ctx()
+ .or_else(|_| self.inner.meta_store.ctx_resolver().resolve(&base))?;
+ let mut seen_paths: HashSet<String> = entries.iter().map(|e| e.path.clone()).collect();
+ let mut dir_paths = vec![base.clone()];
+ for entry in &entries {
+ if entry.info.is_dir {
+ let dir = normalize_prefix_path(&entry.path);
+ if !dir_paths.iter().any(|p| p == &dir) {
+ dir_paths.push(dir);
+ }
+ }
+ }
+
+ for dir in dir_paths {
+ let redirect_meta = match self.inner.meta_store.get_redirect_meta(&dir, &ctx).await {
+ Ok(meta) => meta,
+ Err(_) => continue,
+ };
+ for (name, redirect_entry) in redirect_meta.entries {
+ let virtual_path = if dir == "/" {
+ format!("/{}", name)
+ } else {
+ format!("{}/{}", dir, name)
+ };
+ if seen_paths.contains(&virtual_path) {
+ continue;
+ }
+ let rel_path = if base == "/" {
+ virtual_path.trim_start_matches('/').to_string()
+ } else {
+ virtual_path
+ .strip_prefix(&base)
+ .unwrap_or(&virtual_path)
+ .trim_start_matches('/')
+ .to_string()
+ };
+ let mut extra = HashMap::new();
+ extra.insert("redirect".to_string(), Value::Bool(true));
+ entries.push(TreeEntry {
+ path: virtual_path.clone(),
+ rel_path,
+ info: self
+ .inner
+ .redirect_file_info(&virtual_path, &name, &redirect_entry)
+ .await,
+ extra,
+ });
+ seen_paths.insert(virtual_path);
+ }
+ }
+
+ Ok(entries)
+ }
+}
diff --git a/crates/ragfs/src/core/multibackend_wrapper/retry.rs b/crates/ragfs/src/core/multibackend_wrapper/retry.rs
new file mode 100644
index 000000000..c4581bf22
--- /dev/null
+++ b/crates/ragfs/src/core/multibackend_wrapper/retry.rs
@@ -0,0 +1,270 @@
+use std::time::{Duration, Instant};
+
+use rand::Rng;
+use tracing::warn;
+
+use super::*;
+
+impl Inner {
+ /// Mark one directory as pending retry work.
+ pub(super) async fn mark_pending_dir(&self, dir: &str) {
+ self.pending_dirs.lock().await.insert(dir.to_string());
+ }
+
+ /// Remove one directory from the retry set.
+ async fn clear_pending_dir(&self, dir: &str) {
+ self.pending_dirs.lock().await.remove(dir);
+ }
+
+ /// Snapshot the current set of pending retry directories.
+ pub(crate) async fn pending_dirs_snapshot(&self) -> Vec<String> {
+ self.pending_dirs.lock().await.iter().cloned().collect()
+ }
+
+ /// Recompute whether one directory still has pending retry work.
+ pub(super) async fn refresh_pending_dir(&self, dir: &str, ctx: &FsContext) -> Result<()> {
+ if self.sync_work_in_dir(dir, ctx).await?.iter().any(|work| {
+ work.entry.is_primary_committed()
+ && work.targets.iter().any(|target| {
+ !work.entry.is_in_sync(target) && !work.entry.is_quarantined(target)
+ })
+ }) {
+ self.mark_pending_dir(dir).await;
+ } else {
+ self.clear_pending_dir(dir).await;
+ }
+ Ok(())
+ }
+
+ /// Collect sync work entries for one metadata directory.
+ pub(crate) async fn sync_work_in_dir(
+ &self,
+ dir: &str,
+ ctx: &FsContext,
+ ) -> Result<Vec<SyncWorkEntry>> {
+ let sync_log = self.meta_store.get_sync_log_meta(dir, ctx).await?;
+ if sync_log.entries.is_empty() {
+ return Ok(Vec::new());
+ }
+
+ let redirect_meta = self.meta_store.get_redirect_meta(dir, ctx).await?;
+ Ok(sync_log
+ .entries
+ .into_iter()
+ .map(|(file_name, entry)| {
+ let file_path = if dir == "/" {
+ format!("/{}", file_name)
+ } else {
+ format!("{}/{}", dir, file_name)
+ };
+ let targets =
+ self.target_backend_names(&redirect_meta, &file_name, &file_path, &entry);
+ SyncWorkEntry {
+ file_path,
+ entry,
+ targets,
+ }
+ })
+ .collect())
+ }
+
+ /// Record a started background task.
+ pub(super) fn background_task_started(&self) {
+ self.background_tasks.fetch_add(1, Ordering::SeqCst);
+ }
+
+ /// Record a finished background task and wake idle waiters if needed.
+ pub(super) fn background_task_finished(&self) {
+ if self.background_tasks.fetch_sub(1, Ordering::SeqCst) == 1 {
+ self.idle_notify.notify_waiters();
+ }
+ }
+
+ /// Wait for all background tasks to drain.
+ pub(super) async fn wait_idle(&self, timeout: Duration) -> Result<()> {
+ let deadline = Instant::now() + timeout;
+ loop {
+ if self.background_tasks.load(Ordering::SeqCst) == 0 {
+ return Ok(());
+ }
+ let now = Instant::now();
+ if now >= deadline {
+ return Err(Error::timeout(
+ "timed out while waiting multi-write background tasks to drain",
+ ));
+ }
+ let wait = deadline.saturating_duration_since(now);
+ tokio::time::timeout(wait, self.idle_notify.notified())
+ .await
+ .map_err(|_| {
+ Error::timeout("timed out while waiting multi-write background tasks to drain")
+ })?;
+ }
+ }
+
+ /// Update the acked_seq for a backup in the sync log.
+ pub(super) async fn update_backup_acked_seq(
+ &self,
+ path: &str,
+ backup_name: &str,
+ ctx: &FsContext,
+ ) -> Result<()> {
+ self.update_backend_state(path, backup_name, ctx, |state, latest_seq| {
+ state.mark_acked(latest_seq);
+ })
+ .await?;
+ self.refresh_pending_dir(&parent_dir(path), ctx).await
+ }
+
+ /// Record a replay failure and quarantine the target after repeated failures.
+ pub(crate) async fn record_backup_retry_failure(
+ &self,
+ path: &str,
+ backup_name: &str,
+ ctx: &FsContext,
+ ) -> Result<()> {
+ self.update_backend_state(path, backup_name, ctx, |state, _latest_seq| {
+ state.mark_retry_failed(self.quarantine_after_failures);
+ })
+ .await?;
+ self.refresh_pending_dir(&parent_dir(path), ctx).await
+ }
+
+ /// Update per-backend sync state for a file entry.
+ async fn update_backend_state<F>(
+ &self,
+ path: &str,
+ backup_name: &str,
+ ctx: &FsContext,
+ update: F,
+ ) -> Result<()>
+ where
+ F: FnOnce(&mut BackendSyncState, u64) + Send,
+ {
+ let dir = parent_dir(path);
+ let backup_name = backup_name.to_string();
+ let name = file_name(path).to_string();
+ self.meta_store
+ .update_dir_meta(&dir, ctx, move |_redirect, sync_log| {
+ if let Some(entry) = sync_log.entries.get_mut(&name) {
+ let state = entry.backends.entry(backup_name).or_default();
+ update(state, entry.latest_seq);
+ }
+ Ok(())
+ })
+ .await
+ }
+
+ /// Replay a single operation on a lagging backup.
+ pub(crate) async fn replay_operation(
+ &self,
+ file_path: &str,
+ backup_name: &str,
+ ctx: &FsContext,
+ ) -> Result<()> {
+ let queue_key = Self::backup_queue_key(file_path, backup_name);
+ self.path_queues
+ .with_path_lock(&queue_key, || async {
+ let dir = parent_dir(file_path);
+ let name = file_name(file_path).to_string();
+ let sync_log = self.meta_store.get_sync_log_meta(&dir, ctx).await?;
+ let entry = sync_log
+ .entries
+ .get(&name)
+ .ok_or_else(|| Error::not_found(file_path))?
+ .clone();
+ let backup = match self.backup_by_name(backup_name) {
+ Some(b) => b,
+ None => {
+ return Err(Error::internal(format!(
+ "backup '{}' not found",
+ backup_name
+ )))
+ }
+ };
+
+ entry
+ .op
+ .replay(
+ self.primary().backend.clone(),
+ backup.backend.clone(),
+ file_path,
+ ctx,
+ )
+ .await?;
+
+ self.update_backup_acked_seq(file_path, backup_name, ctx)
+ .await?;
+
+ Ok(())
+ })
+ .await
+ }
+
+ /// Retry all pending backup work for one directory and keep metadata errors visible.
+ pub(crate) async fn retry_pending_dir(&self, dir: &str) -> Result<()> {
+ let ctx = self.meta_store.ctx_resolver().resolve(dir)?;
+
+ for work in self.sync_work_in_dir(dir, &ctx).await? {
+ if !work.entry.is_primary_committed() {
+ continue;
+ }
+
+ for backup_name in &work.targets {
+ if work.entry.is_in_sync(backup_name) || work.entry.is_quarantined(backup_name) {
+ continue;
+ }
+
+ let mut success = false;
+ for attempt in 0..self.max_retry_per_round {
+ if self.retry_cancelled.load(Ordering::SeqCst) {
+ return Ok(());
+ }
+ if self
+ .replay_operation(&work.file_path, backup_name, &ctx)
+ .await
+ .is_ok()
+ {
+ success = true;
+ break;
+ }
+ let base_ms = self.retry_backoff_base_ms.saturating_mul(1u64 << attempt);
+ let jitter_ms = rand::thread_rng().gen_range(0..=50);
+ tokio::time::sleep(Duration::from_millis(base_ms + jitter_ms)).await;
+ }
+
+ if !success {
+ self.record_backup_retry_failure(&work.file_path, backup_name, &ctx)
+ .await?;
+ }
+ }
+ }
+
+ self.refresh_pending_dir(dir, &ctx).await
+ }
+
+ /// Background retry loop: periodically scans sync_log for lagging backups and replays.
+ pub(super) async fn retry_loop(inner: Arc<Inner>) {
+ loop {
+ if inner.retry_cancelled.load(Ordering::SeqCst) {
+ break;
+ }
+
+ tokio::select! {
+ _ = tokio::time::sleep(inner.retry_interval) => {}
+ _ = inner.retry_shutdown.notified() => break,
+ }
+ let dirs = inner.pending_dirs_snapshot().await;
+ for dir in dirs {
+ if let Err(err) = inner.retry_pending_dir(&dir).await {
+ warn!(
+ dir = %dir,
+ error = %err,
+ "multi-write retry skipped one pending directory due to metadata or replay error"
+ );
+ }
+ }
+ }
+ inner.background_task_finished();
+ }
+}
diff --git a/crates/ragfs/src/core/multibackend_wrapper/routing.rs b/crates/ragfs/src/core/multibackend_wrapper/routing.rs
new file mode 100644
index 000000000..8de9babf2
--- /dev/null
+++ b/crates/ragfs/src/core/multibackend_wrapper/routing.rs
@@ -0,0 +1,120 @@
+use std::sync::Arc;
+
+use serde_json::{json, Value};
+
+use super::*;
+use crate::multibackend::meta::{current_required_ctx, file_name, parent_dir};
+
+impl Inner {
+ /// Keep write-side call sites explicit even though read routing is uncached.
+ pub(super) async fn invalidate_read_route(&self, _path: &str) {}
+
+ /// Record read-route counters in one place so hot paths stay explicit.
+ fn record_read_route(&self, source: ReadRouteSource) {
+ match source {
+ ReadRouteSource::Backup => {
+ self.read_backup_hits.fetch_add(1, Ordering::Relaxed);
+ }
+ ReadRouteSource::Primary => {
+ self.read_primary_hits.fetch_add(1, Ordering::Relaxed);
+ }
+ ReadRouteSource::Redirect => {
+ self.read_redirect_hits.fetch_add(1, Ordering::Relaxed);
+ }
+ ReadRouteSource::Miss => {
+ self.read_misses.fetch_add(1, Ordering::Relaxed);
+ }
+ }
+ }
+
+ /// Export read-route metrics for operational introspection.
+ pub(crate) fn read_route_metrics(&self) -> Value {
+ json!({
+ "cache_hits": 0,
+ "backup_hits": self.read_backup_hits.load(Ordering::Relaxed),
+ "primary_hits": self.read_primary_hits.load(Ordering::Relaxed),
+ "redirect_hits": self.read_redirect_hits.load(Ordering::Relaxed),
+ "misses": self.read_misses.load(Ordering::Relaxed),
+ })
+ }
+
+ /// Stat the first reachable redirect target and return user-visible metadata.
+ pub(super) async fn redirect_file_info(
+ &self,
+ path: &str,
+ name: &str,
+ redirect_entry: &RedirectEntry,
+ ) -> FileInfo {
+ for target_name in &redirect_entry.targets {
+ if let Some(be) = self.backup_by_name(target_name) {
+ if let Ok(mut info) = be.backend.stat(path).await {
+ info.name = name.to_string();
+ return info;
+ }
+ }
+ }
+ FileInfo::new_file(name.to_string(), 0, 0o644)
+ }
+
+ /// Resolve the read backend for a path using the fallback chain.
+ pub(super) async fn resolve_read_backend(&self, path: &str) -> Option<Arc<dyn FileSystem>> {
+ let normalized = normalize_prefix_path(path);
+ let read_backups = self.read_backups_sorted();
+ let backup_exists = futures::future::join_all(read_backups.iter().map(|backup| async {
+ (
+ backup.name.clone(),
+ backup.backend.clone(),
+ backup.backend.exists(&normalized).await,
+ )
+ }))
+ .await;
+ for (_name, backend, exists) in backup_exists {
+ if exists {
+ self.record_read_route(ReadRouteSource::Backup);
+ return Some(backend);
+ }
+ }
+
+ if self.primary().backend.exists(&normalized).await {
+ self.record_read_route(ReadRouteSource::Primary);
+ return Some(self.primary().backend.clone());
+ }
+
+ let dir = parent_dir(&normalized);
+ let name = file_name(&normalized).to_string();
+ let ctx = current_required_ctx()
+ .or_else(|_| self.meta_store.ctx_resolver().resolve(&dir))
+ .ok()?;
+ if let Ok(redirect_meta) = self.meta_store.get_redirect_meta(&dir, &ctx).await {
+ if let Some(entry) = redirect_meta.entries.get(&name) {
+ let redirect_targets: Vec<(String, Arc<dyn FileSystem>)> = entry
+ .targets
+ .iter()
+ .filter_map(|target_name| {
+ self.backup_by_name(target_name)
+ .map(|be| (be.name.clone(), be.backend.clone()))
+ })
+ .collect();
+ let redirect_exists = futures::future::join_all(redirect_targets.iter().map(
+ |(target_name, backend)| async {
+ (
+ target_name.clone(),
+ backend.clone(),
+ backend.exists(&normalized).await,
+ )
+ },
+ ))
+ .await;
+ for (_target_name, backend, exists) in redirect_exists {
+ if exists {
+ self.record_read_route(ReadRouteSource::Redirect);
+ return Some(backend);
+ }
+ }
+ }
+ }
+
+ self.record_read_route(ReadRouteSource::Miss);
+ None
+ }
+}
diff --git a/crates/ragfs/src/core/multibackend_wrapper/tests.rs b/crates/ragfs/src/core/multibackend_wrapper/tests.rs
new file mode 100644
index 000000000..357c56fcf
--- /dev/null
+++ b/crates/ragfs/src/core/multibackend_wrapper/tests.rs
@@ -0,0 +1,774 @@
+use std::sync::Arc;
+
+use serde_json::Value;
+
+use super::*;
+use crate::core::context::FsContextInner;
+use crate::plugins::memfs::MemFileSystem;
+
+/// Create the shared test context used by multi-write tests.
+fn test_ctx() -> FsContext {
+ Arc::new(FsContextInner::new("acct".to_string()))
+}
+
+/// Create a sync multi-write filesystem with one memfs backup.
+fn test_multiwrite_fs(redirects: Vec<RedirectPolicy>) -> MultiWriteWrappedFS {
+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let builder = MultiWriteWrappedFS::builder(primary.clone())
+ .with_primary_raw_backend(primary)
+ .with_backups(vec![BackendEntry {
+ name: "backup1".to_string(),
+ role: BackendRole::Backup,
+ backend: backup,
+ raw_backend: None,
+ operations: Vec::new(),
+ excludes: Vec::new(),
+ }])
+ .sync_mode(SyncMode::Sync {
+ ack_count: 1,
+ timeout_ms: 0,
+ });
+
+ if redirects.is_empty() {
+ builder
+ } else {
+ builder.with_redirects(redirects)
+ }
+ .build()
+ .unwrap()
+}
+
+#[test]
+fn test_file_policy_over_size() {
+ let policy = RedirectPolicy::FileOverSizePolicy {
+ max_size_mb: 1,
+ target: Some(vec!["backup1".to_string()]),
+ };
+ assert!(policy.matches("/a/big.bin", 2 * 1024 * 1024));
+ assert!(!policy.matches("/a/small.txt", 512));
+}
+
+#[test]
+fn test_file_policy_extension() {
+ let policy = RedirectPolicy::FileExtensionPolicy {
+ extensions: vec!["(pdf|ppt)".to_string()],
+ target: Some(vec!["backup1".to_string()]),
+ };
+ assert!(policy.matches("/a/doc.pdf", 0));
+ assert!(policy.matches("/a/slides.ppt", 0));
+ assert!(!policy.matches("/a/text.txt", 0));
+}
+
+#[tokio::test]
+async fn test_read_dir_redirect_entries_use_target_stat() {
+ let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {
+ extensions: vec!["\\.pdf$".to_string()],
+ target: Some(vec!["backup1".to_string()]),
+ }]);
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx, async {
+ fs.ensure_parent_dirs("/local/acct/docs/report.pdf", 0o755)
+ .await?;
+ fs.write(
+ "/local/acct/docs/report.pdf",
+ b"pdf body",
+ 0,
+ WriteFlag::Create,
+ )
+ .await?;
+
+ let entries = fs.read_dir("/local/acct/docs").await?;
+ let report = entries
+ .iter()
+ .find(|entry| entry.name == "report.pdf")
+ .expect("redirected file should be visible in read_dir");
+ assert_eq!(report.size, 8);
+ assert_eq!(report.mode, 0o644);
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_recursive_grep_finds_nested_redirected_files() {
+ let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {
+ extensions: vec!["\\.md$".to_string()],
+ target: Some(vec!["backup1".to_string()]),
+ }]);
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx, async {
+ fs.ensure_parent_dirs("/local/acct/resources/doc-1/page.md", 0o755)
+ .await?;
+ fs.write(
+ "/local/acct/resources/doc-1/page.md",
+ b"MULTIWRITE_TEST_MARKER\n",
+ 0,
+ WriteFlag::Create,
+ )
+ .await?;
+
+ let result = fs
+ .grep(
+ "/local/acct/resources",
+ "MULTIWRITE_TEST_MARKER",
+ true,
+ false,
+ None,
+ None,
+ Some(5),
+ )
+ .await?;
+ assert_eq!(result.count, 1);
+ assert_eq!(result.matches.len(), 1);
+ assert_eq!(result.matches[0].file, "doc-1/page.md");
+ assert_eq!(result.matches[0].line, 1);
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_redirect_write_marks_first_target_acked_before_returning() {
+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup1: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup2: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup1_handle = backup1.clone();
+ let fs = MultiWriteWrappedFS::builder(primary.clone())
+ .with_primary_raw_backend(primary)
+ .with_backups(vec![
+ BackendEntry {
+ name: "backup1".to_string(),
+ role: BackendRole::Backup,
+ backend: backup1,
+ raw_backend: None,
+ operations: Vec::new(),
+ excludes: Vec::new(),
+ },
+ BackendEntry {
+ name: "backup2".to_string(),
+ role: BackendRole::Backup,
+ backend: backup2,
+ raw_backend: None,
+ operations: Vec::new(),
+ excludes: Vec::new(),
+ },
+ ])
+ .with_redirects(vec![RedirectPolicy::FileExtensionPolicy {
+ extensions: vec!["\\.pdf$".to_string()],
+ target: Some(vec!["backup1".to_string(), "backup2".to_string()]),
+ }])
+ .sync_mode(SyncMode::Async)
+ .build()
+ .unwrap();
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx.clone(), async {
+ fs.ensure_parent_dirs("/local/acct/docs/report.pdf", 0o755)
+ .await?;
+ fs.write(
+ "/local/acct/docs/report.pdf",
+ b"pdf body",
+ 0,
+ WriteFlag::Create,
+ )
+ .await?;
+
+ let redirect_meta = fs
+ .inner
+ .meta_store
+ .get_redirect_meta("/local/acct/docs", &ctx)
+ .await?;
+ let entry = redirect_meta
+ .entries
+ .get("report.pdf")
+ .expect("redirect metadata should only appear after one target succeeds");
+ assert_eq!(
+ entry.targets,
+ vec!["backup1".to_string(), "backup2".to_string()]
+ );
+
+ let sync_log = fs
+ .inner
+ .meta_store
+ .get_sync_log_meta("/local/acct/docs", &ctx)
+ .await?;
+ let log_entry = sync_log
+ .entries
+ .get("report.pdf")
+ .expect("redirected file should be tracked in sync log");
+ assert!(log_entry.is_primary_committed());
+ assert!(
+ log_entry.is_in_sync("backup1"),
+ "the first redirect target must be acknowledged before returning"
+ );
+ assert!(
+ backup1_handle.exists("/local/acct/docs/report.pdf").await,
+ "the first redirect target must already contain the file"
+ );
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_redirect_write_does_not_publish_metadata_when_first_target_fails() {
+ let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {
+ extensions: vec!["\\.pdf$".to_string()],
+ target: Some(vec!["missing-backup".to_string()]),
+ }]);
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx.clone(), async {
+ let err = fs
+ .write(
+ "/local/acct/docs/fail.pdf",
+ b"content",
+ 0,
+ WriteFlag::Create,
+ )
+ .await
+ .expect_err("redirect write should fail when the first target is not durable");
+
+ assert!(
+ err.to_string().contains("resolved no writable targets"),
+ "unexpected error: {err}"
+ );
+
+ let redirect_meta = fs
+ .inner
+ .meta_store
+ .get_redirect_meta("/local/acct/docs", &ctx)
+ .await?;
+ assert!(
+ !redirect_meta.entries.contains_key("fail.pdf"),
+ "redirect metadata must not be published before the first target succeeds"
+ );
+
+ let sync_log = fs
+ .inner
+ .meta_store
+ .get_sync_log_meta("/local/acct/docs", &ctx)
+ .await?;
+ assert!(
+ !sync_log.entries.contains_key("fail.pdf"),
+ "sync log must not record a committed redirect write before durability"
+ );
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_cross_dir_rename_does_not_create_target_write_log() {
+ let fs = test_multiwrite_fs(Vec::new());
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx.clone(), async {
+ fs.ensure_parent_dirs("/local/acct/src/file.txt", 0o755)
+ .await?;
+ fs.ensure_parent_dirs("/local/acct/dst/file.txt", 0o755)
+ .await?;
+ fs.write(
+ "/local/acct/src/file.txt",
+ b"rename me",
+ 0,
+ WriteFlag::Create,
+ )
+ .await?;
+ fs.rename("/local/acct/src/file.txt", "/local/acct/dst/file.txt")
+ .await?;
+
+ let target_log = fs
+ .inner
+ .meta_store
+ .get_sync_log_meta("/local/acct/dst", &ctx)
+ .await?;
+ assert!(
+ !target_log.entries.contains_key("file.txt"),
+ "target directory must not record a fake write for cross-dir rename"
+ );
+ let source_log = fs
+ .inner
+ .meta_store
+ .get_sync_log_meta("/local/acct/src", &ctx)
+ .await?;
+ let rename_entry = source_log
+ .entries
+ .get("file.txt")
+ .expect("source directory should record rename command");
+ assert!(matches!(
+ &rename_entry.op,
+ SyncOp::Rename { to } if to == "/local/acct/dst/file.txt"
+ ));
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_rename_redirected_file_renames_original_redirect_target() {
+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup_handle = backup.clone();
+ let fs = MultiWriteWrappedFS::builder(primary.clone())
+ .with_primary_raw_backend(primary)
+ .with_backups(vec![BackendEntry {
+ name: "backup1".to_string(),
+ role: BackendRole::Backup,
+ backend: backup,
+ raw_backend: None,
+ operations: vec![OperationItemConfig {
+ operation: "read".to_string(),
+ priority: 1,
+ }],
+ excludes: Vec::new(),
+ }])
+ .with_redirects(vec![RedirectPolicy::FileExtensionPolicy {
+ extensions: vec!["\\.pdf$".to_string()],
+ target: Some(vec!["backup1".to_string()]),
+ }])
+ .sync_mode(SyncMode::Sync {
+ ack_count: 1,
+ timeout_ms: 0,
+ })
+ .build()
+ .unwrap();
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx, async {
+ fs.ensure_parent_dirs("/local/acct/src/a.pdf", 0o755)
+ .await?;
+ fs.ensure_parent_dirs("/local/acct/dst/b.pdf", 0o755)
+ .await?;
+ fs.write(
+ "/local/acct/src/a.pdf",
+ b"pdf payload",
+ 0,
+ WriteFlag::Create,
+ )
+ .await?;
+ assert!(backup_handle.exists("/local/acct/src/a.pdf").await);
+
+ fs.rename("/local/acct/src/a.pdf", "/local/acct/dst/b.pdf")
+ .await?;
+
+ assert!(
+ backup_handle.exists("/local/acct/dst/b.pdf").await,
+ "redirect target backend should receive rename fanout"
+ );
+ assert!(
+ !backup_handle.exists("/local/acct/src/a.pdf").await,
+ "redirect target backend should not retain the old path"
+ );
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_copy_within_primary_preserves_primary_bytes_and_fanouts_backups() {
+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup_handle = backup.clone();
+ let fs = MultiWriteWrappedFS::builder(primary.clone())
+ .with_primary_raw_backend(primary)
+ .with_backups(vec![BackendEntry {
+ name: "backup1".to_string(),
+ role: BackendRole::Backup,
+ backend: backup,
+ raw_backend: None,
+ operations: Vec::new(),
+ excludes: Vec::new(),
+ }])
+ .sync_mode(SyncMode::Sync {
+ ack_count: 1,
+ timeout_ms: 0,
+ })
+ .build()
+ .unwrap();
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx.clone(), async {
+ fs.ensure_parent_dirs("/local/acct/docs/src.bin", 0o755)
+ .await?;
+ fs.write("/local/acct/docs/src.bin", b"payload", 0, WriteFlag::Create)
+ .await?;
+
+ let copied = fs
+ .copy_within_primary("/local/acct/docs/src.bin", "/local/acct/docs/dst.bin")
+ .await?;
+ assert!(
+ copied,
+ "copy_within_primary should use the primary fast-path"
+ );
+
+ assert_eq!(
+ fs.inner
+ .primary()
+ .backend
+ .read("/local/acct/docs/dst.bin", 0, 0)
+ .await?,
+ b"payload"
+ );
+ assert_eq!(
+ backup_handle.read("/local/acct/docs/dst.bin", 0, 0).await?,
+ b"payload"
+ );
+
+ let sync_log = fs
+ .inner
+ .meta_store
+ .get_sync_log_meta("/local/acct/docs", &ctx)
+ .await?;
+ let entry = sync_log
+ .entries
+ .get("dst.bin")
+ .expect("copied file should be tracked for backup fanout");
+ assert!(entry.is_primary_committed());
+ assert!(matches!(entry.op, SyncOp::SyncFile { size } if size == 7));
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_sync_log_write_entry_does_not_embed_large_payload() {
+ let fs = test_multiwrite_fs(Vec::new());
+ let ctx = test_ctx();
+ let payload = vec![b'x'; 256 * 1024];
+
+ FS_CTX
+ .scope(ctx.clone(), async {
+ fs.ensure_parent_dirs("/local/acct/docs/large.bin", 0o755)
+ .await?;
+ fs.write("/local/acct/docs/large.bin", &payload, 0, WriteFlag::Create)
+ .await?;
+
+ let sync_log = fs
+ .inner
+ .meta_store
+ .get_sync_log_meta("/local/acct/docs", &ctx)
+ .await?;
+ let encoded = serde_json::to_vec(&sync_log)?;
+ assert!(
+ encoded.len() < 16 * 1024,
+ "sync log should stay metadata-sized, got {} bytes",
+ encoded.len()
+ );
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_sync_log_truncate_entry_does_not_embed_snapshot() {
+ let fs = test_multiwrite_fs(Vec::new());
+ let ctx = test_ctx();
+ let payload = vec![b'y'; 256 * 1024];
+
+ FS_CTX
+ .scope(ctx.clone(), async {
+ fs.ensure_parent_dirs("/local/acct/docs/large.bin", 0o755)
+ .await?;
+ fs.write("/local/acct/docs/large.bin", &payload, 0, WriteFlag::Create)
+ .await?;
+ fs.truncate("/local/acct/docs/large.bin", 128).await?;
+
+ let sync_log = fs
+ .inner
+ .meta_store
+ .get_sync_log_meta("/local/acct/docs", &ctx)
+ .await?;
+ let encoded = serde_json::to_vec(&sync_log)?;
+ assert!(
+ encoded.len() < 16 * 1024,
+ "truncate sync log should stay metadata-sized, got {} bytes",
+ encoded.len()
+ );
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_fanout_sync_returns_structured_quorum_error() {
+ let fs = test_multiwrite_fs(Vec::new());
+ let ctx = test_ctx();
+ let targets = fs.inner.write_targets("/local/acct/docs/fail.txt", 0);
+
+ let err = Inner::fanout_sync(
+ &fs.inner,
+ "/local/acct/docs/fail.txt",
+ &targets,
+ &ctx,
+ BackupWriteOp::Replay(SyncOp::SyncFile { size: 1 }),
+ )
+ .await
+ .expect_err("sync fanout should surface structured quorum failure");
+
+ match err {
+ Error::SyncWriteQuorum {
+ succeeded,
+ required,
+ attempted,
+ failures,
+ } => {
+ assert_eq!(succeeded, 0);
+ assert_eq!(required, 1);
+ assert_eq!(attempted, 1);
+ assert_eq!(failures.len(), 1);
+ assert_eq!(failures[0].backend, "backup1");
+ assert_eq!(failures[0].kind, "not_found");
+ assert_eq!(failures[0].message, "not found: /local/acct/docs/fail.txt");
+ }
+ other => panic!("unexpected error: {other}"),
+ }
+}
+
+#[tokio::test]
+async fn test_sync_write_errors_when_required_ack_has_no_targets() {
+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let fs = MultiWriteWrappedFS::builder(primary.clone())
+ .with_primary_raw_backend(primary)
+ .with_backups(vec![BackendEntry {
+ name: "backup1".to_string(),
+ role: BackendRole::Backup,
+ backend: backup,
+ raw_backend: None,
+ operations: Vec::new(),
+ excludes: vec![RedirectPolicy::FileExtensionPolicy {
+ extensions: vec!["\\.txt$".to_string()],
+ target: None,
+ }],
+ }])
+ .sync_mode(SyncMode::Sync {
+ ack_count: 1,
+ timeout_ms: 0,
+ })
+ .build()
+ .unwrap();
+ let ctx = test_ctx();
+
+ let err = FS_CTX
+ .scope(ctx, async {
+ fs.ensure_parent_dirs("/local/acct/docs/no-target.txt", 0o755)
+ .await?;
+ fs.write(
+ "/local/acct/docs/no-target.txt",
+ b"payload",
+ 0,
+ WriteFlag::Create,
+ )
+ .await
+ })
+ .await
+ .expect_err("sync mode must not silently pass when all backups are excluded");
+
+ match err {
+ Error::SyncWriteQuorum {
+ succeeded,
+ required,
+ attempted,
+ failures,
+ } => {
+ assert_eq!(succeeded, 0);
+ assert_eq!(required, 1);
+ assert_eq!(attempted, 0);
+ assert!(failures.is_empty());
+ }
+ other => panic!("unexpected error: {other}"),
+ }
+}
+
+#[tokio::test]
+async fn test_read_route_metrics_capture_backup_fallbacks() {
+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup_handle = backup.clone();
+ let fs = MultiWriteWrappedFS::builder(primary)
+ .with_backups(vec![BackendEntry {
+ name: "backup1".to_string(),
+ role: BackendRole::Backup,
+ backend: backup,
+ raw_backend: None,
+ operations: vec![OperationItemConfig {
+ operation: "read".to_string(),
+ priority: 1,
+ }],
+ excludes: Vec::new(),
+ }])
+ .build()
+ .unwrap();
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx, async {
+ backup_handle
+ .ensure_parent_dirs("/local/acct/hot/cache.txt", 0o755)
+ .await?;
+ backup_handle
+ .write("/local/acct/hot/cache.txt", b"hot", 0, WriteFlag::Create)
+ .await?;
+
+ assert_eq!(fs.read("/local/acct/hot/cache.txt", 0, 0).await?, b"hot");
+ assert_eq!(fs.read("/local/acct/hot/cache.txt", 0, 0).await?, b"hot");
+
+ let metrics = fs.inner.read_route_metrics();
+ assert_eq!(metrics.get("backup_hits").and_then(Value::as_u64), Some(2));
+ assert_eq!(metrics.get("cache_hits").and_then(Value::as_u64), Some(0));
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_shutdown_drains_background_retry_loop() {
+ let fs = test_multiwrite_fs(Vec::new());
+ fs.shutdown().await.unwrap();
+}
+
+#[tokio::test]
+async fn test_system_sync_status_reports_corrupt_redirect_meta() {
+ let fs = test_multiwrite_fs(Vec::new());
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx.clone(), async {
+ fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)
+ .await?;
+ fs.write("/local/acct/docs/a.txt", b"hello", 0, WriteFlag::Create)
+ .await?;
+ fs.inner
+ .primary()
+ .backend
+ .write(
+ "/local/acct/docs/.redirect.json",
+ b"{not-json",
+ 0,
+ WriteFlag::Create,
+ )
+ .await?;
+
+ let err = fs
+ .system_sync_status("/local/acct/docs/a.txt")
+ .await
+ .expect_err("corrupt redirect metadata must be reported");
+ assert!(
+ err.to_string().contains("redirect")
+ || err.to_string().contains("json")
+ || err.to_string().contains("serialization"),
+ "unexpected error: {err}"
+ );
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_retry_pending_dir_reports_corrupt_redirect_meta() {
+ let fs = test_multiwrite_fs(Vec::new());
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx.clone(), async {
+ fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)
+ .await?;
+ fs.write("/local/acct/docs/a.txt", b"hello", 0, WriteFlag::Create)
+ .await?;
+ fs.inner
+ .meta_store
+ .update_dir_meta("/local/acct/docs", &ctx, |_redirect, sync_log| {
+ sync_log
+ .entries
+ .insert("a.txt".to_string(), SyncLogEntry::new(1, SyncOp::Create));
+ Ok(())
+ })
+ .await?;
+ fs.inner
+ .primary()
+ .backend
+ .write(
+ "/local/acct/docs/.redirect.json",
+ b"{not-json",
+ 0,
+ WriteFlag::Create,
+ )
+ .await?;
+
+ let err = fs
+ .inner
+ .retry_pending_dir("/local/acct/docs")
+ .await
+ .expect_err("corrupt redirect metadata must be reported to retry logic");
+ assert!(
+ err.to_string().contains("redirect")
+ || err.to_string().contains("json")
+ || err.to_string().contains("serialization"),
+ "unexpected error: {err}"
+ );
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
+
+#[tokio::test]
+async fn test_uncommitted_sync_log_entry_is_visible_but_not_retried() {
+ let fs = test_multiwrite_fs(Vec::new());
+ let ctx = test_ctx();
+
+ FS_CTX
+ .scope(ctx.clone(), async {
+ fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)
+ .await?;
+ fs.inner
+ .meta_store
+ .update_dir_meta("/local/acct/docs", &ctx, |_redirect, sync_log| {
+ sync_log
+ .entries
+ .insert("a.txt".to_string(), SyncLogEntry::new(1, SyncOp::Create));
+ Ok(())
+ })
+ .await?;
+
+ let status = fs.system_sync_status("/local/acct/docs").await?;
+ assert_eq!(status["entry_count"], 1);
+ assert_eq!(status["pending_target_count"], 0);
+ assert_eq!(status["entries"][0]["primary_committed"], false);
+ assert_eq!(
+ status["entries"][0]["targets"][0]["primary_committed"],
+ false
+ );
+
+ let retry = fs.system_sync_retry("/local/acct/docs").await?;
+ assert_eq!(retry["retried"], 0);
+ assert_eq!(retry["failed"], 0);
+ assert_eq!(retry["skipped"], 1);
+ assert_eq!(retry["results"][0]["status"], "awaiting_primary_commit");
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+}
diff --git a/crates/ragfs/src/core/plugin.rs b/crates/ragfs/src/core/plugin.rs
index 2bbcaf1cc..07326e902 100644
--- a/crates/ragfs/src/core/plugin.rs
+++ b/crates/ragfs/src/core/plugin.rs
@@ -211,7 +211,13 @@ mod tests {
async fn read(&self, _path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {
Ok(vec![])
}
- async fn write(&self, _path: &str, _data: &[u8], _offset: u64, _flags: WriteFlag) -> Result<u64> {
+ async fn write(
+ &self,
+ _path: &str,
+ _data: &[u8],
+ _offset: u64,
+ _flags: WriteFlag,
+ ) -> Result<u64> {
Ok(_data.len() as u64)
}
async fn read_dir(&self, _path: &str) -> Result<Vec<FileInfo>> {
@@ -254,11 +260,7 @@ mod tests {
async fn test_plugin_lifecycle() {
let plugin = MockPlugin;
- let config = PluginConfig {
- name: "mock".to_string(),
- mount_path: "/mock".to_string(),
- params: HashMap::new(),
- };
+ let config = PluginConfig::single_backend("mock", "/mock", HashMap::new());
assert!(plugin.validate(&config).await.is_ok());
assert!(plugin.initialize(config).await.is_ok());
diff --git a/crates/ragfs/src/core/stats_wrapper.rs b/crates/ragfs/src/core/stats_wrapper.rs
index bab6c60f2..0a32380ea 100644
--- a/crates/ragfs/src/core/stats_wrapper.rs
+++ b/crates/ragfs/src/core/stats_wrapper.rs
@@ -18,6 +18,11 @@ pub struct StatsWrappedFS {
}
impl StatsWrappedFS {
+ /// Get the wrapped filesystem for specialized delegation.
+ pub(crate) fn inner_fs(&self) -> &Arc<dyn FileSystem> {
+ &self.inner
+ }
+
/// Create a new statistics-wrapped filesystem
pub fn new(inner: Box<dyn FileSystem>) -> Self {
Self {
diff --git a/crates/ragfs/src/core/types.rs b/crates/ragfs/src/core/types.rs
index 1a502c084..eb98a5be9 100644
--- a/crates/ragfs/src/core/types.rs
+++ b/crates/ragfs/src/core/types.rs
@@ -3,6 +3,7 @@
//! This module defines the fundamental data structures used throughout RAGFS,
//! including file metadata, write flags, and configuration types.
+use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::SystemTime;
@@ -74,7 +75,11 @@ impl GrepResult {
/// Add a match to the result
pub fn add_match(&mut self, file: String, line: u64, content: String) {
- self.matches.push(GrepMatch { file, line, content });
+ self.matches.push(GrepMatch {
+ file,
+ line,
+ content,
+ });
self.count += 1;
}
@@ -149,7 +154,7 @@ impl FileInfo {
}
/// Write operation flags
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum WriteFlag {
/// Create new file or truncate existing
Create,
@@ -231,6 +236,136 @@ pub struct PluginConfig {
/// Configuration parameters
pub params: HashMap<String, ConfigValue>,
+
+ /// Multi-write backups config (None = single backend mode)
+ #[serde(default)]
+ pub backups: Option<BackendsConfig>,
+
+ /// Global encryption enabled (server.encryption.enabled)
+ #[serde(default)]
+ pub server_encryption_enabled: bool,
+
+ /// Primary encryption enabled (follows global, not independently configurable)
+ #[serde(default)]
+ pub primary_encryption_enabled: bool,
+
+ /// Primary redirect policies
+ #[serde(default)]
+ pub primary_redirects: Vec<RedirectPolicy>,
+}
+
+impl Default for PluginConfig {
+ /// Build an empty plugin config with all optional multi-write fields disabled.
+ fn default() -> Self {
+ Self {
+ name: String::new(),
+ mount_path: String::new(),
+ params: HashMap::new(),
+ backups: None,
+ server_encryption_enabled: false,
+ primary_encryption_enabled: false,
+ primary_redirects: Vec::new(),
+ }
+ }
+}
+
+impl PluginConfig {
+ /// Build one single-backend plugin config with defaulted optional fields.
+ pub fn single_backend(
+ name: impl Into<String>,
+ mount_path: impl Into<String>,
+ params: HashMap<String, ConfigValue>,
+ ) -> Self {
+ Self {
+ name: name.into(),
+ mount_path: mount_path.into(),
+ params,
+ ..Self::default()
+ }
+ }
+
+ /// Build one plugin config from raw params, extracting multi-write fields in one place.
+ pub fn from_raw_parts(
+ name: impl Into<String>,
+ mount_path: impl Into<String>,
+ mut params: HashMap<String, ConfigValue>,
+ ) -> crate::core::Result<Self> {
+ let backups = Self::take_optional_json("backups", &mut params)?;
+ let server_encryption_enabled =
+ Self::take_bool_with_default("server_encryption_enabled", &mut params, false)?;
+ let primary_encryption_enabled =
+ Self::take_bool_with_default("primary_encryption_enabled", &mut params, false)?;
+ let primary_redirects =
+ Self::take_json_with_default("primary_redirects", &mut params, Vec::new())?;
+
+ Ok(Self {
+ name: name.into(),
+ mount_path: mount_path.into(),
+ params,
+ backups,
+ server_encryption_enabled,
+ primary_encryption_enabled,
+ primary_redirects,
+ })
+ }
+
+ /// Remove one optional bool config value and type-check it.
+ fn take_bool_with_default(
+ field_name: &str,
+ params: &mut HashMap<String, ConfigValue>,
+ default: bool,
+ ) -> crate::core::Result<bool> {
+ match params.remove(field_name) {
+ None => Ok(default),
+ Some(ConfigValue::Bool(value)) => Ok(value),
+ Some(other) => Err(crate::core::Error::config(format!(
+ "'{}' must be a boolean, got {}",
+ field_name,
+ Self::config_value_kind(&other)
+ ))),
+ }
+ }
+
+ /// Remove one optional JSON config value and deserialize it when present.
+ fn take_optional_json<T: DeserializeOwned>(
+ field_name: &str,
+ params: &mut HashMap<String, ConfigValue>,
+ ) -> crate::core::Result<Option<T>> {
+ match params.remove(field_name) {
+ None => Ok(None),
+ Some(ConfigValue::Json(value)) => {
+ let parsed = serde_json::from_value(value).map_err(|err| {
+ crate::core::Error::config(format!("invalid '{}': {}", field_name, err))
+ })?;
+ Ok(Some(parsed))
+ }
+ Some(other) => Err(crate::core::Error::config(format!(
+ "'{}' must be a JSON object/array, got {}",
+ field_name,
+ Self::config_value_kind(&other)
+ ))),
+ }
+ }
+
+ /// Remove one optional JSON config value and fall back to the provided default.
+ fn take_json_with_default<T: DeserializeOwned>(
+ field_name: &str,
+ params: &mut HashMap<String, ConfigValue>,
+ default: T,
+ ) -> crate::core::Result<T> {
+ Ok(Self::take_optional_json(field_name, params)?.unwrap_or(default))
+ }
+
+ /// Return a human-readable config value kind for error messages.
+ fn config_value_kind(value: &ConfigValue) -> &'static str {
+ match value {
+ ConfigValue::String(_) => "string",
+ ConfigValue::Int(_) => "integer",
+ ConfigValue::Bool(_) => "boolean",
+ ConfigValue::StringList(_) => "string_list",
+ ConfigValue::Json(_) => "json",
+ }
+ }
}
/// Configuration value types
@@ -248,6 +383,9 @@ pub enum ConfigValue {
/// List of strings
StringList(Vec<String>),
+
+ /// Nested JSON value (for complex config like backups)
+ Json(serde_json::Value),
}
impl ConfigValue {
@@ -284,6 +422,298 @@ impl ConfigValue {
}
}
+// ── Multi-write configuration types ──
+
+/// Multi-write backends container configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BackendsConfig {
+ /// Sync type: "sync" or "async", default async
+ #[serde(default = "default_sync_type")]
+ pub sync_type: String,
+ /// Minimum backup ack count for sync mode
+ pub write_ack_count: Option<usize>,
+ /// Timeout for waiting backup ack in sync mode (ms)
+ pub write_ack_timeout_ms: Option<u64>,
+ /// Max concurrent async writes
+ pub write_concurrency: Option<usize>,
+ /// Retry loop interval in milliseconds
+ pub retry_interval_ms: Option<u64>,
+ /// Base backoff in milliseconds for retry attempts
+ pub retry_backoff_base_ms: Option<u64>,
+ /// Maximum retry attempts per file/target in one round
+ pub retry_max_retries_per_round: Option<usize>,
+ /// Failure threshold before quarantining one file/target pair
+ pub retry_quarantine_after_failures: Option<u32>,
+ /// Deprecated no-op retained for config compatibility.
+ #[serde(default)]
+ pub read_probe_cache_ttl_ms: Option<u64>,
+ /// Backup items
+ pub items: Vec<BackendItemConfig>,
+}
+
+fn default_sync_type() -> String {
+ "async".to_string()
+}
+
+/// Single backup backend item configuration
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct BackendItemConfig {
+ /// Logical name, globally unique
+ pub name: String,
+ /// Plugin type (local/s3/memfs/kvfs/...)
+ pub backend: String,
+ /// Plugin-specific params (nested JSON)
+ #[serde(default)]
+ pub params: serde_json::Value,
+ /// Timeout in seconds
+ pub timeout: Option<u64>,
+ /// Encryption config for this backup
+ pub encryption: Option<EncryptionConfig>,
+ /// Operations this backup participates in
+ pub operations: Option<Vec<OperationItemConfig>>,
+ /// Exclude policies
+ pub excludes: Option<Vec<RedirectPolicy>>,
+}
+
+/// Per-operation priority config
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OperationItemConfig {
+ /// Operation type: "read" | "write"
+ pub operation: String,
+ /// Priority (smaller = higher priority)
+ pub priority: u32,
+}
+
+/// Encryption on/off config for a backend
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct EncryptionConfig {
+ /// Whether encryption is enabled for this backend
+ pub enabled: bool,
+}
+
+/// Redirect / exclude policy (shared trait)
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type")]
+pub enum RedirectPolicy {
+ /// Redirect/exclude files exceeding a size threshold
+ #[serde(rename = "FileOverSizePolicy")]
+ FileOverSizePolicy {
+ /// Max file size in MB
+ max_size_mb: u64,
+ /// Target backend names (redirect only)
+ target: Option<Vec<String>>,
+ },
+ /// Redirect/exclude files matching extension patterns
+ #[serde(rename = "FileExtensionPolicy")]
+ FileExtensionPolicy {
+ /// Regex patterns for file extensions
+ extensions: Vec<String>,
+ /// Target backend names (redirect only)
+ target: Option<Vec<String>>,
+ },
+}
+
+/// Strongly typed multi-write operation stored in `.sync_log.json`.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(tag = "type")]
+pub enum SyncOp {
+ /// Create an empty file.
+ Create,
+ /// Synchronize file content by copying the current primary state to the backup.
+ SyncFile {
+ /// File size used by retry/exclude policy decisions.
+ size: u64,
+ },
+ /// Create a directory.
+ Mkdir {
+ /// Directory mode.
+ mode: u32,
+ },
+ /// Change file mode.
+ Chmod {
+ /// File mode.
+ mode: u32,
+ },
+ /// Remove one file.
+ Remove,
+ /// Remove a file tree.
+ RemoveAll,
+ /// Rename to another path.
+ Rename {
+ /// Rename target path.
+ to: String,
+ },
+}
+
+/// Per-backend sync state
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
+pub struct BackendSyncState {
+ /// Acknowledged sequence number
+ pub acked_seq: u64,
+ /// Consecutive retry failures for the current operation.
+ #[serde(default)]
+ pub retry_failures: u32,
+ /// Whether this backend/path pair is quarantined for manual intervention.
+ #[serde(default)]
+ pub quarantined: bool,
+}
+
+impl BackendSyncState {
+ /// Create an acknowledged backend sync state.
+ pub fn acked(acked_seq: u64) -> Self {
+ Self {
+ acked_seq,
+ retry_failures: 0,
+ quarantined: false,
+ }
+ }
+
+ /// Mark this backend as acknowledged and clear retry state.
+ pub fn mark_acked(&mut self, acked_seq: u64) {
+ self.acked_seq = acked_seq;
+ self.retry_failures = 0;
+ self.quarantined = false;
+ }
+
+ /// Record one retry failure and quarantine after the configured threshold.
+ pub fn mark_retry_failed(&mut self, quarantine_after_failures: u32) {
+ self.retry_failures = self.retry_failures.saturating_add(1);
+ if self.retry_failures >= quarantine_after_failures {
+ self.quarantined = true;
+ }
+ }
+}
+
+/// Sync log entry for a single file path.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SyncLogEntry {
+ /// Latest monotonic sequence number.
+ pub latest_seq: u64,
+ /// Whether the primary backend has durably completed this operation.
+ #[serde(default = "default_primary_committed")]
+ pub primary_committed: bool,
+ /// Strongly typed operation payload.
+ pub op: SyncOp,
+ /// Per-backend acked sequence numbers.
+ pub backends: std::collections::HashMap<String, BackendSyncState>,
+}
+
+fn default_primary_committed() -> bool {
+ true
+}
+
+impl SyncLogEntry {
+ /// Create a new prepared sync log entry for a sequenced operation.
+ pub fn new(latest_seq: u64, op: SyncOp) -> Self {
+ Self {
+ latest_seq,
+ primary_committed: false,
+ op,
+ backends: std::collections::HashMap::new(),
+ }
+ }
+
+ /// Create a committed sync log entry for an operation already applied on primary.
+ pub fn committed(latest_seq: u64, op: SyncOp) -> Self {
+ Self {
+ latest_seq,
+ primary_committed: true,
+ op,
+ backends: std::collections::HashMap::new(),
+ }
+ }
+
+ /// Mark this entry as committed on primary.
+ pub fn mark_primary_committed(&mut self) {
+ self.primary_committed = true;
+ }
+
+ /// Return whether this entry is committed on primary.
+ pub fn is_primary_committed(&self) -> bool {
+ self.primary_committed
+ }
+
+ /// Return the backend sync state if present.
+ pub fn backend_state(&self, backend_name: &str) -> Option<&BackendSyncState> {
+ self.backends.get(backend_name)
+ }
+
+ /// Return the acknowledged sequence for a backend, or 0 if unknown.
+ pub fn acked_seq(&self, backend_name: &str) -> u64 {
+ self.backend_state(backend_name)
+ .map(|state| state.acked_seq)
+ .unwrap_or(0)
+ }
+
+ /// Return whether the backend is quarantined.
+ pub fn is_quarantined(&self, backend_name: &str) -> bool {
+ self.backend_state(backend_name)
+ .map(|state| state.quarantined)
+ .unwrap_or(false)
+ }
+
+ /// Return whether the backend has acknowledged the latest operation.
+ pub fn is_in_sync(&self, backend_name: &str) -> bool {
+ self.acked_seq(backend_name) >= self.latest_seq
+ }
+}
+
+/// Sync log file content
+#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
+pub struct SyncLogMeta {
+ /// File entries keyed by file name (current directory)
+ pub entries: std::collections::HashMap<String, SyncLogEntry>,
+}
+
+/// Redirect metadata file content
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct RedirectMeta {
+ /// Schema version
+ #[serde(default = "default_redirect_version")]
+ pub version: u32,
+ /// Redirect entries keyed by file name
+ #[serde(default)]
+ pub entries: std::collections::HashMap<String, RedirectEntry>,
+}
+
+fn default_redirect_version() -> u32 {
+ 1
+}
+
+impl Default for RedirectMeta {
+ fn default() -> Self {
+ Self {
+ version: 1,
+ entries: HashMap::new(),
+ }
+ }
+}
+
+/// Single redirect entry
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct RedirectEntry {
+ /// Target backend names
+ pub targets: Vec<String>,
+}
+
+/// Backend role
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BackendRole {
+ /// Primary backend (authoritative source)
+ Primary,
+ /// Backup backend (replica)
+ Backup,
+}
+
+/// Sync type enum
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum SyncType {
+ /// Synchronous: wait for backup ack before returning
+ Sync,
+ /// Asynchronous: return after primary write, sync in background
+ Async,
+}
+
/// Custom serde module for SystemTime
mod systemtime_serde {
use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -342,4 +772,87 @@ mod tests {
assert!(param.required);
assert_eq!(param.param_type, "string");
}
+
+ #[test]
+ fn test_sync_log_entry_legacy_payload_defaults_primary_committed_true() {
+ let entry: SyncLogEntry = serde_json::from_value(serde_json::json!({
+ "latest_seq": 7,
+ "op": {
+ "type": "Create"
+ },
+ "backends": {}
+ }))
+ .unwrap();
+
+ assert!(entry.is_primary_committed());
+ assert_eq!(entry.latest_seq, 7);
+ }
+
+ #[test]
+ fn test_plugin_config_from_raw_parts_handles_multiwrite_shapes() {
+ let mut valid_params = HashMap::new();
+ valid_params.insert(
+ "root_path".to_string(),
+ ConfigValue::String("/tmp/data".to_string()),
+ );
+ valid_params.insert(
+ "backups".to_string(),
+ ConfigValue::Json(serde_json::json!({
+ "items": [{"name": "backup1", "backend": "memory"}]
+ })),
+ );
+ valid_params.insert(
+ "server_encryption_enabled".to_string(),
+ ConfigValue::Bool(true),
+ );
+ valid_params.insert(
+ "primary_encryption_enabled".to_string(),
+ ConfigValue::Bool(true),
+ );
+ valid_params.insert(
+ "primary_redirects".to_string(),
+ ConfigValue::Json(serde_json::json!([
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": [".bin"],
+ "target": ["backup1"]
+ }
+ ])),
+ );
+
+ let config = PluginConfig::from_raw_parts("localfs", "/local", valid_params).unwrap();
+ assert!(config.backups.is_some());
+ assert!(config.server_encryption_enabled);
+ assert!(config.primary_encryption_enabled);
+ assert_eq!(config.primary_redirects.len(), 1);
+ assert_eq!(
+ config.params.get("root_path"),
+ Some(&ConfigValue::String("/tmp/data".to_string()))
+ );
+ assert!(!config.params.contains_key("backups"));
+ assert!(!config.params.contains_key("server_encryption_enabled"));
+ assert!(!config.params.contains_key("primary_encryption_enabled"));
+ assert!(!config.params.contains_key("primary_redirects"));
+
+ for (params, expected_message) in [
+ (
+ HashMap::from([(
+ "backups".to_string(),
+ ConfigValue::Json(serde_json::json!({"items": "invalid"})),
+ )]),
+ "invalid 'backups'",
+ ),
+ (
+ HashMap::from([(
+ "server_encryption_enabled".to_string(),
+ ConfigValue::String("true".to_string()),
+ )]),
+ "'server_encryption_enabled' must be a boolean",
+ ),
+ ] {
+ let err = PluginConfig::from_raw_parts("localfs", "/local", params).unwrap_err();
+ assert!(matches!(err, crate::core::Error::Config(_)));
+ assert!(err.to_string().contains(expected_message));
+ }
+ }
}
diff --git a/crates/ragfs/src/crypto/mod.rs b/crates/ragfs/src/crypto/mod.rs
index 63e17f401..a9d3c5afd 100644
--- a/crates/ragfs/src/crypto/mod.rs
+++ b/crates/ragfs/src/crypto/mod.rs
@@ -226,7 +226,8 @@ mod tests {
assert!(is_encrypted(&env));
let (_p, enc_key2, key_iv2, data_iv2, ct2) = parse_envelope(&env).unwrap();
- let fk_bytes = aes_gcm_decrypt(&account_key, key_iv2.try_into().unwrap(), enc_key2).unwrap();
+ let fk_bytes =
+ aes_gcm_decrypt(&account_key, key_iv2.try_into().unwrap(), enc_key2).unwrap();
let fk: [u8; 32] = fk_bytes.try_into().unwrap();
let back = aes_gcm_decrypt(&fk, data_iv2.try_into().unwrap(), ct2).unwrap();
assert_eq!(back, plaintext);
@@ -268,7 +269,11 @@ mod tests {
let got = hkdf_sha256(&root, b"tenant-7");
let expected =
hex_to_vec("a5447823b691e80727c6bf913bd3ac051f32c36ad387f39bf062f81c68307ae4");
- assert_eq!(&got[..], &expected[..], "HKDF must match Python byte-for-byte");
+ assert_eq!(
+ &got[..],
+ &expected[..],
+ "HKDF must match Python byte-for-byte"
+ );
}
#[test]
@@ -283,9 +288,8 @@ mod tests {
c42d408a1d21e8707cbd617acc62cb9b75dffea1ef5e7889163d970d13a8622cd4d31cddc163f4736556\
c0180cbfb5",
);
- let expected_plaintext = hex_to_vec(
- "63726f73732d6c616e6720696e7465726f7020636865636b20e4bda0e5a5bd",
- );
+ let expected_plaintext =
+ hex_to_vec("63726f73732d6c616e6720696e7465726f7020636865636b20e4bda0e5a5bd");
// Replicate EncryptionWrappedFS::read decryption path.
let account_key = hkdf_sha256(&root, b"tenant-7");
diff --git a/crates/ragfs/src/lib.rs b/crates/ragfs/src/lib.rs
index 20775715a..9d3079cc5 100644
--- a/crates/ragfs/src/lib.rs
+++ b/crates/ragfs/src/lib.rs
@@ -31,7 +31,9 @@
pub mod core;
pub mod crypto;
+pub mod multibackend;
pub mod plugins;
+pub mod shape;
// Re-export core types for convenience
pub use core::{
diff --git a/crates/ragfs/src/multibackend/admin.rs b/crates/ragfs/src/multibackend/admin.rs
new file mode 100644
index 000000000..f9d12331b
--- /dev/null
+++ b/crates/ragfs/src/multibackend/admin.rs
@@ -0,0 +1,234 @@
+use std::collections::HashSet;
+use std::time::Duration;
+
+use serde_json::{json, Value};
+
+use crate::core::errors::Result;
+use crate::core::filesystem::{normalize_prefix_path, FileSystem};
+use crate::core::multibackend_wrapper::{MultiWriteWrappedFS, SyncWorkEntry};
+use crate::multibackend::meta::{current_required_ctx, parent_dir};
+
+impl MultiWriteWrappedFS {
+ /// Collect effective sync work entries under a path using the current request context.
+ async fn collect_sync_work(&self, path: &str) -> Result<Vec<SyncWorkEntry>> {
+ let ctx = current_required_ctx()?;
+ let inner = &self.inner;
+ let normalized = normalize_prefix_path(path);
+ let path_info = <Self as FileSystem>::stat(self, &normalized).await?;
+ let mut dirs = Vec::new();
+ let mut seen_dirs = HashSet::new();
+
+ let add_dir = |dirs: &mut Vec<String>, seen_dirs: &mut HashSet<String>, dir: String| {
+ if seen_dirs.insert(dir.clone()) {
+ dirs.push(dir);
+ }
+ };
+
+ if path_info.is_dir {
+ add_dir(&mut dirs, &mut seen_dirs, normalized.clone());
+ for dir in inner.pending_dirs_snapshot().await {
+ if dir == normalized
+ || dir
+ .strip_prefix(&normalized)
+ .is_some_and(|suffix| suffix.starts_with('/'))
+ {
+ add_dir(&mut dirs, &mut seen_dirs, dir);
+ }
+ }
+ if dirs.len() == 1 {
+ for entry in inner
+ .primary()
+ .backend
+ .tree_directory(&normalized, true, None, None)
+ .await?
+ {
+ if entry.info.is_dir {
+ add_dir(
+ &mut dirs,
+ &mut seen_dirs,
+ normalize_prefix_path(&entry.path),
+ );
+ }
+ }
+ }
+ } else {
+ add_dir(
+ &mut dirs,
+ &mut seen_dirs,
+ normalize_prefix_path(&parent_dir(&normalized)),
+ );
+ }
+
+ let mut work = Vec::new();
+ for dir in dirs {
+ work.extend(
+ inner
+ .sync_work_in_dir(&dir, &ctx)
+ .await?
+ .into_iter()
+ .filter(|work| path_info.is_dir || work.file_path == normalized),
+ );
+ }
+
+ Ok(work)
+ }
+
+ /// Query effective multi-write sync status under a file or directory path.
+ pub async fn system_sync_status(&self, path: &str) -> Result<Value> {
+ let work = self.collect_sync_work(path).await?;
+ let mut entries = Vec::new();
+ let mut pending_target_count = 0usize;
+
+ for work in work {
+ let mut targets = Vec::new();
+ let primary_committed = work.entry.is_primary_committed();
+ let mut all_synced = primary_committed;
+
+ for backend_name in work.targets {
+ let acked_seq = if primary_committed {
+ work.entry.acked_seq(&backend_name)
+ } else {
+ 0
+ };
+ let in_sync = primary_committed && work.entry.is_in_sync(&backend_name);
+ if primary_committed && !in_sync {
+ pending_target_count += 1;
+ all_synced = false;
+ }
+ let state = work.entry.backend_state(&backend_name);
+ targets.push(json!({
+ "name": backend_name,
+ "acked_seq": acked_seq,
+ "retry_failures": state.map(|state| state.retry_failures).unwrap_or(0),
+ "quarantined": state.map(|state| state.quarantined).unwrap_or(false),
+ "primary_committed": primary_committed,
+ "in_sync": in_sync,
+ }));
+ }
+
+ entries.push(json!({
+ "path": work.file_path,
+ "latest_seq": work.entry.latest_seq,
+ "primary_committed": primary_committed,
+ "op": serde_json::to_value(&work.entry.op)?,
+ "all_synced": all_synced,
+ "targets": targets,
+ }));
+ }
+
+ entries.sort_by(|a, b| {
+ let ap = a.get("path").and_then(Value::as_str).unwrap_or_default();
+ let bp = b.get("path").and_then(Value::as_str).unwrap_or_default();
+ ap.cmp(bp)
+ });
+
+ Ok(json!({
+ "path": normalize_prefix_path(path),
+ "entry_count": entries.len(),
+ "pending_target_count": pending_target_count,
+ "capabilities": {
+ "multi_instance_safe": false
+ },
+ "read_route_metrics": self.inner.read_route_metrics(),
+ "entries": entries,
+ }))
+ }
+
+ /// Manually retry lagging multi-write targets under a file or directory path.
+ pub async fn system_sync_retry(&self, path: &str) -> Result<Value> {
+ let ctx = current_required_ctx()?;
+ let work = self.collect_sync_work(path).await?;
+ let mut results = Vec::new();
+ let mut retried = 0usize;
+ let mut failed = 0usize;
+ let mut skipped = 0usize;
+
+ for work in work {
+ if !work.entry.is_primary_committed() {
+ skipped += work.targets.len();
+ for backend_name in work.targets {
+ results.push(json!({
+ "path": work.file_path,
+ "target": backend_name,
+ "status": "awaiting_primary_commit",
+ "latest_seq": work.entry.latest_seq,
+ "primary_committed": false,
+ "acked_seq": 0,
+ }));
+ }
+ continue;
+ }
+
+ for backend_name in work.targets {
+ let acked_seq = work.entry.acked_seq(&backend_name);
+ let was_quarantined = work.entry.is_quarantined(&backend_name);
+ if work.entry.is_in_sync(&backend_name) {
+ skipped += 1;
+ results.push(json!({
+ "path": work.file_path,
+ "target": backend_name,
+ "status": "skipped",
+ "latest_seq": work.entry.latest_seq,
+ "acked_seq": acked_seq,
+ }));
+ continue;
+ }
+
+ let mut last_error = None;
+ let mut success = false;
+ for _attempt in 0..self.inner.max_retry_per_round {
+ match self
+ .inner
+ .replay_operation(&work.file_path, &backend_name, &ctx)
+ .await
+ {
+ Ok(()) => {
+ success = true;
+ break;
+ }
+ Err(err) => {
+ last_error = Some(err.to_string());
+ tokio::time::sleep(Duration::from_millis(
+ self.inner.retry_backoff_base_ms,
+ ))
+ .await;
+ }
+ }
+ }
+
+ if success {
+ retried += 1;
+ results.push(json!({
+ "path": work.file_path,
+ "target": backend_name,
+ "status": "retried",
+ "latest_seq": work.entry.latest_seq,
+ "acked_seq": work.entry.latest_seq,
+ }));
+ } else {
+ self.inner
+ .record_backup_retry_failure(&work.file_path, &backend_name, &ctx)
+ .await?;
+ failed += 1;
+ results.push(json!({
+ "path": work.file_path,
+ "target": backend_name,
+ "status": "failed",
+ "latest_seq": work.entry.latest_seq,
+ "acked_seq": acked_seq,
+ "was_quarantined": was_quarantined,
+ "error": last_error.unwrap_or_else(|| "unknown replay error".to_string()),
+ }));
+ }
+ }
+ }
+
+ Ok(json!({
+ "path": normalize_prefix_path(path),
+ "retried": retried,
+ "failed": failed,
+ "skipped": skipped,
+ "results": results,
+ }))
+ }
+}
diff --git a/crates/ragfs/src/multibackend/config.rs b/crates/ragfs/src/multibackend/config.rs
new file mode 100644
index 000000000..be305f73c
--- /dev/null
+++ b/crates/ragfs/src/multibackend/config.rs
@@ -0,0 +1,127 @@
+//! Multi-backend config validation and normalization helpers.
+
+use std::collections::HashMap;
+
+use serde_json::Value;
+
+use crate::core::errors::{Error, Result};
+use crate::core::multibackend_wrapper::{BackendEntry, SyncMode};
+use crate::core::types::{BackendsConfig, ConfigValue, PluginConfig, RedirectPolicy};
+
+/// Convert one nested backup `params` object into plugin config values.
+pub fn item_params_to_config_values(value: &Value) -> Result<HashMap<String, ConfigValue>> {
+ if value.is_null() {
+ return Ok(HashMap::new());
+ }
+ let obj = value.as_object().ok_or_else(|| {
+ Error::config(format!(
+ "backup item params must be a JSON object, got: {:?}",
+ value
+ ))
+ })?;
+ let mut params = HashMap::new();
+ for (k, v) in obj {
+ let cv = match v {
+ Value::String(s) => ConfigValue::String(s.clone()),
+ Value::Number(n) => {
+ if let Some(i) = n.as_i64() {
+ ConfigValue::Int(i)
+ } else {
+ ConfigValue::String(n.to_string())
+ }
+ }
+ Value::Bool(b) => ConfigValue::Bool(*b),
+ Value::Array(arr) => ConfigValue::StringList(
+ arr.iter()
+ .map(|item| match item {
+ Value::String(s) => s.clone(),
+ other => other.to_string(),
+ })
+ .collect(),
+ ),
+ Value::Object(_) => ConfigValue::Json(v.clone()),
+ _ => ConfigValue::String(v.to_string()),
+ };
+ params.insert(k.clone(), cv);
+ }
+ Ok(params)
+}
+
+/// Validate encryption flags on the primary mount config.
+pub fn validate_primary_encryption_flags(
+ config: &PluginConfig,
+ global_encryption_enabled: bool,
+) -> Result<()> {
+ if global_encryption_enabled && !config.server_encryption_enabled {
+ return Err(Error::config(
+ "server_encryption_enabled must be true when global encryption is configured"
+ .to_string(),
+ ));
+ }
+ if global_encryption_enabled && !config.primary_encryption_enabled {
+ return Err(Error::config(
+ "primary_encryption_enabled cannot be false when global encryption is enabled",
+ ));
+ }
+ Ok(())
+}
+
+/// Validate redirect targets against the assembled backup entries.
+pub fn validate_redirect_targets(
+ redirects: &[RedirectPolicy],
+ backup_entries: &[BackendEntry],
+) -> Result<()> {
+ for policy in redirects {
+ let targets = match policy {
+ RedirectPolicy::FileOverSizePolicy { target, .. } => target.as_ref(),
+ RedirectPolicy::FileExtensionPolicy { target, .. } => target.as_ref(),
+ };
+ let target_names = targets
+ .ok_or_else(|| Error::config("redirect policy target must not be empty".to_string()))?;
+ if target_names.is_empty() {
+ return Err(Error::config(
+ "redirect policy target must not be empty".to_string(),
+ ));
+ }
+ for name in target_names {
+ if !backup_entries.iter().any(|be| &be.name == name) {
+ return Err(Error::config(format!(
+ "redirect target '{}' not found in backup entries",
+ name
+ )));
+ }
+ }
+ }
+ Ok(())
+}
+
+/// Validate that exclude policies do not silently carry redirect targets.
+pub fn validate_backup_excludes(bc: &BackendsConfig) -> Result<()> {
+ for item in &bc.items {
+ for policy in item.excludes.as_deref().unwrap_or(&[]) {
+ match policy {
+ RedirectPolicy::FileOverSizePolicy { target, .. }
+ | RedirectPolicy::FileExtensionPolicy { target, .. } => {
+ if target.is_some() {
+ return Err(Error::config(format!(
+ "exclude policy for backup '{}' must not contain target",
+ item.name
+ )));
+ }
+ }
+ }
+ }
+ }
+ Ok(())
+}
+
+/// Convert config sync settings into the runtime sync mode.
+pub fn sync_mode_from_config(bc: &BackendsConfig) -> SyncMode {
+ match bc.sync_type.as_str() {
+ "sync" => SyncMode::Sync {
+ ack_count: bc.write_ack_count.unwrap_or(usize::MAX),
+ timeout_ms: bc.write_ack_timeout_ms.unwrap_or(0),
+ },
+ _ => SyncMode::Async,
+ }
+}
diff --git a/crates/ragfs/src/multibackend/factory.rs b/crates/ragfs/src/multibackend/factory.rs
new file mode 100644
index 000000000..efaec6824
--- /dev/null
+++ b/crates/ragfs/src/multibackend/factory.rs
@@ -0,0 +1,161 @@
+//! Multi-backend runtime assembly from validated mount configuration.
+
+use std::collections::{HashMap, HashSet};
+use std::sync::Arc;
+use std::time::Duration;
+
+use tokio::sync::RwLock;
+
+use crate::core::encryption_wrapper::EncryptionWrappedFS;
+use crate::core::errors::{Error, Result};
+use crate::core::filesystem::FileSystem;
+use crate::core::multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS};
+use crate::core::plugin::ServicePlugin;
+use crate::core::types::{BackendRole, BackendsConfig, ConfigValue, PluginConfig};
+use crate::multibackend::config::{
+ item_params_to_config_values, sync_mode_from_config, validate_backup_excludes,
+ validate_primary_encryption_flags, validate_redirect_targets,
+};
+use crate::multibackend::meta::MountRootFsContextResolver;
+use crate::multibackend::types::MultiBackendBuildContext;
+use crate::shape::validate::ensure_backend_shape;
+
+/// Initialize one backend plugin instance from config params.
+pub async fn init_backend_plugin(
+ registry: &Arc<RwLock<HashMap<String, Arc<dyn ServicePlugin>>>>,
+ plugin_name: &str,
+ params: &HashMap<String, ConfigValue>,
+) -> Result<Arc<dyn FileSystem>> {
+ let plugin = {
+ let registry = registry.read().await;
+ registry
+ .get(plugin_name)
+ .cloned()
+ .ok_or_else(|| Error::plugin(format!("Plugin '{}' not registered", plugin_name)))?
+ };
+
+ let plugin_config = PluginConfig::single_backend(plugin_name, String::new(), params.clone());
+
+ plugin.validate(&plugin_config).await?;
+ let fs = plugin.initialize(plugin_config).await?;
+ Ok(Arc::from(fs))
+}
+
+/// Build the multi-backend wrapper from primary and backup configuration.
+pub async fn build_multi_write_fs(
+ registry: &Arc<RwLock<HashMap<String, Arc<dyn ServicePlugin>>>>,
+ config: &PluginConfig,
+ bc: &BackendsConfig,
+ build_ctx: MultiBackendBuildContext,
+) -> Result<MultiWriteWrappedFS> {
+ let global_encryption_enabled = build_ctx.global_encryption_enabled();
+ validate_primary_encryption_flags(config, global_encryption_enabled)?;
+
+ let primary_raw = init_backend_plugin(registry, &config.name, &config.params).await?;
+ ensure_backend_shape(
+ &primary_raw,
+ &config.name,
+ global_encryption_enabled,
+ build_ctx.enc_provider_type,
+ build_ctx.enc_root_key,
+ )
+ .await?;
+ let primary_backend: Arc<dyn FileSystem> = if global_encryption_enabled {
+ Arc::new(EncryptionWrappedFS::new(
+ primary_raw.clone(),
+ build_ctx
+ .enc_root_key
+ .expect("global encryption validated before building primary backend"),
+ build_ctx
+ .enc_provider_type
+ .expect("global encryption validated before building primary backend"),
+ ))
+ } else {
+ primary_raw.clone()
+ };
+
+ let mut backup_entries: Vec<BackendEntry> = Vec::new();
+ let mut seen_names: HashSet<String> = HashSet::new();
+
+ for item in &bc.items {
+ if item.name == "primary" {
+ return Err(Error::config(
+ "backup backend name 'primary' is reserved".to_string(),
+ ));
+ }
+ if !seen_names.insert(item.name.clone()) {
+ return Err(Error::config(format!(
+ "duplicate backup name '{}'",
+ item.name
+ )));
+ }
+
+ let backup_params = item_params_to_config_values(&item.params)?;
+ let backup_raw = init_backend_plugin(registry, &item.backend, &backup_params).await?;
+ let backup_encrypted = global_encryption_enabled
+ && item
+ .encryption
+ .as_ref()
+ .map(|encryption| encryption.enabled)
+ .unwrap_or(true);
+ ensure_backend_shape(
+ &backup_raw,
+ &item.backend,
+ backup_encrypted,
+ if backup_encrypted {
+ build_ctx.enc_provider_type
+ } else {
+ None
+ },
+ if backup_encrypted {
+ build_ctx.enc_root_key
+ } else {
+ None
+ },
+ )
+ .await?;
+
+ let backup_backend: Arc<dyn FileSystem> = if backup_encrypted {
+ Arc::new(EncryptionWrappedFS::new(
+ backup_raw,
+ build_ctx
+ .enc_root_key
+ .expect("global encryption validated before building backup backend"),
+ build_ctx
+ .enc_provider_type
+ .expect("global encryption validated before building backup backend"),
+ ))
+ } else {
+ backup_raw
+ };
+
+ backup_entries.push(BackendEntry {
+ name: item.name.clone(),
+ role: BackendRole::Backup,
+ backend: backup_backend,
+ raw_backend: None,
+ operations: item.operations.clone().unwrap_or_default(),
+ excludes: item.excludes.clone().unwrap_or_default(),
+ });
+ }
+
+ validate_redirect_targets(&config.primary_redirects, &backup_entries)?;
+ validate_backup_excludes(bc)?;
+
+ MultiWriteWrappedFS::builder(primary_backend)
+ .with_primary_raw_backend(primary_raw)
+ .with_backups(backup_entries)
+ .with_redirects(config.primary_redirects.clone())
+ .sync_mode(sync_mode_from_config(bc))
+ .write_concurrency(bc.write_concurrency)
+ .retry_interval(Duration::from_millis(
+ bc.retry_interval_ms.unwrap_or(30_000),
+ ))
+ .retry_backoff_base_ms(bc.retry_backoff_base_ms.unwrap_or(1_000))
+ .max_retry_per_round(bc.retry_max_retries_per_round.unwrap_or(3))
+ .quarantine_after_failures(bc.retry_quarantine_after_failures.unwrap_or(9))
+ .ctx_resolver(Arc::new(MountRootFsContextResolver::new(
+ &config.mount_path,
+ )))
+ .build()
+}
diff --git a/crates/ragfs/src/multibackend/meta.rs b/crates/ragfs/src/multibackend/meta.rs
new file mode 100644
index 000000000..10666ddaf
--- /dev/null
+++ b/crates/ragfs/src/multibackend/meta.rs
@@ -0,0 +1,607 @@
+//! Multi-write metadata management.
+//!
+//! Provides `MetaStateStore` for serialized read-modify-write of `.redirect.json` and
+//! `.sync_log.json` through `primary_backend`, and `FsContextResolver` for recovering
+//! `FsContext` from paths in background tasks.
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Mutex;
+
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+
+use crate::core::context::{FsContext, FsContextInner, FS_CTX};
+use crate::core::errors::{Error, Result};
+use crate::core::filesystem::FileSystem;
+use crate::core::types::{RedirectMeta, SyncLogMeta, WriteFlag};
+
+/// Trait for resolving `FsContext` from a filesystem path.
+///
+/// Used by background tasks (retry_loop, backfill, system_sync_retry) that lack a
+/// foreground request context. Implementations extract `account_id` from the path
+/// (e.g. `/local/{account_id}/...`).
+pub trait FsContextResolver: Send + Sync {
+ /// Recover `FsContext` from a normalized path.
+ /// Returns an error if the path cannot be resolved to a valid context.
+ fn resolve(&self, path: &str) -> Result<FsContext>;
+}
+
+/// Default resolver that extracts `account_id` from `/local/{account_id}/...` paths.
+pub struct DefaultFsContextResolver;
+
+impl FsContextResolver for DefaultFsContextResolver {
+ fn resolve(&self, path: &str) -> Result<FsContext> {
+ let parts: Vec<&str> = path.trim_start_matches('/').split('/').collect();
+ // Path format: /local/{account_id}/...
+ if parts.len() >= 2 && parts[0] == "local" && !parts[1].is_empty() {
+ Ok(Arc::new(FsContextInner::new(parts[1].to_string())))
+ } else {
+ Err(Error::internal(format!(
+ "cannot resolve FsContext from path: {}",
+ path
+ )))
+ }
+ }
+}
+
+/// Resolver that extracts `account_id` from paths under one concrete mount root.
+pub struct MountRootFsContextResolver {
+ mount_root: String,
+}
+
+impl MountRootFsContextResolver {
+ /// Create a resolver for one normalized mount root such as `/local` or `/s3`.
+ pub fn new(mount_root: &str) -> Self {
+ let mut normalized = mount_root.trim_end_matches('/').to_string();
+ if normalized.is_empty() {
+ normalized = "/".to_string();
+ } else if !normalized.starts_with('/') {
+ normalized = format!("/{normalized}");
+ }
+ Self {
+ mount_root: normalized,
+ }
+ }
+}
+
+impl FsContextResolver for MountRootFsContextResolver {
+ fn resolve(&self, path: &str) -> Result<FsContext> {
+ let normalized = if path.starts_with('/') {
+ path.to_string()
+ } else {
+ format!("/{path}")
+ };
+ let suffix = if self.mount_root == "/" {
+ normalized.trim_start_matches('/')
+ } else {
+ normalized
+ .strip_prefix(&self.mount_root)
+ .and_then(|suffix| suffix.strip_prefix('/'))
+ .ok_or_else(|| {
+ Error::internal(format!("cannot resolve FsContext from path: {path}"))
+ })?
+ };
+ let account_id = suffix.split('/').next().unwrap_or_default();
+ if account_id.is_empty() {
+ return Err(Error::internal(format!(
+ "cannot resolve FsContext from path: {path}"
+ )));
+ }
+ Ok(Arc::new(FsContextInner::new(account_id.to_string())))
+ }
+}
+
+/// Internal file name for redirect metadata.
+pub(crate) const REDIRECT_FILE: &str = ".redirect.json";
+/// Internal file name for sync-log metadata.
+pub(crate) const SYNC_LOG_FILE: &str = ".sync_log.json";
+/// Hidden multi-write internal file names (redirect / sync-log metadata).
+pub(crate) const MULTIWRITE_INTERNAL_NAMES: &[&str] = &[SYNC_LOG_FILE, REDIRECT_FILE];
+const GLOBAL_STATE_FILE: &str = "/local/_system/.multiwrite.global.json";
+const GLOBAL_STATE_VERSION: u32 = 1;
+
+type TrackedLock = Mutex<()>;
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+struct GlobalMultiWriteState {
+ version: u32,
+ next_seq: u64,
+}
+
+impl Default for GlobalMultiWriteState {
+ /// Create the default persisted global state for multi-write sequencing.
+ fn default() -> Self {
+ Self {
+ version: GLOBAL_STATE_VERSION,
+ next_seq: 1,
+ }
+ }
+}
+
+/// Unified metadata store for `.redirect.json` and `.sync_log.json`.
+///
+/// All reads and writes go through `primary_backend`, inheriting its encryption
+/// configuration. Directory-level locks ensure serialized access to both metadata
+/// files within the same directory.
+pub struct MetaStateStore {
+ /// Primary backend (may be encrypted)
+ primary_backend: Arc<dyn FileSystem>,
+ /// Per-directory locks for serialized read-modify-write
+ dir_locks: Mutex<HashMap<String, Arc<TrackedLock>>>,
+ /// Dedicated global state lock for next_seq persistence.
+ global_state_lock: Mutex<()>,
+ /// Context resolver for background tasks
+ ctx_resolver: Arc<dyn FsContextResolver>,
+}
+
+impl MetaStateStore {
+ /// Create a new MetaStateStore.
+ pub fn new(
+ primary_backend: Arc<dyn FileSystem>,
+ ctx_resolver: Arc<dyn FsContextResolver>,
+ ) -> Self {
+ Self::with_cache_config(primary_backend, ctx_resolver, 0, Duration::ZERO)
+ }
+
+ /// Create a new MetaStateStore; cache parameters are retained as no-op compatibility knobs.
+ pub fn with_cache_config(
+ primary_backend: Arc<dyn FileSystem>,
+ ctx_resolver: Arc<dyn FsContextResolver>,
+ _meta_cache_capacity: usize,
+ _meta_cache_ttl: Duration,
+ ) -> Self {
+ Self {
+ primary_backend,
+ dir_locks: Mutex::new(HashMap::new()),
+ global_state_lock: Mutex::new(()),
+ ctx_resolver,
+ }
+ }
+
+ /// Get or create a per-directory lock.
+ async fn get_dir_lock(&self, dir: &str) -> Arc<TrackedLock> {
+ let mut locks = self.dir_locks.lock().await;
+ if let Some(lock) = locks.get(dir).cloned() {
+ return lock;
+ }
+ let lock = Arc::new(TrackedLock::new(()));
+ locks.insert(dir.to_string(), lock.clone());
+ lock
+ }
+
+ /// Return the dedicated `_system` context used by global metadata.
+ fn system_ctx() -> FsContext {
+ Arc::new(FsContextInner::new("_system".to_string()))
+ }
+
+ /// Build the full path for a metadata file in a directory.
+ fn meta_path(dir: &str, filename: &str) -> String {
+ if dir == "/" {
+ format!("/{}", filename)
+ } else {
+ format!("{}/{}", dir, filename)
+ }
+ }
+
+ /// Read one JSON metadata file, returning default for missing or empty files.
+ async fn read_meta<T>(&self, dir: &str, filename: &str, ctx: &FsContext) -> Result<T>
+ where
+ T: DeserializeOwned + Default,
+ {
+ let path = Self::meta_path(dir, filename);
+ match FS_CTX
+ .scope(ctx.clone(), async {
+ self.primary_backend.read(&path, 0, 0).await
+ })
+ .await
+ {
+ Ok(data) => {
+ if data.is_empty() {
+ Ok(T::default())
+ } else {
+ serde_json::from_slice(&data).map_err(Error::from)
+ }
+ }
+ Err(Error::NotFound(_)) => Ok(T::default()),
+ Err(e) => Err(e),
+ }
+ }
+
+ /// Read redirect metadata from a directory (returns default if not found).
+ async fn read_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result<RedirectMeta> {
+ self.read_meta(dir, REDIRECT_FILE, ctx).await
+ }
+
+ /// Read sync log metadata from a directory (returns default if not found).
+ async fn read_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result<SyncLogMeta> {
+ self.read_meta(dir, SYNC_LOG_FILE, ctx).await
+ }
+
+ /// Read both metadata files from the primary backend.
+ async fn read_dir_meta_pair(
+ &self,
+ dir: &str,
+ ctx: &FsContext,
+ ) -> Result<(RedirectMeta, SyncLogMeta)> {
+ let redirect = self.read_redirect_meta(dir, ctx).await?;
+ let sync_log = self.read_sync_log_meta(dir, ctx).await?;
+ Ok((redirect, sync_log))
+ }
+
+ /// Write one JSON metadata file to a directory.
+ async fn write_meta<T>(
+ &self,
+ dir: &str,
+ filename: &str,
+ meta: &T,
+ ctx: &FsContext,
+ ) -> Result<()>
+ where
+ T: Serialize,
+ {
+ let path = Self::meta_path(dir, filename);
+ let data = serde_json::to_vec(meta)?;
+ FS_CTX
+ .scope(ctx.clone(), async {
+ self.primary_backend
+ .write(&path, &data, 0, WriteFlag::Create)
+ .await
+ .map(|_| ())
+ })
+ .await
+ }
+
+ /// Write redirect metadata to a directory.
+ async fn write_redirect_meta(
+ &self,
+ dir: &str,
+ meta: &RedirectMeta,
+ ctx: &FsContext,
+ ) -> Result<()> {
+ self.write_meta(dir, REDIRECT_FILE, meta, ctx).await
+ }
+
+ /// Write sync log metadata to a directory.
+ async fn write_sync_log_meta(
+ &self,
+ dir: &str,
+ meta: &SyncLogMeta,
+ ctx: &FsContext,
+ ) -> Result<()> {
+ self.write_meta(dir, SYNC_LOG_FILE, meta, ctx).await
+ }
+
+ /// Serialized read-modify-write of both `.redirect.json` and `.sync_log.json` in a directory.
+ ///
+ /// Acquires the directory lock, reads both metadata files, applies `op`, and writes both back.
+ /// This prevents concurrent updates from losing entries.
+ pub async fn update_dir_meta<F>(&self, dir: &str, ctx: &FsContext, op: F) -> Result<()>
+ where
+ F: FnOnce(&mut RedirectMeta, &mut SyncLogMeta) -> Result<()>,
+ {
+ let lock = self.get_dir_lock(dir).await;
+ let _guard = lock.lock().await;
+
+ let (mut redirect_meta, mut sync_log_meta) = self.read_dir_meta_pair(dir, ctx).await?;
+ let original_redirect = redirect_meta.clone();
+ let original_sync_log = sync_log_meta.clone();
+
+ op(&mut redirect_meta, &mut sync_log_meta)?;
+
+ if redirect_meta != original_redirect {
+ self.write_redirect_meta(dir, &redirect_meta, ctx).await?;
+ }
+ if sync_log_meta != original_sync_log {
+ self.write_sync_log_meta(dir, &sync_log_meta, ctx).await?;
+ }
+ Ok(())
+ }
+
+ /// Serialized read-modify-write of two directories' metadata (for cross-directory rename).
+ ///
+ /// Acquires both directory locks in lexicographic order to prevent deadlock,
+ /// then reads and updates all four metadata files within the same critical section.
+ /// Caller must ensure source_dir != target_dir; use update_dir_meta for same-directory case.
+ pub async fn update_dual_dir_meta<F>(
+ &self,
+ source_dir: &str,
+ target_dir: &str,
+ ctx: &FsContext,
+ op: F,
+ ) -> Result<()>
+ where
+ F: FnOnce(
+ &mut RedirectMeta,
+ &mut SyncLogMeta,
+ &mut RedirectMeta,
+ &mut SyncLogMeta,
+ ) -> Result<()>,
+ {
+ // Acquire locks in lexicographic order to avoid deadlock.
+ let (first_dir, second_dir) = if source_dir < target_dir {
+ (source_dir, target_dir)
+ } else {
+ (target_dir, source_dir)
+ };
+
+ let lock1 = self.get_dir_lock(first_dir).await;
+ let lock2 = self.get_dir_lock(second_dir).await;
+ let _guard1 = lock1.lock().await;
+ let _guard2 = lock2.lock().await;
+
+ let (mut src_redirect, mut src_sync_log) = self.read_dir_meta_pair(source_dir, ctx).await?;
+ let (mut tgt_redirect, mut tgt_sync_log) = self.read_dir_meta_pair(target_dir, ctx).await?;
+ let original_src_redirect = src_redirect.clone();
+ let original_src_sync_log = src_sync_log.clone();
+ let original_tgt_redirect = tgt_redirect.clone();
+ let original_tgt_sync_log = tgt_sync_log.clone();
+
+ op(
+ &mut src_redirect,
+ &mut src_sync_log,
+ &mut tgt_redirect,
+ &mut tgt_sync_log,
+ )?;
+
+ if src_redirect != original_src_redirect {
+ self.write_redirect_meta(source_dir, &src_redirect, ctx)
+ .await?;
+ }
+ if src_sync_log != original_src_sync_log {
+ self.write_sync_log_meta(source_dir, &src_sync_log, ctx)
+ .await?;
+ }
+ if tgt_redirect != original_tgt_redirect {
+ self.write_redirect_meta(target_dir, &tgt_redirect, ctx)
+ .await?;
+ }
+ if tgt_sync_log != original_tgt_sync_log {
+ self.write_sync_log_meta(target_dir, &tgt_sync_log, ctx)
+ .await?;
+ }
+ Ok(())
+ }
+
+ /// Read redirect metadata for a directory (public, used by read_dir to merge redirect entries).
+ pub async fn get_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result<RedirectMeta> {
+ self.read_dir_meta_pair(dir, ctx)
+ .await
+ .map(|(redirect, _)| redirect)
+ }
+
+ /// Read sync log metadata for a directory (public, used by retry_loop).
+ pub async fn get_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result<SyncLogMeta> {
+ self.read_dir_meta_pair(dir, ctx)
+ .await
+ .map(|(_, sync_log)| sync_log)
+ }
+
+ /// Get a reference to the context resolver.
+ pub fn ctx_resolver(&self) -> &Arc<dyn FsContextResolver> {
+ &self.ctx_resolver
+ }
+
+ /// Get a reference to the primary backend.
+ pub fn primary_backend(&self) -> &Arc<dyn FileSystem> {
+ &self.primary_backend
+ }
+
+ /// Allocate and persist the next global sequence number.
+ pub async fn next_seq(&self) -> Result<u64> {
+ let _guard = self.global_state_lock.lock().await;
+ let mut state = self.read_global_state().await?;
+ let seq = state.next_seq;
+ state.next_seq = state.next_seq.saturating_add(1);
+ self.write_global_state(&state).await?;
+ Ok(seq)
+ }
+
+ /// Read the persisted global state file.
+ async fn read_global_state(&self) -> Result<GlobalMultiWriteState> {
+ let ctx = Self::system_ctx();
+ match FS_CTX
+ .scope(ctx.clone(), async {
+ self.primary_backend.read(GLOBAL_STATE_FILE, 0, 0).await
+ })
+ .await
+ {
+ Ok(data) => {
+ if data.is_empty() {
+ Ok(GlobalMultiWriteState::default())
+ } else {
+ let state: GlobalMultiWriteState = serde_json::from_slice(&data)?;
+ if state.version != GLOBAL_STATE_VERSION {
+ return Err(Error::config(format!(
+ "unsupported multi-write global state version {}",
+ state.version
+ )));
+ }
+ Ok(state)
+ }
+ }
+ Err(Error::NotFound(_)) => Ok(GlobalMultiWriteState::default()),
+ Err(e) => Err(e),
+ }
+ }
+
+ /// Persist the global sequence state through the primary backend.
+ async fn write_global_state(&self, state: &GlobalMultiWriteState) -> Result<()> {
+ let ctx = Self::system_ctx();
+ let data = serde_json::to_vec(state)?;
+ FS_CTX
+ .scope(ctx.clone(), async {
+ self.primary_backend
+ .ensure_parent_dirs(GLOBAL_STATE_FILE, 0o755)
+ .await?;
+ self.primary_backend
+ .write(GLOBAL_STATE_FILE, &data, 0, WriteFlag::Create)
+ .await
+ .map(|_| ())
+ })
+ .await
+ }
+}
+
+/// Per-path serialization queue for async write ordering.
+///
+/// Ensures that multiple writes to the same path are executed in FIFO order
+/// on backup backends, preventing out-of-order application.
+pub struct PathSerializer {
+ queues: Mutex<HashMap<String, Arc<TrackedLock>>>,
+}
+
+impl PathSerializer {
+ /// Create a new PathSerializer.
+ pub fn new() -> Self {
+ Self::with_limits(0, Duration::ZERO)
+ }
+
+ /// Create a PathSerializer; limit arguments are ignored for compatibility.
+ pub fn with_limits(_capacity: usize, _ttl: Duration) -> Self {
+ Self {
+ queues: Mutex::new(HashMap::new()),
+ }
+ }
+
+ /// Run one async operation under the per-path serialization lock.
+ pub async fn with_path_lock<F, Fut, T>(&self, path: &str, op: F) -> T
+ where
+ F: FnOnce() -> Fut,
+ Fut: std::future::Future<Output = T>,
+ {
+ let mut queues = self.queues.lock().await;
+ let lock = queues
+ .entry(path.to_string())
+ .or_insert_with(|| Arc::new(TrackedLock::new(())))
+ .clone();
+ drop(queues);
+
+ let _guard = lock.lock().await;
+ op().await
+ }
+
+ /// Return the number of tracked queue entries.
+ #[cfg(test)]
+ pub async fn len(&self) -> usize {
+ self.queues.lock().await.len()
+ }
+}
+
+impl Default for PathSerializer {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+/// Extract the directory path from a file path.
+pub(crate) fn parent_dir(path: &str) -> String {
+ match path.rfind('/') {
+ Some(0) => "/".to_string(),
+ Some(pos) => path[..pos].to_string(),
+ None => "/".to_string(),
+ }
+}
+
+/// Extract the file name from a path.
+pub(crate) fn file_name(path: &str) -> &str {
+ match path.rfind('/') {
+ Some(pos) => &path[pos + 1..],
+ None => path,
+ }
+}
+
+/// Snapshot the current FsContext from the task-local, returning an error if unset.
+pub fn current_required_ctx() -> Result<FsContext> {
+ FS_CTX
+ .try_with(|c| c.clone())
+ .map_err(|_| Error::context_missing("FsContext not set in current task"))
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::plugins::memfs::MemFileSystem;
+ use std::sync::Arc;
+
+ #[test]
+ fn test_parent_dir() {
+ assert_eq!(parent_dir("/a/b/c.txt"), "/a/b");
+ assert_eq!(parent_dir("/a"), "/");
+ assert_eq!(parent_dir("/"), "/");
+ }
+
+ #[test]
+ fn test_file_name() {
+ assert_eq!(file_name("/a/b/c.txt"), "c.txt");
+ assert_eq!(file_name("/a"), "a");
+ }
+
+ #[test]
+ fn test_default_resolver() {
+ let resolver = DefaultFsContextResolver;
+ let ctx = resolver
+ .resolve("/local/tenant-1/resources/file.txt")
+ .unwrap();
+ assert_eq!(ctx.account_id(), "tenant-1");
+ }
+
+ #[test]
+ fn test_default_resolver_invalid_path() {
+ let resolver = DefaultFsContextResolver;
+ assert!(resolver.resolve("/invalid/path").is_err());
+ }
+
+ #[test]
+ fn test_mount_root_resolver_extracts_account_from_non_local_path() {
+ let resolver = MountRootFsContextResolver::new("/s3");
+ let ctx = resolver.resolve("/s3/acct-1/resources/file.txt").unwrap();
+ assert_eq!(ctx.account_id(), "acct-1");
+ assert!(resolver
+ .resolve("/local/acct-1/resources/file.txt")
+ .is_err());
+ }
+
+ #[tokio::test]
+ async fn test_invalid_sync_log_json_returns_error() {
+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let store = MetaStateStore::new(primary.clone(), Arc::new(DefaultFsContextResolver));
+ let ctx = Arc::new(FsContextInner::new("acct".to_string()));
+
+ FS_CTX
+ .scope(ctx.clone(), async {
+ primary
+ .ensure_parent_dirs("/local/acct/docs/.sync_log.json", 0o755)
+ .await?;
+ primary
+ .write(
+ "/local/acct/docs/.sync_log.json",
+ b"{not valid json",
+ 0,
+ WriteFlag::Create,
+ )
+ .await?;
+ Ok::<(), Error>(())
+ })
+ .await
+ .unwrap();
+
+ let result = store.get_sync_log_meta("/local/acct/docs", &ctx).await;
+ assert!(result.is_err(), "corrupted metadata must fail fast");
+ }
+
+ #[tokio::test]
+ async fn test_next_seq_uses_dedicated_global_lock() {
+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let store = MetaStateStore::new(primary, Arc::new(DefaultFsContextResolver));
+
+ assert_eq!(store.next_seq().await.unwrap(), 1);
+ assert_eq!(store.next_seq().await.unwrap(), 2);
+
+ let dir_locks = store.dir_locks.lock().await;
+ assert!(
+ !dir_locks.contains_key(GLOBAL_STATE_FILE),
+ "global sequence locking should not occupy the directory lock pool"
+ );
+ }
+}
diff --git a/crates/ragfs/src/multibackend/mod.rs b/crates/ragfs/src/multibackend/mod.rs
new file mode 100644
index 000000000..44e8e4ab2
--- /dev/null
+++ b/crates/ragfs/src/multibackend/mod.rs
@@ -0,0 +1,14 @@
+//! Multi-backend configuration, factory, metadata, and admin support.
+
+/// Multi-backend administrative APIs.
+pub mod admin;
+/// Multi-backend config validation and normalization helpers.
+pub mod config;
+/// Multi-backend runtime assembly from validated config.
+pub mod factory;
+/// Metadata state management shared by multi-backend runtime paths.
+pub mod meta;
+/// Shared build-time types for multi-backend assembly.
+pub mod types;
+
+pub use meta::{FsContextResolver, MetaStateStore};
diff --git a/crates/ragfs/src/multibackend/types.rs b/crates/ragfs/src/multibackend/types.rs
new file mode 100644
index 000000000..8e097ee4f
--- /dev/null
+++ b/crates/ragfs/src/multibackend/types.rs
@@ -0,0 +1,17 @@
+//! Shared build-time types for multi-backend assembly.
+
+/// Encryption context used while assembling one multi-backend mount.
+#[derive(Clone, Copy)]
+pub struct MultiBackendBuildContext {
+ /// Global root key, if server-side encryption is enabled.
+ pub enc_root_key: Option<[u8; 32]>,
+ /// Global provider type, if server-side encryption is enabled.
+ pub enc_provider_type: Option<u8>,
+}
+
+impl MultiBackendBuildContext {
+ /// Return whether global encryption is enabled for this mount.
+ pub fn global_encryption_enabled(&self) -> bool {
+ self.enc_root_key.is_some() && self.enc_provider_type.is_some()
+ }
+}
diff --git a/crates/ragfs/src/plugins/kvfs/mod.rs b/crates/ragfs/src/plugins/kvfs/mod.rs
index 9f4d7b9b9..e2b1775b0 100644
--- a/crates/ragfs/src/plugins/kvfs/mod.rs
+++ b/crates/ragfs/src/plugins/kvfs/mod.rs
@@ -67,11 +67,7 @@ impl KVFileSystem {
/// List all keys with a given prefix
fn list_keys_with_prefix(&self, store: &HashMap<String, KVEntry>, prefix: &str) -> Vec<String> {
- let search_prefix = if prefix == "/" {
- ""
- } else {
- prefix
- };
+ let search_prefix = if prefix == "/" { "" } else { prefix };
store
.keys()
@@ -450,41 +446,41 @@ KVFS has no configuration parameters.
mod tests {
use super::*;
+ /// Write one key in KVFS tests.
+ async fn write_key(fs: &KVFileSystem, path: &str, data: &[u8]) {
+ fs.write(path, data, 0, WriteFlag::Create).await.unwrap();
+ }
+
+ /// Read one key in KVFS tests.
+ async fn read_key(fs: &KVFileSystem, path: &str) -> Vec<u8> {
+ fs.read(path, 0, 0).await.unwrap()
+ }
+
#[tokio::test]
async fn test_kvfs_basic_operations() {
let fs = KVFileSystem::new();
// Create and write
- fs.write("/key1", b"value1", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_key(&fs, "/key1", b"value1").await;
// Read
- let data = fs.read("/key1", 0, 0).await.unwrap();
- assert_eq!(data, b"value1");
+ assert_eq!(read_key(&fs, "/key1").await, b"value1");
// Update
fs.write("/key1", b"value2", 0, WriteFlag::Truncate)
.await
.unwrap();
- let data = fs.read("/key1", 0, 0).await.unwrap();
- assert_eq!(data, b"value2");
+ assert_eq!(read_key(&fs, "/key1").await, b"value2");
}
#[tokio::test]
async fn test_kvfs_list_keys() {
let fs = KVFileSystem::new();
- fs.write("/key1", b"val1", 0, WriteFlag::Create)
- .await
- .unwrap();
- fs.write("/key2", b"val2", 0, WriteFlag::Create)
- .await
- .unwrap();
- fs.write("/key3", b"val3", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_key(&fs, "/key1", b"val1").await;
+ write_key(&fs, "/key2", b"val2").await;
+ write_key(&fs, "/key3", b"val3").await;
let entries = fs.read_dir("/").await.unwrap();
assert_eq!(entries.len(), 3);
@@ -497,12 +493,8 @@ mod tests {
// Create parent "directory" first
fs.mkdir("/user", 0o755).await.unwrap();
- fs.write("/user/123", b"alice", 0, WriteFlag::Create)
- .await
- .unwrap();
- fs.write("/user/456", b"bob", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_key(&fs, "/user/123", b"alice").await;
+ write_key(&fs, "/user/456", b"bob").await;
let entries = fs.read_dir("/user").await.unwrap();
assert_eq!(entries.len(), 2);
@@ -512,9 +504,7 @@ mod tests {
async fn test_kvfs_delete() {
let fs = KVFileSystem::new();
- fs.write("/key1", b"value1", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_key(&fs, "/key1", b"value1").await;
fs.remove("/key1").await.unwrap();
assert!(fs.read("/key1", 0, 0).await.is_err());
@@ -524,14 +514,11 @@ mod tests {
async fn test_kvfs_rename() {
let fs = KVFileSystem::new();
- fs.write("/oldkey", b"data", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_key(&fs, "/oldkey", b"data").await;
fs.rename("/oldkey", "/newkey").await.unwrap();
assert!(fs.read("/oldkey", 0, 0).await.is_err());
- let data = fs.read("/newkey", 0, 0).await.unwrap();
- assert_eq!(data, b"data");
+ assert_eq!(read_key(&fs, "/newkey").await, b"data");
}
#[tokio::test]
@@ -539,11 +526,7 @@ mod tests {
let plugin = KVFSPlugin;
assert_eq!(plugin.name(), "kvfs");
- let config = PluginConfig {
- name: "kvfs".to_string(),
- mount_path: "/kvfs".to_string(),
- params: HashMap::new(),
- };
+ let config = PluginConfig::single_backend("kvfs", "/kvfs", HashMap::new());
assert!(plugin.validate(&config).await.is_ok());
assert!(plugin.initialize(config).await.is_ok());
diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rs
index 4c98f9b4c..08cd2d6d3 100644
--- a/crates/ragfs/src/plugins/localfs/mod.rs
+++ b/crates/ragfs/src/plugins/localfs/mod.rs
@@ -793,9 +793,9 @@ impl FileSystem for LocalFileSystem {
// Get parent directory
if let Some(parent) = local_path.parent() {
// Create all parent directories if they don't exist
- tokio::fs::create_dir_all(parent)
- .await
- .map_err(|e| Error::plugin(format!("failed to create parent directories: {}", e)))?;
+ tokio::fs::create_dir_all(parent).await.map_err(|e| {
+ Error::plugin(format!("failed to create parent directories: {}", e))
+ })?;
}
Ok(())
@@ -869,6 +869,8 @@ impl FileSystem for LocalFileSystem {
#[cfg(test)]
mod tests {
use super::*;
+ use std::path::Path;
+ use tempfile::TempDir;
struct EnvVarGuard {
key: &'static str,
@@ -893,19 +895,52 @@ mod tests {
}
}
- #[tokio::test]
- async fn test_localfs_grep_fallback_respects_gitignore_and_hidden() {
- use tempfile::TempDir;
-
+ /// Create a LocalFS fixture pinned to the fallback grep implementation.
+ fn fallback_localfs() -> (TempDir, LocalFileSystem) {
let dir = TempDir::new().unwrap();
- std::fs::write(dir.path().join(".gitignore"), "ignored.txt\n").unwrap();
- std::fs::write(dir.path().join("a.txt"), "hello\n").unwrap();
- std::fs::write(dir.path().join("ignored.txt"), "hello\n").unwrap();
- std::fs::create_dir_all(dir.path().join(".hidden")).unwrap();
- std::fs::write(dir.path().join(".hidden/secret.txt"), "hello\n").unwrap();
-
let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
fs.has_rg = false;
+ (dir, fs)
+ }
+
+ /// Write a test file, creating parent directories when needed.
+ fn write_file(root: &Path, path: &str, content: &str) {
+ let path = root.join(path);
+ if let Some(parent) = path.parent() {
+ std::fs::create_dir_all(parent).unwrap();
+ }
+ std::fs::write(path, content).unwrap();
+ }
+
+ /// Install a fake rg executable and prepend it to PATH for the current test.
+ #[cfg(unix)]
+ fn install_fake_rg(dir: &TempDir, script: &str) -> EnvVarGuard {
+ let bin_dir = dir.path().join("bin");
+ std::fs::create_dir_all(&bin_dir).unwrap();
+ let rg_path = bin_dir.join("rg");
+ std::fs::write(&rg_path, script).unwrap();
+
+ use std::os::unix::fs::PermissionsExt;
+ let mut perms = std::fs::metadata(&rg_path).unwrap().permissions();
+ perms.set_mode(0o755);
+ std::fs::set_permissions(&rg_path, perms).unwrap();
+
+ let old_path = std::env::var_os("PATH").unwrap_or_default();
+ let mut new_path = std::ffi::OsString::new();
+ new_path.push(bin_dir.as_os_str());
+ new_path.push(std::ffi::OsStr::new(":"));
+ new_path.push(old_path);
+ EnvVarGuard::set("PATH", &new_path)
+ }
+
+ #[tokio::test]
+ async fn test_localfs_grep_fallback_respects_gitignore_and_hidden() {
+ let (dir, fs) = fallback_localfs();
+ write_file(dir.path(), ".gitignore", "ignored.txt\n");
+ write_file(dir.path(), "a.txt", "hello\n");
+ write_file(dir.path(), "ignored.txt", "hello\n");
+ std::fs::create_dir_all(dir.path().join(".hidden")).unwrap();
+ write_file(dir.path(), ".hidden/secret.txt", "hello\n");
let result = fs
.grep("/", "hello", true, false, None, None, None)
@@ -918,13 +953,8 @@ mod tests {
#[tokio::test]
async fn test_localfs_grep_case_insensitive() {
- use tempfile::TempDir;
-
- let dir = TempDir::new().unwrap();
- std::fs::write(dir.path().join("a.txt"), "HELLO\n").unwrap();
-
- let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
- fs.has_rg = false;
+ let (dir, fs) = fallback_localfs();
+ write_file(dir.path(), "a.txt", "HELLO\n");
let result = fs
.grep("/", "hello", true, true, None, None, None)
@@ -936,11 +966,7 @@ mod tests {
#[tokio::test]
async fn test_localfs_grep_missing_path_returns_not_found() {
- use tempfile::TempDir;
-
- let dir = TempDir::new().unwrap();
- let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
- fs.has_rg = false;
+ let (_dir, fs) = fallback_localfs();
let err = fs
.grep("/does-not-exist", "hello", true, false, None, None, None)
@@ -951,15 +977,13 @@ mod tests {
#[tokio::test]
async fn test_localfs_grep_does_not_inherit_parent_gitignore() {
- use tempfile::TempDir;
-
let dir = TempDir::new().unwrap();
let mount_dir = dir.path().join("mount");
std::fs::create_dir_all(&mount_dir).unwrap();
// Parent .gitignore ignores the mounted directory entirely.
- std::fs::write(dir.path().join(".gitignore"), "mount/\n").unwrap();
- std::fs::write(mount_dir.join("a.txt"), "hello\n").unwrap();
+ write_file(dir.path(), ".gitignore", "mount/\n");
+ write_file(&mount_dir, "a.txt", "hello\n");
let mut fs = LocalFileSystem::new(mount_dir.to_str().unwrap()).unwrap();
fs.has_rg = false;
@@ -974,14 +998,8 @@ mod tests {
#[tokio::test]
async fn test_localfs_grep_returns_query_root_relative_paths() {
- use tempfile::TempDir;
-
- let dir = TempDir::new().unwrap();
- std::fs::create_dir_all(dir.path().join("sub")).unwrap();
- std::fs::write(dir.path().join("sub/a.txt"), "hello\n").unwrap();
-
- let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
- fs.has_rg = false;
+ let (dir, fs) = fallback_localfs();
+ write_file(dir.path(), "sub/a.txt", "hello\n");
let result = fs
.grep("/sub", "hello", true, false, None, None, None)
@@ -1001,29 +1019,12 @@ mod tests {
#[cfg(unix)]
#[tokio::test]
async fn test_localfs_grep_node_limit_zero_does_not_invoke_rg() {
- use tempfile::TempDir;
-
let dir = TempDir::new().unwrap();
- std::fs::write(dir.path().join("a.txt"), "hello\n").unwrap();
+ write_file(dir.path(), "a.txt", "hello\n");
// Create a fake `rg` that fails if invoked. The expected behavior is to short-circuit
// and return empty result when node_limit=0.
- let bin_dir = dir.path().join("bin");
- std::fs::create_dir_all(&bin_dir).unwrap();
- let rg_path = bin_dir.join("rg");
- std::fs::write(&rg_path, "#!/bin/sh\nexit 2\n").unwrap();
- use std::os::unix::fs::PermissionsExt;
- let mut perms = std::fs::metadata(&rg_path).unwrap().permissions();
- perms.set_mode(0o755);
- std::fs::set_permissions(&rg_path, perms).unwrap();
-
- // Prepend fake rg to PATH.
- let old_path = std::env::var_os("PATH").unwrap_or_default();
- let mut new_path = std::ffi::OsString::new();
- new_path.push(bin_dir.as_os_str());
- new_path.push(std::ffi::OsStr::new(":"));
- new_path.push(old_path);
- let _path_guard = EnvVarGuard::set("PATH", &new_path);
+ let _path_guard = install_fake_rg(&dir, "#!/bin/sh\nexit 2\n");
let fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
let result = fs
@@ -1036,16 +1037,9 @@ mod tests {
#[tokio::test]
async fn test_localfs_grep_exclude_path_applies_before_node_limit() {
- use tempfile::TempDir;
-
- let dir = TempDir::new().unwrap();
- std::fs::create_dir_all(dir.path().join("excluded")).unwrap();
- std::fs::create_dir_all(dir.path().join("ok")).unwrap();
- std::fs::write(dir.path().join("excluded/x.txt"), "hit\n").unwrap();
- std::fs::write(dir.path().join("ok/y.txt"), "hit\n").unwrap();
-
- let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
- fs.has_rg = false;
+ let (dir, fs) = fallback_localfs();
+ write_file(dir.path(), "excluded/x.txt", "hit\n");
+ write_file(dir.path(), "ok/y.txt", "hit\n");
let out = fs
.grep("/", "hit", true, false, Some(1), Some("/excluded"), None)
@@ -1059,17 +1053,10 @@ mod tests {
#[tokio::test]
async fn test_localfs_grep_exclude_path_is_query_root_relative_for_nested_query_root() {
- use tempfile::TempDir;
-
- let dir = TempDir::new().unwrap();
+ let (dir, fs) = fallback_localfs();
let nested_root = dir.path().join("acct/resources/docs");
- std::fs::create_dir_all(nested_root.join("excluded")).unwrap();
- std::fs::create_dir_all(nested_root.join("ok")).unwrap();
- std::fs::write(nested_root.join("excluded/x.txt"), "hit\n").unwrap();
- std::fs::write(nested_root.join("ok/y.txt"), "hit\n").unwrap();
-
- let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
- fs.has_rg = false;
+ write_file(&nested_root, "excluded/x.txt", "hit\n");
+ write_file(&nested_root, "ok/y.txt", "hit\n");
// Simulate MountableFS -> LocalFS: query root and exclude path are plugin-relative and nested.
let out = fs
@@ -1093,37 +1080,17 @@ mod tests {
#[cfg(unix)]
#[tokio::test]
async fn test_localfs_grep_rg_fast_path_disables_parent_ignore_inheritance() {
- use tempfile::TempDir;
-
let dir = TempDir::new().unwrap();
- std::fs::write(dir.path().join(".gitignore"), "mount/\n").unwrap();
+ write_file(dir.path(), ".gitignore", "mount/\n");
let mount_dir = dir.path().join("mount");
- std::fs::create_dir_all(&mount_dir).unwrap();
- std::fs::write(mount_dir.join("a.txt"), "hello\n").unwrap();
+ write_file(&mount_dir, "a.txt", "hello\n");
// Fake `rg` verifies `--no-ignore-parent` is present, then emits one JSON match.
- let bin_dir = dir.path().join("bin");
- std::fs::create_dir_all(&bin_dir).unwrap();
- let rg_path = bin_dir.join("rg");
- std::fs::write(
- &rg_path,
- format!(
+ let script = format!(
"#!/bin/sh\nfor arg in \"$@\"; do\n if [ \"$arg\" = \"--no-ignore-parent\" ]; then\n printf '%s\\n' '{{\"type\":\"match\",\"data\":{{\"path\":{{\"text\":\"a.txt\"}},\"line_number\":1,\"lines\":{{\"text\":\"hello\\\\n\"}}}}}}'\n exit 0\n fi\ndone\necho missing --no-ignore-parent 1>&2\nexit 2\n"
- ),
- )
- .unwrap();
- use std::os::unix::fs::PermissionsExt;
- let mut perms = std::fs::metadata(&rg_path).unwrap().permissions();
- perms.set_mode(0o755);
- std::fs::set_permissions(&rg_path, perms).unwrap();
-
- let old_path = std::env::var_os("PATH").unwrap_or_default();
- let mut new_path = std::ffi::OsString::new();
- new_path.push(bin_dir.as_os_str());
- new_path.push(std::ffi::OsStr::new(":"));
- new_path.push(old_path);
- let _path_guard = EnvVarGuard::set("PATH", &new_path);
+ );
+ let _path_guard = install_fake_rg(&dir, &script);
let fs = LocalFileSystem::new(mount_dir.to_str().unwrap()).unwrap();
let result = fs
diff --git a/crates/ragfs/src/plugins/memfs/mod.rs b/crates/ragfs/src/plugins/memfs/mod.rs
index 3d9757a73..444fa36c2 100644
--- a/crates/ragfs/src/plugins/memfs/mod.rs
+++ b/crates/ragfs/src/plugins/memfs/mod.rs
@@ -65,10 +65,7 @@ impl MemFileSystem {
let mut entries = HashMap::new();
// Create root directory
- entries.insert(
- "/".to_string(),
- FileEntry::new_dir(0o755),
- );
+ entries.insert("/".to_string(), FileEntry::new_dir(0o755));
Self {
entries: Arc::new(RwLock::new(entries)),
@@ -117,11 +114,7 @@ impl MemFileSystem {
}
let normalized = Self::normalize_path(path);
- normalized
- .split('/')
- .last()
- .unwrap_or("")
- .to_string()
+ normalized.split('/').last().unwrap_or("").to_string()
}
/// List entries in a directory
@@ -643,11 +636,7 @@ mod tests {
let plugin = MemFSPlugin;
assert_eq!(plugin.name(), "memfs");
- let config = PluginConfig {
- name: "memfs".to_string(),
- mount_path: "/memfs".to_string(),
- params: HashMap::new(),
- };
+ let config = PluginConfig::single_backend("memfs", "/memfs", HashMap::new());
assert!(plugin.validate(&config).await.is_ok());
assert!(plugin.initialize(config).await.is_ok());
diff --git a/crates/ragfs/src/plugins/queuefs/backend.rs b/crates/ragfs/src/plugins/queuefs/backend.rs
index 49568bcbe..253c2281a 100644
--- a/crates/ragfs/src/plugins/queuefs/backend.rs
+++ b/crates/ragfs/src/plugins/queuefs/backend.rs
@@ -9,11 +9,10 @@ use rusqlite::{params, types::ValueRef, Connection, Row};
use serde::{Deserialize, Deserializer, Serialize};
use std::collections::{HashMap, VecDeque};
use std::sync::Mutex;
-use std::time::{Duration, UNIX_EPOCH};
use std::time::SystemTime;
+use std::time::{Duration, UNIX_EPOCH};
use uuid::Uuid;
-
/// A message in the queue
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Message {
@@ -220,7 +219,10 @@ impl MemoryBackend {
impl QueueBackend for MemoryBackend {
fn create_queue(&mut self, name: &str) -> Result<()> {
if self.queues.contains_key(name) {
- return Err(Error::AlreadyExists(format!("queue '{}' already exists", name)));
+ return Err(Error::AlreadyExists(format!(
+ "queue '{}' already exists",
+ name
+ )));
}
self.queues.insert(name.to_string(), Queue::new());
Ok(())
@@ -250,9 +252,10 @@ impl QueueBackend for MemoryBackend {
}
fn enqueue(&mut self, queue_name: &str, msg: Message) -> Result<()> {
- let queue = self.queues.get_mut(queue_name).ok_or_else(|| {
- Error::NotFound(format!("queue '{}' not found", queue_name))
- })?;
+ let queue = self
+ .queues
+ .get_mut(queue_name)
+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
queue.last_enqueue_time = SystemTime::now();
queue.messages.push_back(msg);
@@ -260,50 +263,56 @@ impl QueueBackend for MemoryBackend {
}
fn dequeue(&mut self, queue_name: &str) -> Result<Option<Message>> {
- let queue = self.queues.get_mut(queue_name).ok_or_else(|| {
- Error::NotFound(format!("queue '{}' not found", queue_name))
- })?;
+ let queue = self
+ .queues
+ .get_mut(queue_name)
+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
Ok(queue.messages.pop_front())
}
fn peek(&self, queue_name: &str) -> Result<Option<Message>> {
- let queue = self.queues.get(queue_name).ok_or_else(|| {
- Error::NotFound(format!("queue '{}' not found", queue_name))
- })?;
+ let queue = self
+ .queues
+ .get(queue_name)
+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
Ok(queue.messages.front().cloned())
}
fn size(&self, queue_name: &str) -> Result<usize> {
- let queue = self.queues.get(queue_name).ok_or_else(|| {
- Error::NotFound(format!("queue '{}' not found", queue_name))
- })?;
+ let queue = self
+ .queues
+ .get(queue_name)
+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
Ok(queue.messages.len())
}
fn clear(&mut self, queue_name: &str) -> Result<()> {
- let queue = self.queues.get_mut(queue_name).ok_or_else(|| {
- Error::NotFound(format!("queue '{}' not found", queue_name))
- })?;
+ let queue = self
+ .queues
+ .get_mut(queue_name)
+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
queue.messages.clear();
Ok(())
}
fn get_last_enqueue_time(&self, queue_name: &str) -> Result<SystemTime> {
- let queue = self.queues.get(queue_name).ok_or_else(|| {
- Error::NotFound(format!("queue '{}' not found", queue_name))
- })?;
+ let queue = self
+ .queues
+ .get(queue_name)
+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
Ok(queue.last_enqueue_time)
}
fn ack(&mut self, queue_name: &str, msg_id: &str) -> Result<bool> {
- let queue = self.queues.get_mut(queue_name).ok_or_else(|| {
- Error::NotFound(format!("queue '{}' not found", queue_name))
- })?;
+ let queue = self
+ .queues
+ .get_mut(queue_name)
+ .ok_or_else(|| Error::NotFound(format!("queue '{}' not found", queue_name)))?;
// Find and remove message by ID
let original_len = queue.messages.len();
@@ -397,7 +406,12 @@ impl SQLiteQueueBackend {
WHERE queue_name IS NOT NULL AND queue_name != ''",
[],
)
- .map_err(|e| Error::internal(format!("sqlite migration backfill queue metadata error: {}", e)))?;
+ .map_err(|e| {
+ Error::internal(format!(
+ "sqlite migration backfill queue metadata error: {}",
+ e
+ ))
+ })?;
Ok(())
}
@@ -472,7 +486,10 @@ impl QueueBackend for SQLiteQueueBackend {
)
.map_err(|e| Error::internal(format!("sqlite create queue error: {}", e)))?;
if changed == 0 {
- return Err(Error::AlreadyExists(format!("queue '{}' already exists", name)));
+ return Err(Error::AlreadyExists(format!(
+ "queue '{}' already exists",
+ name
+ )));
}
Ok(())
}
@@ -518,10 +535,11 @@ impl QueueBackend for SQLiteQueueBackend {
};
if prefix.is_empty() {
- let mut stmt = match conn.prepare("SELECT queue_name FROM queue_metadata ORDER BY queue_name") {
- Ok(stmt) => stmt,
- Err(_) => return Vec::new(),
- };
+ let mut stmt =
+ match conn.prepare("SELECT queue_name FROM queue_metadata ORDER BY queue_name") {
+ Ok(stmt) => stmt,
+ Err(_) => return Vec::new(),
+ };
let rows = match stmt.query_map([], |row| row.get::<_, String>(0)) {
Ok(rows) => rows,
Err(_) => return Vec::new(),
@@ -585,7 +603,12 @@ impl QueueBackend for SQLiteQueueBackend {
let (id, raw_data) = match row {
Ok(row) => row,
Err(rusqlite::Error::QueryReturnedNoRows) => return Ok(None),
- Err(e) => return Err(Error::internal(format!("sqlite dequeue query error: {}", e))),
+ Err(e) => {
+ return Err(Error::internal(format!(
+ "sqlite dequeue query error: {}",
+ e
+ )))
+ }
};
tx.execute(
@@ -711,6 +734,24 @@ impl QueueBackend for SQLiteQueueBackend {
mod tests {
use super::*;
use tempfile::tempdir;
+ use tempfile::TempDir;
+
+ /// Create an in-memory backend with a test queue.
+ fn memory_backend_with_queue(queue: &str) -> MemoryBackend {
+ let mut backend = MemoryBackend::new();
+ backend.create_queue(queue).unwrap();
+ backend
+ }
+
+ /// Create a SQLite backend backed by a temporary database file.
+ fn sqlite_backend() -> (TempDir, String, SQLiteQueueBackend) {
+ let dir = tempdir().unwrap();
+ let db_path = dir.path().join("queue.db");
+ let db_path_str = db_path.to_str().unwrap().to_string();
+ let backend =
+ SQLiteQueueBackend::open(&db_path_str, SQLiteQueueOptions::default()).unwrap();
+ (dir, db_path_str, backend)
+ }
#[test]
fn test_create_queue() {
@@ -726,9 +767,7 @@ mod tests {
#[test]
fn test_remove_queue() {
- let mut backend = MemoryBackend::new();
-
- backend.create_queue("test").unwrap();
+ let mut backend = memory_backend_with_queue("test");
backend.remove_queue("test").unwrap();
assert!(!backend.queue_exists("test"));
@@ -755,8 +794,7 @@ mod tests {
#[test]
fn test_enqueue_dequeue() {
- let mut backend = MemoryBackend::new();
- backend.create_queue("test").unwrap();
+ let mut backend = memory_backend_with_queue("test");
let msg1 = Message::new(b"message 1".to_vec());
let msg2 = Message::new(b"message 2".to_vec());
@@ -778,8 +816,7 @@ mod tests {
#[test]
fn test_peek() {
- let mut backend = MemoryBackend::new();
- backend.create_queue("test").unwrap();
+ let mut backend = memory_backend_with_queue("test");
let msg = Message::new(b"test message".to_vec());
backend.enqueue("test", msg.clone()).unwrap();
@@ -796,11 +833,14 @@ mod tests {
#[test]
fn test_clear() {
- let mut backend = MemoryBackend::new();
- backend.create_queue("test").unwrap();
+ let mut backend = memory_backend_with_queue("test");
- backend.enqueue("test", Message::new(b"msg1".to_vec())).unwrap();
- backend.enqueue("test", Message::new(b"msg2".to_vec())).unwrap();
+ backend
+ .enqueue("test", Message::new(b"msg1".to_vec()))
+ .unwrap();
+ backend
+ .enqueue("test", Message::new(b"msg2".to_vec()))
+ .unwrap();
assert_eq!(backend.size("test").unwrap(), 2);
@@ -814,8 +854,12 @@ mod tests {
backend.create_queue("queue1").unwrap();
backend.create_queue("queue2").unwrap();
- backend.enqueue("queue1", Message::new(b"msg1".to_vec())).unwrap();
- backend.enqueue("queue2", Message::new(b"msg2".to_vec())).unwrap();
+ backend
+ .enqueue("queue1", Message::new(b"msg1".to_vec()))
+ .unwrap();
+ backend
+ .enqueue("queue2", Message::new(b"msg2".to_vec()))
+ .unwrap();
assert_eq!(backend.size("queue1").unwrap(), 1);
assert_eq!(backend.size("queue2").unwrap(), 1);
@@ -831,7 +875,9 @@ mod tests {
fn test_operations_on_nonexistent_queue() {
let mut backend = MemoryBackend::new();
- assert!(backend.enqueue("nonexistent", Message::new(b"data".to_vec())).is_err());
+ assert!(backend
+ .enqueue("nonexistent", Message::new(b"data".to_vec()))
+ .is_err());
assert!(backend.dequeue("nonexistent").is_err());
assert!(backend.peek("nonexistent").is_err());
assert!(backend.size("nonexistent").is_err());
@@ -840,11 +886,7 @@ mod tests {
#[test]
fn test_sqlite_backend_basic_flow() {
- let dir = tempdir().unwrap();
- let db_path = dir.path().join("queue.db");
- let mut backend =
- SQLiteQueueBackend::open(db_path.to_str().unwrap(), SQLiteQueueOptions::default())
- .unwrap();
+ let (_dir, _db_path, mut backend) = sqlite_backend();
backend.create_queue("test").unwrap();
let msg1 = Message::new(b"message 1".to_vec());
@@ -865,15 +907,13 @@ mod tests {
#[test]
fn test_sqlite_backend_preserves_non_utf8_payload_bytes() {
- let dir = tempdir().unwrap();
- let db_path = dir.path().join("queue.db");
- let mut backend =
- SQLiteQueueBackend::open(db_path.to_str().unwrap(), SQLiteQueueOptions::default())
- .unwrap();
+ let (_dir, _db_path, mut backend) = sqlite_backend();
backend.create_queue("test").unwrap();
let payload = vec![0xff, 0x00, 0x80, b'a'];
- backend.enqueue("test", Message::new(payload.clone())).unwrap();
+ backend
+ .enqueue("test", Message::new(payload.clone()))
+ .unwrap();
let peeked = backend.peek("test").unwrap().unwrap();
assert_eq!(peeked.data, payload);
@@ -884,9 +924,7 @@ mod tests {
#[test]
fn test_sqlite_backend_recover_stale() {
- let dir = tempdir().unwrap();
- let db_path = dir.path().join("queue.db");
- let db_path_str = db_path.to_str().unwrap().to_string();
+ let (_dir, db_path_str, _backend) = sqlite_backend();
let msg_id = {
let mut backend =
@@ -909,12 +947,8 @@ mod tests {
#[test]
fn test_sqlite_backend_dequeue_legacy_go_row() {
- let dir = tempdir().unwrap();
- let db_path = dir.path().join("queue.db");
- let db_path_str = db_path.to_str().unwrap().to_string();
+ let (_dir, db_path_str, mut backend) = sqlite_backend();
- let mut backend =
- SQLiteQueueBackend::open(&db_path_str, SQLiteQueueOptions::default()).unwrap();
backend.create_queue("Semantic").unwrap();
drop(backend);
@@ -997,13 +1031,11 @@ mod tests {
#[test]
fn test_sqlite_backend_operations_on_nonexistent_queue() {
- let dir = tempdir().unwrap();
- let db_path = dir.path().join("queue.db");
- let mut backend =
- SQLiteQueueBackend::open(db_path.to_str().unwrap(), SQLiteQueueOptions::default())
- .unwrap();
+ let (_dir, _db_path, mut backend) = sqlite_backend();
- assert!(backend.enqueue("nonexistent", Message::new(b"data".to_vec())).is_err());
+ assert!(backend
+ .enqueue("nonexistent", Message::new(b"data".to_vec()))
+ .is_err());
assert!(backend.dequeue("nonexistent").is_err());
assert!(backend.peek("nonexistent").is_err());
assert!(backend.size("nonexistent").is_err());
diff --git a/crates/ragfs/src/plugins/queuefs/mod.rs b/crates/ragfs/src/plugins/queuefs/mod.rs
index 2e69e86c4..12724272a 100644
--- a/crates/ragfs/src/plugins/queuefs/mod.rs
+++ b/crates/ragfs/src/plugins/queuefs/mod.rs
@@ -199,9 +199,7 @@ impl FileSystem for QueueFileSystem {
async fn mkdir(&self, path: &str, _mode: u32) -> Result<()> {
let parsed = Self::parse_queue_path(path)?;
if !parsed.is_dir {
- return Err(Error::InvalidOperation(
- "not a directory path".to_string(),
- ));
+ return Err(Error::InvalidOperation("not a directory path".to_string()));
}
if let Some(queue_name) = parsed.queue_name {
self.backend.lock().await.create_queue(&queue_name)?;
@@ -260,13 +258,7 @@ impl FileSystem for QueueFileSystem {
}
}
- async fn write(
- &self,
- path: &str,
- data: &[u8],
- _offset: u64,
- _flags: WriteFlag,
- ) -> Result<u64> {
+ async fn write(&self, path: &str, data: &[u8], _offset: u64, _flags: WriteFlag) -> Result<u64> {
let parsed = Self::parse_queue_path(path)?;
let queue_name = parsed
@@ -369,10 +361,7 @@ impl FileSystem for QueueFileSystem {
// Leaf queue: return control files
if !backend.queue_exists(&queue_name) {
- return Err(Error::NotFound(format!(
- "queue not found: {}",
- queue_name
- )));
+ return Err(Error::NotFound(format!("queue not found: {}", queue_name)));
}
Ok(Self::leaf_control_files(now))
@@ -399,7 +388,11 @@ impl FileSystem for QueueFileSystem {
let queue_name = parsed.queue_name.unwrap();
if backend.queue_exists(&queue_name) {
Ok(FileInfo {
- name: queue_name.split('/').last().unwrap_or(&queue_name).to_string(),
+ name: queue_name
+ .split('/')
+ .last()
+ .unwrap_or(&queue_name)
+ .to_string(),
size: 0,
mode: 0o755,
mod_time: SystemTime::now(),
@@ -414,8 +407,9 @@ impl FileSystem for QueueFileSystem {
Ok(FileInfo {
name: operation.clone(),
size: 0,
- mode: Self::control_file_mode(operation)
- .ok_or_else(|| Error::NotFound(format!("control file not found: {}", operation)))?,
+ mode: Self::control_file_mode(operation).ok_or_else(|| {
+ Error::NotFound(format!("control file not found: {}", operation))
+ })?,
mod_time: SystemTime::now(),
is_dir: false,
})
@@ -444,9 +438,7 @@ impl FileSystem for QueueFileSystem {
let parsed = Self::parse_queue_path(path)?;
if !parsed.is_dir {
- return Err(Error::InvalidOperation(
- "not a directory".to_string(),
- ));
+ return Err(Error::InvalidOperation("not a directory".to_string()));
}
if let Some(queue_name) = parsed.queue_name {
@@ -665,32 +657,48 @@ mod tests {
data: String,
}
- #[tokio::test]
- async fn test_queuefs_enqueue_dequeue() {
+ /// Create a queue filesystem with one initialized queue.
+ async fn queuefs_with(queue: &str) -> QueueFileSystem {
let fs = QueueFileSystem::new();
+ fs.mkdir(&format!("/{queue}"), 0o755).await.unwrap();
+ fs
+ }
- // Create a queue first
- fs.mkdir("/test", 0o755).await.unwrap();
+ /// Enqueue one message into a test queue.
+ async fn enqueue(fs: &dyn FileSystem, queue: &str, data: &[u8]) {
+ fs.write(&format!("/{queue}/enqueue"), data, 0, WriteFlag::None)
+ .await
+ .unwrap();
+ }
+
+ /// Dequeue and deserialize one message from a test queue.
+ async fn dequeue_msg(fs: &dyn FileSystem, queue: &str) -> TestQueueMessage {
+ let result = fs.read(&format!("/{queue}/dequeue"), 0, 0).await.unwrap();
+ serde_json::from_slice(&result).unwrap()
+ }
+
+ /// Read the queue size control file as usize.
+ async fn queue_size(fs: &dyn FileSystem, queue: &str) -> usize {
+ let size = fs.read(&format!("/{queue}/size"), 0, 0).await.unwrap();
+ String::from_utf8(size).unwrap().parse().unwrap()
+ }
+
+ #[tokio::test]
+ async fn test_queuefs_enqueue_dequeue() {
+ let fs = queuefs_with("test").await;
// Enqueue messages
let data1 = b"message 1";
let data2 = b"message 2";
-
- fs.write("/test/enqueue", data1, 0, WriteFlag::None)
- .await
- .unwrap();
- fs.write("/test/enqueue", data2, 0, WriteFlag::None)
- .await
- .unwrap();
+ enqueue(&fs, "test", data1).await;
+ enqueue(&fs, "test", data2).await;
// Dequeue messages
- let result1 = fs.read("/test/dequeue", 0, 0).await.unwrap();
- let msg1: TestQueueMessage = serde_json::from_slice(&result1).unwrap();
+ let msg1 = dequeue_msg(&fs, "test").await;
assert!(!msg1.id.is_empty());
assert_eq!(msg1.data.as_bytes(), data1);
- let result2 = fs.read("/test/dequeue", 0, 0).await.unwrap();
- let msg2: TestQueueMessage = serde_json::from_slice(&result2).unwrap();
+ let msg2 = dequeue_msg(&fs, "test").await;
assert!(!msg2.id.is_empty());
assert_eq!(msg2.data.as_bytes(), data2);
@@ -701,16 +709,11 @@ mod tests {
#[tokio::test]
async fn test_queuefs_peek() {
- let fs = QueueFileSystem::new();
-
- // Create a queue first
- fs.mkdir("/test", 0o755).await.unwrap();
+ let fs = queuefs_with("test").await;
// Enqueue a message
let data = b"test message";
- fs.write("/test/enqueue", data, 0, WriteFlag::None)
- .await
- .unwrap();
+ enqueue(&fs, "test", data).await;
// Peek should return the message without removing it
let result1 = fs.read("/test/peek", 0, 0).await.unwrap();
@@ -722,54 +725,36 @@ mod tests {
assert_eq!(msg2.data.as_bytes(), data);
// Dequeue should still work
- let result3 = fs.read("/test/dequeue", 0, 0).await.unwrap();
- let msg3: TestQueueMessage = serde_json::from_slice(&result3).unwrap();
+ let msg3 = dequeue_msg(&fs, "test").await;
assert_eq!(msg3.data.as_bytes(), data);
}
#[tokio::test]
async fn test_queuefs_size() {
- let fs = QueueFileSystem::new();
-
- // Create a queue first
- fs.mkdir("/test", 0o755).await.unwrap();
+ let fs = queuefs_with("test").await;
// Initially empty
- let size = fs.read("/test/size", 0, 0).await.unwrap();
- assert_eq!(String::from_utf8(size).unwrap(), "0");
+ assert_eq!(queue_size(&fs, "test").await, 0);
// Add messages
- fs.write("/test/enqueue", b"msg1", 0, WriteFlag::None)
- .await
- .unwrap();
- fs.write("/test/enqueue", b"msg2", 0, WriteFlag::None)
- .await
- .unwrap();
+ enqueue(&fs, "test", b"msg1").await;
+ enqueue(&fs, "test", b"msg2").await;
- let size = fs.read("/test/size", 0, 0).await.unwrap();
- assert_eq!(String::from_utf8(size).unwrap(), "2");
+ assert_eq!(queue_size(&fs, "test").await, 2);
// Dequeue one
fs.read("/test/dequeue", 0, 0).await.unwrap();
- let size = fs.read("/test/size", 0, 0).await.unwrap();
- assert_eq!(String::from_utf8(size).unwrap(), "1");
+ assert_eq!(queue_size(&fs, "test").await, 1);
}
#[tokio::test]
async fn test_queuefs_clear() {
- let fs = QueueFileSystem::new();
-
- // Create a queue first
- fs.mkdir("/test", 0o755).await.unwrap();
+ let fs = queuefs_with("test").await;
// Add messages
- fs.write("/test/enqueue", b"msg1", 0, WriteFlag::None)
- .await
- .unwrap();
- fs.write("/test/enqueue", b"msg2", 0, WriteFlag::None)
- .await
- .unwrap();
+ enqueue(&fs, "test", b"msg1").await;
+ enqueue(&fs, "test", b"msg2").await;
// Clear the queue
fs.write("/test/clear", b"", 0, WriteFlag::None)
@@ -777,8 +762,7 @@ mod tests {
.unwrap();
// Queue should be empty
- let size = fs.read("/test/size", 0, 0).await.unwrap();
- assert_eq!(String::from_utf8(size).unwrap(), "0");
+ assert_eq!(queue_size(&fs, "test").await, 0);
let result = fs.read("/test/dequeue", 0, 0).await;
assert_eq!(result.unwrap(), b"{}");
@@ -786,10 +770,7 @@ mod tests {
#[tokio::test]
async fn test_queuefs_read_dir() {
- let fs = QueueFileSystem::new();
-
- // Create a queue
- fs.mkdir("/test", 0o755).await.unwrap();
+ let fs = queuefs_with("test").await;
// Root should list the queue
let entries = fs.read_dir("/").await.unwrap();
@@ -812,10 +793,7 @@ mod tests {
#[tokio::test]
async fn test_queuefs_stat() {
- let fs = QueueFileSystem::new();
-
- // Create a queue
- fs.mkdir("/test", 0o755).await.unwrap();
+ let fs = queuefs_with("test").await;
// Stat root
let info = fs.stat("/").await.unwrap();
@@ -837,10 +815,7 @@ mod tests {
#[tokio::test]
async fn test_queuefs_invalid_operations() {
- let fs = QueueFileSystem::new();
-
- // Create a queue
- fs.mkdir("/test", 0o755).await.unwrap();
+ let fs = queuefs_with("test").await;
// Cannot read from enqueue
let result = fs.read("/test/enqueue", 0, 0).await;
@@ -886,8 +861,7 @@ mod tests {
}
// Check size
- let size = fs.read("/test/size", 0, 0).await.unwrap();
- assert_eq!(String::from_utf8(size).unwrap(), "10");
+ assert_eq!(queue_size(fs.as_ref(), "test").await, 10);
// Dequeue all messages
for _ in 0..10 {
@@ -895,8 +869,7 @@ mod tests {
}
// Queue should be empty
- let size = fs.read("/test/size", 0, 0).await.unwrap();
- assert_eq!(String::from_utf8(size).unwrap(), "0");
+ assert_eq!(queue_size(fs.as_ref(), "test").await, 0);
}
#[tokio::test]
@@ -907,11 +880,8 @@ mod tests {
assert!(!plugin.readme().is_empty());
assert_eq!(plugin.config_params().len(), 4);
- let config = PluginConfig {
- name: "queuefs".to_string(),
- mount_path: "/queue".to_string(),
- params: std::collections::HashMap::new(),
- };
+ let config =
+ PluginConfig::single_backend("queuefs", "/queue", std::collections::HashMap::new());
plugin.validate(&config).await.unwrap();
let fs = plugin.initialize(config).await.unwrap();
@@ -920,11 +890,8 @@ mod tests {
fs.mkdir("/test", 0o755).await.unwrap();
// Test basic operation
- fs.write("/test/enqueue", b"test", 0, WriteFlag::None)
- .await
- .unwrap();
- let result = fs.read("/test/dequeue", 0, 0).await.unwrap();
- let msg: TestQueueMessage = serde_json::from_slice(&result).unwrap();
+ enqueue(fs.as_ref(), "test", b"test").await;
+ let msg = dequeue_msg(fs.as_ref(), "test").await;
assert_eq!(msg.data, "test");
}
@@ -937,27 +904,19 @@ mod tests {
fs.mkdir("/Semantic", 0o755).await.unwrap();
// Enqueue to both
- fs.write("/Embedding/enqueue", b"embed1", 0, WriteFlag::None)
- .await
- .unwrap();
- fs.write("/Semantic/enqueue", b"semantic1", 0, WriteFlag::None)
- .await
- .unwrap();
+ enqueue(&fs, "Embedding", b"embed1").await;
+ enqueue(&fs, "Semantic", b"semantic1").await;
// Verify isolation
- let size1 = fs.read("/Embedding/size", 0, 0).await.unwrap();
- let size2 = fs.read("/Semantic/size", 0, 0).await.unwrap();
- assert_eq!(String::from_utf8(size1).unwrap(), "1");
- assert_eq!(String::from_utf8(size2).unwrap(), "1");
+ assert_eq!(queue_size(&fs, "Embedding").await, 1);
+ assert_eq!(queue_size(&fs, "Semantic").await, 1);
// Dequeue from specific queue
- let msg = fs.read("/Embedding/dequeue", 0, 0).await.unwrap();
- let msg: TestQueueMessage = serde_json::from_slice(&msg).unwrap();
+ let msg = dequeue_msg(&fs, "Embedding").await;
assert_eq!(msg.data, "embed1");
// Other queue unaffected
- let size2 = fs.read("/Semantic/size", 0, 0).await.unwrap();
- assert_eq!(String::from_utf8(size2).unwrap(), "1");
+ assert_eq!(queue_size(&fs, "Semantic").await, 1);
}
#[tokio::test]
@@ -977,11 +936,8 @@ mod tests {
assert!(names.contains(&"warnings"));
// Can enqueue to nested queue
- fs.write("/logs/errors/enqueue", b"error1", 0, WriteFlag::None)
- .await
- .unwrap();
- let msg = fs.read("/logs/errors/dequeue", 0, 0).await.unwrap();
- let msg: TestQueueMessage = serde_json::from_slice(&msg).unwrap();
+ enqueue(&fs, "logs/errors", b"error1").await;
+ let msg = dequeue_msg(&fs, "logs/errors").await;
assert_eq!(msg.data, "error1");
}
@@ -991,13 +947,10 @@ mod tests {
// Create queue
fs.mkdir("/temp", 0o755).await.unwrap();
- fs.write("/temp/enqueue", b"data", 0, WriteFlag::None)
- .await
- .unwrap();
+ enqueue(&fs, "temp", b"data").await;
// Verify exists
- let size = fs.read("/temp/size", 0, 0).await.unwrap();
- assert_eq!(String::from_utf8(size).unwrap(), "1");
+ assert_eq!(queue_size(&fs, "temp").await, 1);
// Delete queue
fs.remove_all("/temp").await.unwrap();
diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rs
index f9c29b663..79f0bca49 100644
--- a/crates/ragfs/src/plugins/s3fs/client.rs
+++ b/crates/ragfs/src/plugins/s3fs/client.rs
@@ -301,14 +301,26 @@ impl S3Client {
/// Get an object's contents
pub async fn get_object(&self, key: &str) -> Result<Vec<u8>> {
- let resp = self
+ let resp = match self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
- .map_err(|e| Error::internal(format!("S3 GetObject error: {}", e)))?;
+ {
+ Ok(resp) => resp,
+ Err(sdk_err) => {
+ let service_err = sdk_err.into_service_error();
+ if service_err.is_no_such_key() {
+ return Err(Error::NotFound(key.to_string()));
+ }
+ return Err(Error::internal(format!(
+ "S3 GetObject error: {}",
+ service_err
+ )));
+ }
+ };
let bytes = resp
.body
@@ -320,19 +332,14 @@ impl S3Client {
}
/// Get an object's contents with range request
- pub async fn get_object_range(
- &self,
- key: &str,
- offset: u64,
- size: u64,
- ) -> Result<Vec<u8>> {
+ pub async fn get_object_range(&self, key: &str, offset: u64, size: u64) -> Result<Vec<u8>> {
let range = if size == 0 {
format!("bytes={}-", offset)
} else {
format!("bytes={}-{}", offset, offset + size - 1)
};
- let resp = self
+ let resp = match self
.client
.get_object()
.bucket(&self.bucket)
@@ -340,7 +347,19 @@ impl S3Client {
.range(range)
.send()
.await
- .map_err(|e| Error::internal(format!("S3 GetObject range error: {}", e)))?;
+ {
+ Ok(resp) => resp,
+ Err(sdk_err) => {
+ let service_err = sdk_err.into_service_error();
+ if service_err.is_no_such_key() {
+ return Err(Error::NotFound(key.to_string()));
+ }
+ return Err(Error::internal(format!(
+ "S3 GetObject range error: {}",
+ service_err
+ )));
+ }
+ };
let bytes = resp
.body
@@ -471,11 +490,7 @@ impl S3Client {
}
/// List objects with prefix and delimiter
- pub async fn list_objects(
- &self,
- prefix: &str,
- delimiter: Option<&str>,
- ) -> Result<ListResult> {
+ pub async fn list_objects(&self, prefix: &str, delimiter: Option<&str>) -> Result<ListResult> {
let mut files = Vec::new();
let mut directories = Vec::new();
let mut continuation_token: Option<String> = None;
diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rs
index 2aa82aaf1..5be685a0d 100644
--- a/crates/ragfs/src/plugins/s3fs/mod.rs
+++ b/crates/ragfs/src/plugins/s3fs/mod.rs
@@ -1243,64 +1243,45 @@ mod tests {
}
#[test]
- fn test_relative_match_file_same_path() {
- assert_eq!(relative_match_file("/foo", "/foo"), ".");
- assert_eq!(relative_match_file("/", "/"), ".");
- }
-
- #[test]
- fn test_relative_match_file_root_base() {
- assert_eq!(relative_match_file("/", "/a"), "a");
- assert_eq!(relative_match_file("/", "/a/b"), "a/b");
- assert_eq!(
- relative_match_file("/", "/deep/nested/file"),
- "deep/nested/file"
- );
- }
-
- #[test]
- fn test_relative_match_file_subdirectory() {
- assert_eq!(relative_match_file("/a", "/a/b"), "b");
- assert_eq!(relative_match_file("/a", "/a/b/c"), "b/c");
- assert_eq!(relative_match_file("/dir", "/dir/file.txt"), "file.txt");
- }
-
- #[test]
- fn test_relative_match_file_no_prefix_match() {
- assert_eq!(relative_match_file("/a", "/b"), "b");
- assert_eq!(relative_match_file("/x", "/y/z"), "y/z");
- }
-
- #[test]
- fn test_relative_match_file_trailing_slash_base() {
- assert_eq!(relative_match_file("/foo/", "/foo/bar"), "bar");
- assert_eq!(relative_match_file("/foo/", "/foo"), ".");
- }
-
- #[test]
- fn test_relative_match_file_empty_remainder() {
- assert_eq!(relative_match_file("/dir", "/dir/"), ".");
- }
-
- #[test]
- fn test_relative_match_file_leading_slash_in_rest() {
- assert_eq!(relative_match_file("/dir", "/dir//sub"), "sub");
- }
-
- #[test]
- fn test_relative_depth() {
- assert_eq!(relative_depth("."), 0);
- assert_eq!(relative_depth(""), 0);
- assert_eq!(relative_depth("a"), 1);
- assert_eq!(relative_depth("a/b"), 2);
- assert_eq!(relative_depth("a/b/c"), 3);
- assert_eq!(relative_depth("deep/nested/path/to/file"), 5);
+ fn test_relative_match_file_cases() {
+ for (base, path, expected) in [
+ ("/foo", "/foo", "."),
+ ("/", "/", "."),
+ ("/", "/a", "a"),
+ ("/", "/a/b", "a/b"),
+ ("/", "/deep/nested/file", "deep/nested/file"),
+ ("/a", "/a/b", "b"),
+ ("/a", "/a/b/c", "b/c"),
+ ("/dir", "/dir/file.txt", "file.txt"),
+ ("/a", "/b", "b"),
+ ("/x", "/y/z", "y/z"),
+ ("/foo/", "/foo/bar", "bar"),
+ ("/foo/", "/foo", "."),
+ ("/dir", "/dir/", "."),
+ ("/dir", "/dir//sub", "sub"),
+ ] {
+ assert_eq!(
+ relative_match_file(base, path),
+ expected,
+ "{base} -> {path}"
+ );
+ }
}
#[test]
- fn test_relative_depth_filters_empty_segments() {
- assert_eq!(relative_depth("a//b"), 2);
- assert_eq!(relative_depth("///"), 0);
+ fn test_relative_depth_cases() {
+ for (path, expected) in [
+ (".", 0),
+ ("", 0),
+ ("a", 1),
+ ("a/b", 2),
+ ("a/b/c", 3),
+ ("deep/nested/path/to/file", 5),
+ ("a//b", 2),
+ ("///", 0),
+ ] {
+ assert_eq!(relative_depth(path), expected, "{path}");
+ }
}
#[test]
@@ -1334,6 +1315,29 @@ mod tests {
Regex::new(pattern).unwrap()
}
+ /// Run grep_stream against in-memory chunk data.
+ async fn grep_chunks(
+ data: impl AsRef<[u8]>,
+ pattern: &str,
+ chunk_size: u64,
+ max_partial: usize,
+ ) -> Result<Vec<GrepMatch>> {
+ let data = data.as_ref().to_vec();
+ let file_size = data.len() as u64;
+ let mut reader = MockChunkReader { data };
+ let re = build_re(pattern);
+ grep_stream(
+ "f",
+ file_size,
+ &re,
+ 100,
+ chunk_size,
+ max_partial,
+ &mut reader,
+ )
+ .await
+ }
+
// ── Case 3: 正则无效 ──
#[test]
@@ -1367,14 +1371,7 @@ mod tests {
#[tokio::test]
async fn test_grep_stream_exact_chunk_single() {
let data = b"hello\nworld\n";
- let file_size = data.len() as u64;
- let chunk_size = file_size;
- let mut reader = MockChunkReader {
- data: data.to_vec(),
- };
- let re = build_re(".");
-
- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)
+ let matches = grep_chunks(data, ".", data.len() as u64, 1024)
.await
.unwrap();
@@ -1390,16 +1387,7 @@ mod tests {
#[tokio::test]
async fn test_grep_stream_last_chunk_exact_size() {
let data = b"line1\nline2\nline3\nline4\n";
- let file_size = data.len() as u64; // 24
- let chunk_size = 12u64;
- let mut reader = MockChunkReader {
- data: data.to_vec(),
- };
- let re = build_re("line");
-
- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)
- .await
- .unwrap();
+ let matches = grep_chunks(data, "line", 12, 1024).await.unwrap();
assert_eq!(matches.len(), 4, "all 4 lines should match");
assert_eq!(matches[3].content, "line4");
@@ -1410,16 +1398,7 @@ mod tests {
#[tokio::test]
async fn test_grep_stream_cross_chunk_line() {
let data = b"hel\nlo wo\nrld\n";
- let file_size = data.len() as u64; // 15
- let chunk_size = 8u64;
- let mut reader = MockChunkReader {
- data: data.to_vec(),
- };
- let re = build_re("lo");
-
- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)
- .await
- .unwrap();
+ let matches = grep_chunks(data, "lo", 8, 1024).await.unwrap();
assert_eq!(matches.len(), 1, "only 'lo wo' contains 'lo'");
assert_eq!(matches[0].line, 2, "line number should be 2");
@@ -1431,16 +1410,7 @@ mod tests {
#[tokio::test]
async fn test_grep_stream_multi_chunk_no_newline() {
let data = b"abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()"; // 46 bytes, no \n
- let file_size = data.len() as u64;
- let chunk_size = 10u64;
- let mut reader = MockChunkReader {
- data: data.to_vec(),
- };
- let re = build_re("xyz");
-
- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)
- .await
- .unwrap();
+ let matches = grep_chunks(data, "xyz", 10, 1024).await.unwrap();
assert_eq!(matches.len(), 1, "one long line should match");
assert_eq!(matches[0].line, 1);
@@ -1456,26 +1426,7 @@ mod tests {
#[tokio::test]
async fn test_grep_stream_line_exceeds_max_partial() {
let content = "a".repeat(200);
- let data = content.as_bytes();
- let file_size = data.len() as u64; // 200
- let chunk_size = 64u64;
- let max_partial = 100usize;
- let mut reader = MockChunkReader {
- data: data.to_vec(),
- };
- let re = build_re("a");
-
- let matches = grep_stream(
- "f",
- file_size,
- &re,
- 100,
- chunk_size,
- max_partial,
- &mut reader,
- )
- .await
- .unwrap();
+ let matches = grep_chunks(content.as_bytes(), "a", 64, 100).await.unwrap();
assert!(
matches.is_empty(),
@@ -1490,14 +1441,7 @@ mod tests {
let mut data: Vec<u8> = b"hello\nworld\n".to_vec();
data.extend_from_slice(&[0xff, 0xfe, 0xfd]);
data.extend_from_slice(b"\nvalid\n");
- let file_size = data.len() as u64;
- let chunk_size = 64u64;
- let mut reader = MockChunkReader { data };
- let re = build_re("world");
-
- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)
- .await
- .unwrap();
+ let matches = grep_chunks(data, "world", 64, 1024).await.unwrap();
assert_eq!(
matches.len(),
@@ -1513,14 +1457,7 @@ mod tests {
let mut data: Vec<u8> = b"first\n".to_vec();
data.extend_from_slice(&[0xff, 0xfe, 0xfd, b'\n']);
data.extend_from_slice(b"last\n");
- let file_size = data.len() as u64;
- let chunk_size = 64u64;
- let mut reader = MockChunkReader { data };
- let re = build_re("last");
-
- let matches = grep_stream("f", file_size, &re, 100, chunk_size, 1024, &mut reader)
- .await
- .unwrap();
+ let matches = grep_chunks(data, "last", 64, 1024).await.unwrap();
assert_eq!(matches.len(), 1, "should match 'last' after binary line");
assert_eq!(matches[0].content, "last");
diff --git a/crates/ragfs/src/plugins/serverinfofs/mod.rs b/crates/ragfs/src/plugins/serverinfofs/mod.rs
index 13cdabadb..72df71aea 100644
--- a/crates/ragfs/src/plugins/serverinfofs/mod.rs
+++ b/crates/ragfs/src/plugins/serverinfofs/mod.rs
@@ -49,7 +49,13 @@ impl ServerInfoFileSystem {
}}"#,
self.version,
format_duration(uptime),
- format_timestamp(UNIX_EPOCH.elapsed().unwrap_or(Duration::from_secs(0)).as_secs() - uptime_secs),
+ format_timestamp(
+ UNIX_EPOCH
+ .elapsed()
+ .unwrap_or(Duration::from_secs(0))
+ .as_secs()
+ - uptime_secs
+ ),
env!("CARGO_PKG_RUST_VERSION")
)
}
@@ -128,19 +134,27 @@ VERSION: 1.0.0
#[async_trait]
impl FileSystem for ServerInfoFileSystem {
async fn create(&self, _path: &str) -> Result<()> {
- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))
+ Err(Error::plugin(
+ "operation not permitted: serverinfofs is read-only".to_string(),
+ ))
}
async fn mkdir(&self, _path: &str, _mode: u32) -> Result<()> {
- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))
+ Err(Error::plugin(
+ "operation not permitted: serverinfofs is read-only".to_string(),
+ ))
}
async fn remove(&self, _path: &str) -> Result<()> {
- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))
+ Err(Error::plugin(
+ "operation not permitted: serverinfofs is read-only".to_string(),
+ ))
}
async fn remove_all(&self, _path: &str) -> Result<()> {
- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))
+ Err(Error::plugin(
+ "operation not permitted: serverinfofs is read-only".to_string(),
+ ))
}
async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {
@@ -185,8 +199,16 @@ impl FileSystem for ServerInfoFileSystem {
}
}
- async fn write(&self, _path: &str, _data: &[u8], _offset: u64, _flags: WriteFlag) -> Result<u64> {
- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))
+ async fn write(
+ &self,
+ _path: &str,
+ _data: &[u8],
+ _offset: u64,
+ _flags: WriteFlag,
+ ) -> Result<u64> {
+ Err(Error::plugin(
+ "operation not permitted: serverinfofs is read-only".to_string(),
+ ))
}
async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {
@@ -205,9 +227,21 @@ impl FileSystem for ServerInfoFileSystem {
Ok(vec![
FileInfo::new("README".to_string(), readme.len() as u64, 0o444, now, false),
- FileInfo::new("server_info".to_string(), server_info.len() as u64, 0o444, now, false),
+ FileInfo::new(
+ "server_info".to_string(),
+ server_info.len() as u64,
+ 0o444,
+ now,
+ false,
+ ),
FileInfo::new("uptime".to_string(), uptime.len() as u64, 0o444, now, false),
- FileInfo::new("version".to_string(), version.len() as u64, 0o444, now, false),
+ FileInfo::new(
+ "version".to_string(),
+ version.len() as u64,
+ 0o444,
+ now,
+ false,
+ ),
FileInfo::new("stats".to_string(), stats.len() as u64, 0o444, now, false),
])
}
@@ -234,15 +268,25 @@ impl FileSystem for ServerInfoFileSystem {
};
let name = path.strip_prefix('/').unwrap_or(path);
- Ok(FileInfo::new(name.to_string(), data.len() as u64, 0o444, now, false))
+ Ok(FileInfo::new(
+ name.to_string(),
+ data.len() as u64,
+ 0o444,
+ now,
+ false,
+ ))
}
async fn rename(&self, _old_path: &str, _new_path: &str) -> Result<()> {
- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))
+ Err(Error::plugin(
+ "operation not permitted: serverinfofs is read-only".to_string(),
+ ))
}
async fn chmod(&self, _path: &str, _mode: u32) -> Result<()> {
- Err(Error::plugin("operation not permitted: serverinfofs is read-only".to_string()))
+ Err(Error::plugin(
+ "operation not permitted: serverinfofs is read-only".to_string(),
+ ))
}
}
diff --git a/crates/ragfs/src/plugins/sqlfs/backend.rs b/crates/ragfs/src/plugins/sqlfs/backend.rs
index 7c32dc3e3..5fa01efb7 100644
--- a/crates/ragfs/src/plugins/sqlfs/backend.rs
+++ b/crates/ragfs/src/plugins/sqlfs/backend.rs
@@ -37,11 +37,8 @@ pub trait DatabaseBackend: Send + Sync {
fn delete_entry(&self, path: &str) -> Result<()>;
/// Delete entries matching a pattern (for recursive delete)
- fn delete_entries_by_pattern(
- &self,
- pattern: &str,
- exclude_path: Option<&str>,
- ) -> Result<usize>;
+ fn delete_entries_by_pattern(&self, pattern: &str, exclude_path: Option<&str>)
+ -> Result<usize>;
/// Read file data
fn read_file(&self, path: &str) -> Result<Option<(bool, Vec<u8>)>>;
@@ -152,7 +149,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn path_exists(&self, path: &str) -> Result<bool> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let mut stmt = conn
.prepare_cached("SELECT COUNT(*) FROM files WHERE path = ?1")
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
@@ -167,7 +167,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn is_directory(&self, path: &str) -> Result<bool> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let mut stmt = conn
.prepare_cached("SELECT is_dir FROM files WHERE path = ?1")
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
@@ -180,7 +183,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn create_file(&self, path: &str, mode: u32, data: &[u8]) -> Result<()> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let now = chrono::Utc::now().timestamp();
conn.execute(
"INSERT INTO files (path, is_dir, mode, size, mod_time, data) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
@@ -191,7 +197,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn create_directory(&self, path: &str, mode: u32) -> Result<()> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let now = chrono::Utc::now().timestamp();
conn.execute(
"INSERT INTO files (path, is_dir, mode, size, mod_time, data) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
@@ -202,7 +211,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn delete_entry(&self, path: &str) -> Result<()> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
conn.execute("DELETE FROM files WHERE path = ?1", params![path])
.map_err(|e| Error::internal(format!("delete error: {}", e)))?;
Ok(())
@@ -213,7 +225,10 @@ impl DatabaseBackend for SQLiteBackend {
pattern: &str,
exclude_path: Option<&str>,
) -> Result<usize> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let result = if let Some(exclude) = exclude_path {
conn.execute(
@@ -230,7 +245,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn read_file(&self, path: &str) -> Result<Option<(bool, Vec<u8>)>> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let mut stmt = conn
.prepare_cached("SELECT is_dir, data FROM files WHERE path = ?1")
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
@@ -247,7 +265,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn update_file(&self, path: &str, data: &[u8]) -> Result<()> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let now = chrono::Utc::now().timestamp();
conn.execute(
"UPDATE files SET data = ?1, size = ?2, mod_time = ?3 WHERE path = ?4",
@@ -258,11 +279,12 @@ impl DatabaseBackend for SQLiteBackend {
}
fn get_metadata(&self, path: &str) -> Result<Option<FileMetadata>> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let mut stmt = conn
- .prepare_cached(
- "SELECT path, is_dir, mode, size, mod_time FROM files WHERE path = ?1",
- )
+ .prepare_cached("SELECT path, is_dir, mode, size, mod_time FROM files WHERE path = ?1")
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
match stmt.query_row(params![path], |row| {
@@ -282,7 +304,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn update_mode(&self, path: &str, mode: u32) -> Result<()> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let now = chrono::Utc::now().timestamp();
conn.execute(
"UPDATE files SET mode = ?1, mod_time = ?2 WHERE path = ?3",
@@ -293,7 +318,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn rename_path(&self, old_path: &str, new_path: &str) -> Result<()> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
conn.execute(
"UPDATE files SET path = ?1 WHERE path = ?2",
params![new_path, old_path],
@@ -303,7 +331,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn rename_children(&self, old_path: &str, new_path: &str) -> Result<()> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let old_pattern = format!("{}/%", old_path);
let old_len = (old_path.len() + 1) as i32;
let sql = "UPDATE files SET path = ?1 || SUBSTR(path, ?2) WHERE path LIKE ?3";
@@ -313,7 +344,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn list_directory(&self, path: &str) -> Result<Vec<FileMetadata>> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
// Build pattern for direct children only
// For root "/": children are like "/<name>" (no further slashes)
@@ -350,8 +384,7 @@ impl DatabaseBackend for SQLiteBackend {
.map_err(|e| Error::internal(format!("query error: {}", e)))?;
for row_result in rows {
- let meta =
- row_result.map_err(|e| Error::internal(format!("row error: {}", e)))?;
+ let meta = row_result.map_err(|e| Error::internal(format!("row error: {}", e)))?;
// Only include direct children (no further '/' after the prefix)
let remainder = &meta.path[prefix_len..];
@@ -364,7 +397,10 @@ impl DatabaseBackend for SQLiteBackend {
}
fn count_by_pattern(&self, pattern: &str) -> Result<i64> {
- let conn = self.conn.lock().map_err(|e| Error::internal(e.to_string()))?;
+ let conn = self
+ .conn
+ .lock()
+ .map_err(|e| Error::internal(e.to_string()))?;
let mut stmt = conn
.prepare_cached("SELECT COUNT(*) FROM files WHERE path LIKE ?1")
.map_err(|e| Error::internal(format!("prepare error: {}", e)))?;
@@ -451,7 +487,9 @@ mod tests {
assert!(backend.is_directory("/testdir").unwrap());
// Create a file
- backend.create_file("/testdir/file.txt", 0o644, b"hello").unwrap();
+ backend
+ .create_file("/testdir/file.txt", 0o644, b"hello")
+ .unwrap();
assert!(backend.path_exists("/testdir/file.txt").unwrap());
assert!(!backend.is_directory("/testdir/file.txt").unwrap());
diff --git a/crates/ragfs/src/plugins/sqlfs/cache.rs b/crates/ragfs/src/plugins/sqlfs/cache.rs
index dc4fa105d..c2a46fa88 100644
--- a/crates/ragfs/src/plugins/sqlfs/cache.rs
+++ b/crates/ragfs/src/plugins/sqlfs/cache.rs
@@ -147,9 +147,7 @@ impl ListDirCache {
let to_invalidate: Vec<String> = inner
.cache
.iter()
- .filter(|(path, _)| {
- *path == prefix || is_descendant(path, prefix)
- })
+ .filter(|(path, _)| *path == prefix || is_descendant(path, prefix))
.map(|(path, _)| path.clone())
.collect();
diff --git a/crates/ragfs/src/plugins/sqlfs/mod.rs b/crates/ragfs/src/plugins/sqlfs/mod.rs
index 6639908f6..f990850c8 100644
--- a/crates/ragfs/src/plugins/sqlfs/mod.rs
+++ b/crates/ragfs/src/plugins/sqlfs/mod.rs
@@ -38,10 +38,7 @@ impl SQLFileSystem {
// Create database backend (schema init and optimizations happen inside)
let backend = create_backend(&config.params)?;
- tracing::info!(
- "SQLFS backend created: {}",
- backend.driver_name(),
- );
+ tracing::info!("SQLFS backend created: {}", backend.driver_name(),);
// Create cache from config
let cache_enabled = config
@@ -111,22 +108,14 @@ impl SQLFileSystem {
}
let normalized = Self::normalize_path(path);
- normalized
- .rsplit('/')
- .next()
- .unwrap_or("")
- .to_string()
+ normalized.rsplit('/').next().unwrap_or("").to_string()
}
}
impl Default for SQLFileSystem {
fn default() -> Self {
// Create with default SQLite in-memory database
- let config = PluginConfig {
- name: "sqlfs".to_string(),
- mount_path: "/sqlfs".to_string(),
- params: HashMap::new(),
- };
+ let config = PluginConfig::single_backend("sqlfs", "/sqlfs", HashMap::new());
Self::new(&config).expect("Failed to create default SQLFS")
}
@@ -519,12 +508,7 @@ impl SQLFSPlugin {
"Database host (MySQL/TiDB)",
),
ConfigParameter::optional("port", "int", "3306", "Database port (MySQL/TiDB)"),
- ConfigParameter::optional(
- "user",
- "string",
- "root",
- "Database user (MySQL/TiDB)",
- ),
+ ConfigParameter::optional("user", "string", "root", "Database user (MySQL/TiDB)"),
ConfigParameter::optional(
"password",
"string",
@@ -543,18 +527,8 @@ impl SQLFSPlugin {
"true",
"Enable directory listing cache",
),
- ConfigParameter::optional(
- "cache_max_size",
- "int",
- "1000",
- "Maximum cache entries",
- ),
- ConfigParameter::optional(
- "cache_ttl_seconds",
- "int",
- "5",
- "Cache TTL in seconds",
- ),
+ ConfigParameter::optional("cache_max_size", "int", "1000", "Maximum cache entries"),
+ ConfigParameter::optional("cache_ttl_seconds", "int", "5", "Cache TTL in seconds"),
],
}
}
@@ -706,13 +680,20 @@ agfs ls /sqlfs/mydir
mod tests {
use super::*;
+ /// Write one test file in SQLFS.
+ async fn write_file(fs: &SQLFileSystem, path: &str, data: &[u8]) {
+ fs.write(path, data, 0, WriteFlag::Create).await.unwrap();
+ }
+
+ /// Read one test file from SQLFS.
+ async fn read_file(fs: &SQLFileSystem, path: &str) -> Vec<u8> {
+ fs.read(path, 0, 0).await.unwrap()
+ }
+
#[tokio::test]
async fn test_sqlfs_basic() {
- let config = PluginConfig {
- name: "sqlfs".to_string(),
- mount_path: "/sqlfs".to_string(),
- params: std::collections::HashMap::new(),
- };
+ let config =
+ PluginConfig::single_backend("sqlfs", "/sqlfs", std::collections::HashMap::new());
let plugin = SQLFSPlugin::new();
assert!(plugin.validate(&config).await.is_ok());
@@ -742,9 +723,7 @@ mod tests {
fs.mkdir("/testdir", 0o755).await.unwrap();
// Create file in directory
- fs.write("/testdir/file.txt", b"data", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_file(&fs, "/testdir/file.txt", b"data").await;
// List directory
let entries = fs.read_dir("/testdir").await.unwrap();
@@ -763,29 +742,23 @@ mod tests {
async fn test_sqlfs_rename() {
let fs = SQLFileSystem::default();
- fs.write("/old.txt", b"data", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_file(&fs, "/old.txt", b"data").await;
fs.rename("/old.txt", "/new.txt").await.unwrap();
assert!(fs.stat("/old.txt").await.is_err());
- let data = fs.read("/new.txt", 0, 0).await.unwrap();
- assert_eq!(data, b"data");
+ assert_eq!(read_file(&fs, "/new.txt").await, b"data");
}
#[tokio::test]
async fn test_sqlfs_truncate() {
let fs = SQLFileSystem::default();
- fs.write("/trunc.txt", b"hello world", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_file(&fs, "/trunc.txt", b"hello world").await;
fs.truncate("/trunc.txt", 5).await.unwrap();
- let data = fs.read("/trunc.txt", 0, 0).await.unwrap();
- assert_eq!(data, b"hello");
+ assert_eq!(read_file(&fs, "/trunc.txt").await, b"hello");
}
#[tokio::test]
@@ -813,9 +786,7 @@ mod tests {
fs.mkdir("/a", 0o755).await.unwrap();
fs.mkdir("/a/b", 0o755).await.unwrap();
- fs.write("/a/b/file.txt", b"nested", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_file(&fs, "/a/b/file.txt", b"nested").await;
// List /a should only show /a/b
let entries = fs.read_dir("/a").await.unwrap();
@@ -824,17 +795,14 @@ mod tests {
assert!(entries[0].is_dir);
// Read nested file
- let data = fs.read("/a/b/file.txt", 0, 0).await.unwrap();
- assert_eq!(data, b"nested");
+ assert_eq!(read_file(&fs, "/a/b/file.txt").await, b"nested");
}
#[tokio::test]
async fn test_sqlfs_read_with_offset_and_size() {
let fs = SQLFileSystem::default();
- fs.write("/range.txt", b"hello world", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_file(&fs, "/range.txt", b"hello world").await;
// Read with offset
let data = fs.read("/range.txt", 6, 0).await.unwrap();
@@ -853,9 +821,7 @@ mod tests {
async fn test_sqlfs_chmod() {
let fs = SQLFileSystem::default();
- fs.write("/perm.txt", b"data", 0, WriteFlag::Create)
- .await
- .unwrap();
+ write_file(&fs, "/perm.txt", b"data").await;
fs.chmod("/perm.txt", 0o600).await.unwrap();
diff --git a/crates/ragfs/src/shape/manifest.rs b/crates/ragfs/src/shape/manifest.rs
new file mode 100644
index 000000000..56557520a
--- /dev/null
+++ b/crates/ragfs/src/shape/manifest.rs
@@ -0,0 +1,16 @@
+/// Shape guard file stored at the backend root.
+pub const SHAPE_MANIFEST_PATH: &str = "/backend_meta.json";
+
+/// Physical storage layout of one backend.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum StorageShape {
+ /// Backend stores plaintext user files.
+ Plaintext,
+ /// Backend stores encrypted user files.
+ Encrypted {
+ /// Crypto provider identifier embedded in the envelope.
+ provider_type: u8,
+ /// Envelope format version.
+ envelope_version: u8,
+ },
+}
diff --git a/crates/ragfs/src/shape/mod.rs b/crates/ragfs/src/shape/mod.rs
new file mode 100644
index 000000000..4cd3fecfa
--- /dev/null
+++ b/crates/ragfs/src/shape/mod.rs
@@ -0,0 +1,10 @@
+//! Backend storage shape probing and validation.
+
+/// Shape manifest definitions and constants.
+pub mod manifest;
+/// Legacy backend shape probing helpers.
+pub mod probe;
+/// Backend shape validation helpers.
+pub mod validate;
+
+pub use manifest::{StorageShape, SHAPE_MANIFEST_PATH};
diff --git a/crates/ragfs/src/shape/probe.rs b/crates/ragfs/src/shape/probe.rs
new file mode 100644
index 000000000..f176c530c
--- /dev/null
+++ b/crates/ragfs/src/shape/probe.rs
@@ -0,0 +1,129 @@
+use std::sync::Arc;
+
+use crate::core::errors::{Error, Result};
+use crate::core::filesystem::FileSystem;
+use crate::core::WriteFlag;
+use crate::crypto;
+
+use super::manifest::{StorageShape, SHAPE_MANIFEST_PATH};
+
+const SYSTEM_ACCOUNT_ID: &[u8] = b"_system";
+
+fn normalize_shape_path(path: &str) -> String {
+ let mut normalized = path.trim().to_string();
+ if !normalized.starts_with('/') {
+ normalized.insert(0, '/');
+ }
+ if normalized.len() > 1 && normalized.ends_with('/') {
+ normalized.pop();
+ }
+ normalized
+}
+
+/// Read the raw guard-file bytes from the backend root.
+async fn read_shape_guard_raw(raw_fs: &Arc<dyn FileSystem>) -> Result<Option<Vec<u8>>> {
+ match raw_fs.read(SHAPE_MANIFEST_PATH, 0, 0).await {
+ Ok(bytes) => Ok(Some(bytes)),
+ Err(Error::NotFound(_)) => Ok(None),
+ Err(err) => Err(err),
+ }
+}
+
+/// Infer one file's physical storage shape from its raw bytes.
+fn shape_from_raw_bytes(path: &str, bytes: &[u8]) -> Result<StorageShape> {
+ if crypto::is_encrypted(bytes) {
+ if bytes.len() < 6 {
+ return Err(Error::config(format!(
+ "encrypted file '{}' is too short to inspect shape",
+ path
+ )));
+ }
+ Ok(StorageShape::Encrypted {
+ envelope_version: bytes[4],
+ provider_type: bytes[5],
+ })
+ } else {
+ Ok(StorageShape::Plaintext)
+ }
+}
+
+/// Encrypt one empty guard payload so its physical shape matches the backend.
+fn encrypt_shape_guard_bytes(root_key: &[u8; 32], provider_type: u8) -> Result<Vec<u8>> {
+ let account_key = crypto::hkdf_sha256(root_key, SYSTEM_ACCOUNT_ID);
+ let file_key: [u8; 32] = rand::random();
+ let key_iv: [u8; 12] = rand::random();
+ let data_iv: [u8; 12] = rand::random();
+ let ciphertext = crypto::aes_gcm_encrypt(&file_key, &data_iv, &[])?;
+ let enc_key = crypto::aes_gcm_encrypt(&account_key, &key_iv, &file_key)?;
+ Ok(crypto::build_envelope(
+ provider_type,
+ &enc_key,
+ &key_iv,
+ &data_iv,
+ &ciphertext,
+ ))
+}
+
+/// Probe the physical shape of the persisted backend-shape guard file.
+pub async fn probe_shape_guard_storage(
+ raw_fs: &Arc<dyn FileSystem>,
+) -> Result<Option<StorageShape>> {
+ let Some(bytes) = read_shape_guard_raw(raw_fs).await? else {
+ return Ok(None);
+ };
+ Ok(Some(shape_from_raw_bytes(SHAPE_MANIFEST_PATH, &bytes)?))
+}
+
+/// Persist the backend-shape guard file in the same physical shape as the backend.
+pub async fn write_shape_guard(
+ raw_fs: &Arc<dyn FileSystem>,
+ shape: &StorageShape,
+ root_key: Option<&[u8; 32]>,
+) -> Result<()> {
+ let payload = match shape {
+ StorageShape::Plaintext => Vec::new(),
+ StorageShape::Encrypted { provider_type, .. } => {
+ let root_key = root_key.ok_or_else(|| {
+ Error::config("encrypted backend_meta.json requires a root key to be written")
+ })?;
+ encrypt_shape_guard_bytes(root_key, *provider_type)?
+ }
+ };
+ raw_fs
+ .write(SHAPE_MANIFEST_PATH, &payload, 0, WriteFlag::Create)
+ .await?;
+ Ok(())
+}
+
+/// Probe existing backend files to infer the legacy storage shape.
+pub async fn detect_legacy_shape(raw_fs: &Arc<dyn FileSystem>) -> Result<Option<StorageShape>> {
+ let entries = raw_fs.tree_directory("/", true, None, None).await?;
+ let mut detected: Option<StorageShape> = None;
+
+ for entry in entries {
+ if entry.info.is_dir {
+ continue;
+ }
+
+ let normalized = normalize_shape_path(&entry.path);
+ if normalized == SHAPE_MANIFEST_PATH {
+ continue;
+ }
+
+ let header = raw_fs.read(&normalized, 0, 6).await?;
+ let current = shape_from_raw_bytes(&normalized, &header)?;
+
+ match &detected {
+ None => detected = Some(current),
+ Some(existing) if existing == &current => {}
+ Some(existing) => {
+ return Err(Error::config(format!(
+ "backend contains mixed storage shapes: previously {:?}, found {:?} at '{}'",
+ existing, current, normalized
+ )));
+ }
+ }
+ }
+
+ Ok(detected)
+}
diff --git a/crates/ragfs/src/shape/validate.rs b/crates/ragfs/src/shape/validate.rs
new file mode 100644
index 000000000..de157d045
--- /dev/null
+++ b/crates/ragfs/src/shape/validate.rs
@@ -0,0 +1,54 @@
+use std::sync::Arc;
+
+use crate::core::errors::{Error, Result};
+use crate::core::FileSystem;
+use crate::crypto;
+
+use super::manifest::StorageShape;
+use super::probe::{detect_legacy_shape, probe_shape_guard_storage, write_shape_guard};
+
+/// Compute the expected backend shape from encryption settings.
+pub fn expected_shape(encrypted: bool, provider_type: Option<u8>) -> Result<StorageShape> {
+ if encrypted {
+ let provider_type = provider_type
+ .ok_or_else(|| Error::config("encrypted backend shape requires provider_type"))?;
+ Ok(StorageShape::Encrypted {
+ provider_type,
+ envelope_version: crypto::VERSION,
+ })
+ } else {
+ Ok(StorageShape::Plaintext)
+ }
+}
+
+/// Validate that one backend matches the expected storage shape.
+pub async fn ensure_backend_shape(
+ raw_fs: &Arc<dyn FileSystem>,
+ backend_type: &str,
+ encrypted: bool,
+ provider_type: Option<u8>,
+ root_key: Option<[u8; 32]>,
+) -> Result<()> {
+ let expected = expected_shape(encrypted, provider_type)?;
+ if let Some(observed) = probe_shape_guard_storage(raw_fs).await? {
+ if observed != expected {
+ return Err(Error::config(format!(
+ "backend storage shape mismatch for '{}': expected {:?}, found {:?}",
+ backend_type, expected, observed
+ )));
+ }
+ return Ok(());
+ }
+
+ let detected = detect_legacy_shape(raw_fs).await?;
+ match detected {
+ None => write_shape_guard(raw_fs, &expected, root_key.as_ref()).await,
+ Some(found) if found == expected => {
+ write_shape_guard(raw_fs, &found, root_key.as_ref()).await
+ }
+ Some(found) => Err(Error::config(format!(
+ "backend storage shape mismatch for '{}': expected {:?}, found {:?}",
+ backend_type, expected, found
+ ))),
+ }
+}
diff --git a/docs/en/api/03-filesystem.md b/docs/en/api/03-filesystem.md
index 4ec8ac649..e7b46fbce 100644
--- a/docs/en/api/03-filesystem.md
+++ b/docs/en/api/03-filesystem.md
@@ -15,7 +15,7 @@ Phase 1 intentionally keeps the scope narrow:
- Resources only. Memories, skills, sessions, and other namespaces are not exposed.
- Text-first writes. `PUT` currently accepts UTF-8 text content only.
- WebDAV subset only. `OPTIONS`, `PROPFIND`, `GET`, `HEAD`, `PUT`, `DELETE`, `MKCOL`, and `MOVE` are supported.
-- Semantic sidecars stay internal. Derived files such as `.abstract.md`, `.overview.md`, `.relations.json`, and `.path.ovlock` are hidden from listings and cannot be accessed directly through WebDAV.
+- Semantic sidecars and internal system files stay internal. Derived or internal files such as `.abstract.md`, `.overview.md`, `.relations.json`, `.path.ovlock`, `.redirect.json`, and `.sync_log.json` are hidden from listings and cannot be accessed directly through WebDAV.
Behavior notes:
@@ -23,6 +23,7 @@ Behavior notes:
- Replacing an existing file through WebDAV refreshes related semantics and vectors, same as `write()`.
- `PUT` does not create parent collections automatically. Create missing directories with `MKCOL` first.
- User-created dot-directories and dot-files remain visible unless they match one of the reserved internal filenames above.
+- When multi-write storage is enabled, files redirected to a backup are still exposed through the filesystem APIs as normal files; internal redirect and sync metadata never become visible to callers.
## API Reference
diff --git a/docs/en/concepts/05-storage.md b/docs/en/concepts/05-storage.md
index 11ad4f5c6..421002cab 100644
--- a/docs/en/concepts/05-storage.md
+++ b/docs/en/concepts/05-storage.md
@@ -80,6 +80,17 @@ relations = viking_fs.relations("viking://resources/docs/auth")
AGFS provides POSIX-style file operations with multiple backend support.
+### Single-Backend and Multi-Write Modes
+
+By default, AGFS uses a single backend for content storage. Once `storage.agfs.backups` is configured, OpenViking enters multi-write mode:
+
+- Top-level `storage.agfs.backend` is the primary backend and remains the authoritative write target.
+- `storage.agfs.backups.items[]` defines backup backends for replicas, migration, or read acceleration.
+- The Python SDK, HTTP API, and CLI filesystem interfaces stay unchanged.
+- Multi-write uses `.redirect.json` and `.sync_log.json` internally to track redirect mappings and sync progress. These files are not visible to users.
+
+For the conceptual model, see [Multi-Write Storage](./14-multi-write-storage.md). For examples, see the [Multi-Write Storage Guide](../guides/13-multi-write-storage.md).
+
### Backend Types
| Backend | Description | Config |
@@ -165,4 +176,5 @@ viking_fs.mv(
- [Architecture Overview](./01-architecture.md) - System architecture
- [Context Layers](./03-context-layers.md) - L0/L1/L2 model
- [Viking URI](./04-viking-uri.md) - URI specification
+- [Multi-Write Storage](./14-multi-write-storage.md) - Primary/backup roles, routing, and consistency
- [Retrieval Mechanism](./07-retrieval.md) - Retrieval process details
diff --git a/docs/en/concepts/14-multi-write-storage.md b/docs/en/concepts/14-multi-write-storage.md
new file mode 100644
index 000000000..231acd0cb
--- /dev/null
+++ b/docs/en/concepts/14-multi-write-storage.md
@@ -0,0 +1,134 @@
+# Multi-Write Storage
+
+Multi-write storage lets OpenViking use one primary storage backend together with multiple backup backends under a unified filesystem abstraction. It is suitable for high availability, cross-region replicas, read acceleration, and storage migration.
+
+From the API user's point of view, interfaces such as `read()`, `write()`, `ls()`, and `stat()` do not change. Multi-write logic lives inside RAGFS, so callers do not need to care which underlying backend ultimately stores a file.
+
+## Core Model
+
+Multi-write storage consists of one primary backend and multiple backup backends:
+
+| Role | Configuration | Description |
+| --- | --- | --- |
+| primary | `storage.agfs.backend` | Authoritative write target and final read fallback |
+| backup | `storage.agfs.backups.items[]` | Receives replicated writes and may optionally participate in reads |
+
+If `backups` is not configured, OpenViking continues to use the original single-backend mode.
+
+## Write Path
+
+By default, writes land on the primary backend first and are then replicated to write-enabled backup backends.
+
+```text
+Client
+ -> OpenViking API
+ -> RAGFS MultiWrite
+ -> primary
+ -> backup1 / backup2 / ...
+```
+
+If a backup does not define `operations`, it participates in writes by default. This keeps cold-backup setup simple.
+
+## Sync Modes
+
+Multi-write supports two consistency modes.
+
+| Mode | Config value | Behavior | Suitable for |
+| --- | --- | --- | --- |
+| Async multi-write | `async` | Return as soon as the primary write succeeds; backups sync in the background | Low-latency writes, eventual consistency |
+| Sync multi-write | `sync` | Wait for backup acknowledgements after the primary write succeeds | Stronger write confirmation when extra latency is acceptable |
+
+In async mode, backups may lag behind the primary for a short time. In sync mode, `write_ack_count` and `write_ack_timeout_ms` control how many backup acknowledgements are required and how long the system waits.
+
+Even in sync mode, backups that timed out or did not confirm are still retried in the background.
+
+## Read Path
+
+Reads do not hit every backup by default. Only backups that explicitly declare the `read` operation join the read route.
+
+Read order:
+
+```text
+1. Read-enabled backups in ascending priority order
+2. Fallback to the primary backend
+3. If the file is redirected, access the redirect target
+4. Return NotFound if the file is still missing
+```
+
+This avoids having cold-backup nodes participate in reads by default and reduces the risk of serving stale data.
+
+## Redirect
+
+Redirect means "certain files are not written to the primary backend and are written to a specified backup instead."
+
+Common cases:
+
+- Large files go to object storage.
+- Specific file extensions go to a dedicated backend.
+- The primary backend keeps standard content, while special files live elsewhere.
+
+Redirect policies are configured on the primary backend. When a file matches a policy, OpenViking records the mapping in internal metadata. Calls such as `ls()`, `stat()`, and `read()` still expose a normal filesystem view.
+
+## Exclude
+
+Exclude means "a specific backup backend does not receive matching files."
+
+Common cases:
+
+- A memory or cache backend should not keep large files.
+- One backup only stores text resources.
+- A lower-cost backend excludes temporary or oversized files.
+
+Exclude policies are configured on each backup and affect only whether that backup receives a write.
+
+## Internal Metadata
+
+Multi-write uses two internal metadata files:
+
+| File | Purpose |
+| --- | --- |
+| `.redirect.json` | Records which backend stores each redirected file |
+| `.sync_log.json` | Records per-file sync version and backup acknowledgement progress |
+
+These files are hidden from normal users, do not appear in standard directory listings, and should not be read or written through public APIs.
+
+If the primary backend enables at-rest encryption, these internal metadata files follow the same encryption policy.
+
+## Encryption Relationship
+
+Multi-write does not change OpenViking's transparent encryption model.
+
+Rules:
+
+- The Python layer and public APIs stay unaware of encryption details.
+- The primary backend must be encrypted when global encryption is enabled.
+- Each backup may decide independently whether to enable encryption.
+- Internal metadata must go through the primary backend's encryption path.
+
+That means enabling multi-write does not change how clients call the system; encryption remains a configuration concern per backend.
+
+## Relationship with OVPack
+
+Multi-write only handles new writes after it is enabled. It does not automatically sync historical files that already exist in the primary backend.
+
+Recommended migration flow:
+
+1. Use OVPack or another controlled process for a full data migration.
+2. Validate the target backend data.
+3. Enable the multi-write configuration.
+4. Let future new writes and updates continue to replicate through multi-write.
+
+## Limitations
+
+- In async mode, backups may lag temporarily.
+- Historical files that existed before multi-write was enabled need a separate migration or backfill process.
+- Redirected files rely on internal metadata to reconstruct the directory view.
+- Concurrent writes from multiple processes to the same primary backend still need future distributed metadata locking.
+- Hot directories may update internal metadata frequently and can introduce additional write amplification.
+
+## Related Documents
+
+- [Storage Architecture](./05-storage.md)
+- [Configuration Guide](../guides/01-configuration.md)
+- [Multi-Write Storage Guide](../guides/13-multi-write-storage.md)
+- [OVPack Import and Export](../guides/09-ovpack.md)
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index c2f49b125..544e3e6b5 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -986,6 +986,8 @@ Storage configuration for context data, including file storage (RAGFS) and vecto
|-----------|------|-------------|---------|
| `backend` | str | `"local"`, `"s3"`, or `"memory"` | `"local"` |
| `timeout` | float | Request timeout in seconds | `10.0` |
+| `backups` | object | Multi-write storage configuration. When set, the top-level `backend` acts as the primary backend and `backups.items[]` defines backup backends | `null` |
+| `redirects` | array | File redirect policies for multi-write storage. Matching files are written to the specified backup instead of the primary backend | `[]` |
| `queuefs` | object | QueueFS configuration. Controls the namespace mode, backend, and runtime options for `/queue` | `{ "mode": "shared", "backend": "sqlite", "recover_stale_sec": 0, "busy_timeout_ms": 5000 }` |
| `queue_db_path` | str (optional) | Legacy compatibility field for QueueFS sqlite DB path. Superseded by `storage.agfs.queuefs.db_path`. Defaults to `{storage.workspace}/_system/queue/queue.db` when not set. Useful when the workspace volume does not support sqlite (e.g. some network filesystems) | `null` |
| `s3` | object | S3 backend configuration (when backend is 's3') | - |
@@ -997,6 +999,82 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro
> [!WARNING]
> `storage.agfs` no longer supports the AGFS HTTP client mode, and the old HTTP client entry should not be configured anymore. AGFS / RAGFS filesystem access now happens only through the in-process Rust binding (`RAGFSBindingClient`). This does not affect the OpenViking server HTTP API, the `ov` CLI, or `AsyncHTTPClient` / `SyncHTTPClient` when they connect to an OpenViking server.
+##### Multi-Write Storage Configuration
+
+`storage.agfs.backups` enables multi-write storage. If it is not configured, OpenViking stays in single-backend mode.
+
+```json
+{
+ "storage": {
+ "workspace": "./data",
+ "agfs": {
+ "backend": "local",
+ "redirects": [
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": ["(pdf|ppt|zip)"],
+ "target": ["s3-backup"]
+ }
+ ],
+ "backups": {
+ "sync_type": "async",
+ "items": [
+ {
+ "name": "s3-backup",
+ "backend": "s3",
+ "s3": {
+ "bucket": "openviking-backup",
+ "region": "cn-beijing",
+ "endpoint": "https://tos-s3-cn-beijing.volces.com",
+ "access_key": "your-ak",
+ "secret_key": "your-sk",
+ "prefix": "multi-write"
+ }
+ }
+ ]
+ }
+ }
+ }
+}
+```
+
+Common `backups` fields:
+
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `sync_type` | str | Multi-write sync mode. Supports `"async"` or `"sync"` | `"async"` |
+| `write_ack_count` | int | Number of backup acknowledgements required before a `sync` write returns | all backups |
+| `write_ack_timeout_ms` | int | Timeout in milliseconds while waiting for backup acknowledgements in `sync` mode | `null` |
+| `write_concurrency` | int | Maximum async backup write concurrency | `null` |
+| `items` | array | Backup backend list. Each item reuses normal backend configuration and adds fields such as `name`, `operations`, `excludes`, and `encryption` | `[]` |
+
+Common `redirects` fields:
+
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `type` | str | Policy type. Supports `"FileExtensionPolicy"` or `"FileOverSizePolicy"` | required |
+| `extensions` | array | Extension regex list used by `FileExtensionPolicy`, for example `["(pdf\\|ppt)"]` | `[]` |
+| `max_size_mb` | int | File size threshold in MB used by `FileOverSizePolicy` | `null` |
+| `target` | array | Backup `name` list that receives matched files | required |
+
+File-size redirect example:
+
+```json
+{
+ "type": "FileOverSizePolicy",
+ "max_size_mb": 100,
+ "target": ["s3-backup"]
+}
+```
+
+Notes:
+
+- `redirects` is configured at top-level `storage.agfs` and defines redirect policies for the primary backend.
+- `target` must reference an existing backup `name` from `backups.items[]`.
+- Files matched by redirect still appear as normal readable and listable files through the filesystem APIs.
+
+See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more examples.
+
##### QueueFS Configuration
| Parameter | Type | Description | Default |
diff --git a/docs/en/guides/08-encryption.md b/docs/en/guides/08-encryption.md
index 0b10c821c..f9b661929 100644
--- a/docs/en/guides/08-encryption.md
+++ b/docs/en/guides/08-encryption.md
@@ -13,6 +13,19 @@ OpenViking provides transparent at-rest data encryption to ensure data security
See [Data Encryption](../concepts/10-encryption.md) for conceptual explanations.
+## Encryption in Multi-Write Storage
+
+Multi-write storage reuses the same transparent encryption model. Encryption still happens inside RAGFS, so the Python SDK, HTTP API, and CLI do not need to handle encryption or decryption directly.
+
+Rules:
+
+- When global `encryption.enabled=true`, the primary backend must be encrypted.
+- Each backup backend may control its own encryption through `encryption.enabled`.
+- Multi-write internal metadata such as `.redirect.json` and `.sync_log.json` follows the primary backend's encryption policy.
+- OpenViking does not expose and does not need public encryption APIs for operating on these internal files.
+
+See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more multi-write configuration details.
+
## Quick Start
### 1. Initialize Root Key (Local Mode)
diff --git a/docs/en/guides/09-ovpack.md b/docs/en/guides/09-ovpack.md
index 6e88063dc..70796f954 100644
--- a/docs/en/guides/09-ovpack.md
+++ b/docs/en/guides/09-ovpack.md
@@ -28,6 +28,25 @@ scope roots together:
Internal or runtime data such as `temp`, `queue`, `upload`, lock files, watch
control files, and `.relations.json` are outside the OVPack migration scope.
+## Working with Multi-Write Storage
+
+Multi-write storage only replicates writes that happen after it is enabled. It
+does not automatically copy historical files that already existed before
+`storage.agfs.backups` was turned on.
+
+When migrating an existing environment to multi-write mode, first move the
+existing dataset with OVPack, then enable multi-write storage.
+
+Recommended flow:
+
+1. Use `ov backup` or `ov export` to export the current dataset.
+2. Restore or import the dataset into the target storage environment.
+3. Validate data and index integrity in the target environment.
+4. Configure and enable multi-write storage.
+5. Resume normal writes and let multi-write handle future incremental copies.
+
+For more details, see the [Multi-Write Storage Guide](./13-multi-write-storage.md).
+
## Quick Start
### Export and Import a Resource Directory
diff --git a/docs/en/guides/13-multi-write-storage.md b/docs/en/guides/13-multi-write-storage.md
new file mode 100644
index 000000000..c3f8c66ef
--- /dev/null
+++ b/docs/en/guides/13-multi-write-storage.md
@@ -0,0 +1,388 @@
+# Multi-Write Storage Guide
+
+This guide explains how to configure multi-write storage in OpenViking. Multi-write storage lets one primary backend replicate writes to multiple backup backends for high availability, cross-region replicas, read acceleration, and storage migration.
+
+Multi-write lives inside RAGFS. The Python SDK, HTTP API, and CLI usage remain unchanged.
+
+## Prerequisites
+
+- You already have a working `ov.conf`.
+- The primary backend has been verified to read and write correctly.
+- If you plan to use S3-compatible storage, prepare the bucket, endpoint, and access credentials first.
+- If you need to migrate existing data, migrate that dataset before enabling multi-write.
+
+## Minimal Configuration
+
+The following example uses a local directory as the primary backend and replicates writes to another local directory.
+
+```json
+{
+ "storage": {
+ "workspace": "./data",
+ "agfs": {
+ "backend": "local",
+ "backups": {
+ "sync_type": "async",
+ "items": [
+ {
+ "name": "local-backup",
+ "backend": "local",
+ "local": {
+ "local_dir": "./data/backup"
+ }
+ }
+ ]
+ }
+ }
+ }
+}
+```
+
+Notes:
+
+- The top-level `backend` is the primary backend.
+- `backups.items[]` is the backup backend list.
+- `name` is the stable identity of a backup; later sync metadata refers to it.
+- If `sync_type` is omitted, treat it as async by default.
+
+## Configuring Multiple Backups
+
+You can configure more than one backup. The following example writes to both a local replica and S3-compatible object storage.
+
+```json
+{
+ "storage": {
+ "workspace": "./data",
+ "agfs": {
+ "backend": "local",
+ "backups": {
+ "sync_type": "async",
+ "items": [
+ {
+ "name": "local-az2",
+ "backend": "local",
+ "local": {
+ "local_dir": "./data/local-az2"
+ }
+ },
+ {
+ "name": "object-store",
+ "backend": "s3",
+ "s3": {
+ "bucket": "openviking-backup",
+ "region": "us-east-1",
+ "endpoint": "https://s3.example.com",
+ "access_key": "your-access-key",
+ "secret_key": "your-secret-key",
+ "prefix": "openviking"
+ }
+ }
+ ]
+ }
+ }
+ }
+}
+```
+
+Recommendations:
+
+- Do not use unstable hostnames or temporary IDs for `name`.
+- The backup path or bucket should not point to the same physical location as the primary backend.
+- Changing a backup `name` affects historical sync metadata recognition, so treat that as a production change.
+
+## Choosing a Sync Mode
+
+### Async Mode
+
+Async mode fits most deployments.
+
+```json
+{
+ "backups": {
+ "sync_type": "async",
+ "items": []
+ }
+}
+```
+
+Characteristics:
+
+- Returns immediately after the primary write succeeds.
+- Backup writes run in the background.
+- Low write latency.
+- Backups may lag temporarily.
+
+Suitable for:
+
+- Write throughput first.
+- Backups mainly used for disaster recovery.
+- Eventual consistency is acceptable.
+
+### Sync Mode
+
+Sync mode waits for backup acknowledgements.
+
+```json
+{
+ "backups": {
+ "sync_type": "sync",
+ "write_ack_count": 1,
+ "write_ack_timeout_ms": 5000,
+ "items": []
+ }
+}
+```
+
+Parameters:
+
+| Parameter | Description |
+| --- | --- |
+| `write_ack_count` | Minimum number of backup acknowledgements required before the write returns |
+| `write_ack_timeout_ms` | Timeout in milliseconds while waiting for backup acknowledgements |
+
+Characteristics:
+
+- Stronger write confirmation.
+- Write latency depends on backup responsiveness.
+- Unconfirmed backups continue to be retried in the background.
+- The client may still see an error after the primary write has already succeeded when the required backup acknowledgements are not met.
+
+Suitable for:
+
+- Narrowing the confirmation window between the primary and backups.
+- Backup latency is predictable.
+- The caller can accept the extra latency of synchronous writes.
+
+## Configuring Read Acceleration
+
+Backups do not participate in reads by default. To let a backup serve reads, explicitly configure `operations`.
+
+```json
+{
+ "name": "cache-backend",
+ "backend": "memfs",
+ "operations": [
+ {
+ "operation": "read",
+ "priority": 10
+ }
+ ]
+}
+```
+
+Read priority rules:
+
+- Lower `priority` values are tried first.
+- Only backups with `read` configured join the read route.
+- The primary backend always remains the final fallback.
+- Cold-backup nodes usually should not be read-enabled.
+
+If a backup defines only `read` but not `write`, it does not receive normal multi-write replication. Use that only when you explicitly control how the backend gets its data.
+
+## Redirect Configuration
+
+Redirect sends matching files to a specified backup instead of the primary backend.
+
+Redirect by file extension:
+
+```json
+{
+ "storage": {
+ "agfs": {
+ "backend": "local",
+ "redirects": [
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": ["(pdf|ppt|zip)"],
+ "target": ["object-store"]
+ }
+ ],
+ "backups": {
+ "items": [
+ {
+ "name": "object-store",
+ "backend": "s3",
+ "s3": {
+ "bucket": "openviking-large-files",
+ "endpoint": "https://s3.example.com"
+ }
+ }
+ ]
+ }
+ }
+ }
+}
+```
+
+Redirect by file size:
+
+```json
+{
+ "type": "FileOverSizePolicy",
+ "max_size_mb": 100,
+ "target": ["object-store"]
+}
+```
+
+Notes:
+
+- `target` must reference an existing backup `name`.
+- Redirected files still appear as normal readable, listable, and queryable files through the public APIs.
+- Redirect mappings are stored in internal metadata on the primary backend.
+
+## Exclude Configuration
+
+Exclude makes one backup skip matching files.
+
+```json
+{
+ "name": "cache-backend",
+ "backend": "memfs",
+ "excludes": [
+ {
+ "type": "FileOverSizePolicy",
+ "max_size_mb": 50
+ },
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": ["(mp4|zip)"]
+ }
+ ]
+}
+```
+
+Common uses:
+
+- Exclude large files from cache backends.
+- Exclude file types that do not need to be preserved on a lower-cost backup.
+- Keep one backup focused on text or configuration resources only.
+
+If the target backup of a redirect also excludes the same file, the configuration is self-contradictory. Fix the configuration instead of expecting the system to guess another target.
+
+## Encryption Configuration
+
+Multi-write storage reuses OpenViking's transparent at-rest encryption.
+
+Example with global encryption enabled:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ }
+ },
+ "storage": {
+ "workspace": "./data",
+ "agfs": {
+ "backend": "local",
+ "backups": {
+ "items": [
+ {
+ "name": "plain-cache",
+ "backend": "memfs",
+ "encryption": {
+ "enabled": false
+ }
+ },
+ {
+ "name": "encrypted-backup",
+ "backend": "local",
+ "local": {
+ "local_dir": "./data/encrypted-backup"
+ },
+ "encryption": {
+ "enabled": true
+ }
+ }
+ ]
+ }
+ }
+ }
+}
+```
+
+Rules:
+
+- When global `encryption.enabled=true`, the primary backend must be encrypted.
+- Each backup may independently control encryption through `encryption.enabled`.
+- The Python SDK, HTTP API, and CLI do not need to handle encryption or decryption.
+- Internal metadata such as `.redirect.json` and `.sync_log.json` follows the primary backend's encryption policy.
+
+## Migrating Existing Data
+
+Multi-write only replicates writes that happen after it is enabled. It does not automatically copy historical files.
+
+Recommended migration flow:
+
+1. Stop writes or freeze the write window.
+2. Use OVPack or another controlled tool to migrate historical data to the target backup.
+3. Validate the target backend's data integrity.
+4. Configure and enable `storage.agfs.backups`.
+5. Resume writes.
+6. Observe sync state and error logs.
+
+If freezing writes is not possible, do one full migration first, then a short write pause for incremental validation, and only then enable multi-write.
+
+## Verifying the Configuration
+
+Before startup, it is recommended to run:
+
+```bash
+openviking-server doctor
+```
+
+After startup, verify with ordinary file APIs:
+
+```bash
+openviking write viking://resources/multiwrite-check.txt \
+ --content "multi-write check" \
+ --wait
+
+openviking read viking://resources/multiwrite-check.txt
+```
+
+If you use a local backup, you can also inspect the backup directory directly. In production, system health checks and sync-status commands are preferable.
+
+## FAQ
+
+### Why is a backup not serving reads?
+
+Backups are write-only by default. To make a backup serve reads, configure:
+
+```json
+{
+ "operations": [
+ {
+ "operation": "read",
+ "priority": 10
+ }
+ ]
+}
+```
+
+### Why do historical files not appear in the backup after enabling multi-write?
+
+Multi-write only handles new writes after it is enabled. Historical data must be migrated separately through OVPack, object-storage copy workflows, or future backfill capabilities.
+
+### Can async mode guarantee that the newest data is immediately readable from a backup?
+
+No. Async mode provides eventual consistency only. If you need stronger read consistency, let reads fall back to the primary backend or avoid routing reads to backups that may lag.
+
+### Will internal metadata files appear in normal user listings?
+
+No. `.redirect.json` and `.sync_log.json` are internal files and are hidden from ordinary directory listings.
+
+### If sync mode returns a failure, does that mean the primary backend definitely did not write the data?
+
+No. The primary write may already have succeeded while the required backup acknowledgements were not met. In that case the client can see a failure even though the data already exists on the primary backend, and lagging backups will continue to be repaired in the background.
+
+## Related Documents
+
+- [Multi-Write Storage](../concepts/14-multi-write-storage.md)
+- [Storage Architecture](../concepts/05-storage.md)
+- [Configuration Guide](./01-configuration.md)
+- [Encryption Guide](./08-encryption.md)
+- [OVPack Import and Export](./09-ovpack.md)
diff --git a/docs/zh/api/03-filesystem.md b/docs/zh/api/03-filesystem.md
index 508aa9dcc..39b9e5304 100644
--- a/docs/zh/api/03-filesystem.md
+++ b/docs/zh/api/03-filesystem.md
@@ -15,7 +15,7 @@ Phase 1 有意把范围控制得比较小:
- 仅开放 `resources` 命名空间,不暴露 memories、skills、sessions 等其他空间。
- 以文本写入为主,当前 `PUT` 只接受 UTF-8 文本内容。
- 只实现一小部分 WebDAV 方法:`OPTIONS`、`PROPFIND`、`GET`、`HEAD`、`PUT`、`DELETE`、`MKCOL`、`MOVE`。
-- 语义侧边文件保持内部可见。`.abstract.md`、`.overview.md`、`.relations.json`、`.path.ovlock` 这些派生文件不会出现在 WebDAV 列表中,也不能被直接访问。
+- 语义侧边文件和系统内部文件保持内部可见。`.abstract.md`、`.overview.md`、`.relations.json`、`.path.ovlock`、`.redirect.json`、`.sync_log.json` 这些派生或内部文件不会出现在 WebDAV 列表中,也不能被直接访问。
行为说明:
@@ -23,6 +23,7 @@ Phase 1 有意把范围控制得比较小:
- 通过 WebDAV 覆盖已有文件时,会像 `write()` 一样刷新相关语义和向量。
- `PUT` 不会自动创建父目录。缺失的目录需要先用 `MKCOL` 创建。
- 用户自己创建的点目录或点文件仍然可见,只有上面列出的保留内部文件名会被隐藏。
+- 启用多写存储时,被 redirect 到 backup 的文件仍会通过文件系统 API 呈现为普通文件;内部 redirect 和同步元数据不会暴露给调用方。
## API 参考
diff --git a/docs/zh/concepts/05-storage.md b/docs/zh/concepts/05-storage.md
index e4b13351f..5af13966b 100644
--- a/docs/zh/concepts/05-storage.md
+++ b/docs/zh/concepts/05-storage.md
@@ -78,6 +78,17 @@ relations = viking_fs.relations("viking://resources/docs/auth")
AGFS 提供 POSIX 风格的文件操作,支持多种后端。
+### 单后端与多写模式
+
+默认情况下,AGFS 使用一个后端作为内容存储。配置 `storage.agfs.backups` 后,OpenViking 会启用多写模式:
+
+- 顶层 `storage.agfs.backend` 是 primary,作为权威写入目标。
+- `storage.agfs.backups.items[]` 是 backup,用于副本、迁移或读加速。
+- Python SDK、HTTP API 和 CLI 的文件系统接口保持不变。
+- 多写内部使用 `.redirect.json` 和 `.sync_log.json` 维护 redirect 映射与同步进度,这些文件对用户不可见。
+
+更多概念说明见 [多写存储](./14-multi-write-storage.md),配置示例见 [多写存储指南](../guides/13-multi-write-storage.md)。
+
### 后端类型
| 后端 | 说明 | 配置 |
@@ -163,4 +174,5 @@ viking_fs.mv(
- [架构概述](./01-architecture.md) - 系统整体架构
- [上下文层级](./03-context-layers.md) - L0/L1/L2 模型
- [Viking URI](./04-viking-uri.md) - URI 规范
+- [多写存储](./14-multi-write-storage.md) - primary/backup、多写路由与一致性
- [检索机制](./07-retrieval.md) - 检索流程详解
diff --git a/docs/zh/concepts/14-multi-write-storage.md b/docs/zh/concepts/14-multi-write-storage.md
new file mode 100644
index 000000000..6cd24d574
--- /dev/null
+++ b/docs/zh/concepts/14-multi-write-storage.md
@@ -0,0 +1,134 @@
+# 多写存储
+
+多写存储让 OpenViking 在一个统一的文件系统抽象下,同时使用一个主存储和多个备份存储。它适合数据高可用、跨区域副本、读加速、存储迁移等场景。
+
+从 API 使用者视角看,`read()`、`write()`、`ls()`、`stat()` 等接口不变。多写逻辑位于 RAGFS 内部,调用方不需要关心文件最终落在哪个底层后端。
+
+## 核心模型
+
+多写存储由一个 primary 和多个 backup 组成:
+
+| 角色 | 配置位置 | 说明 |
+| --- | --- | --- |
+| primary | `storage.agfs.backend` | 权威写入目标,也是读取兜底 |
+| backup | `storage.agfs.backups.items[]` | 接收复制写入,可选参与读取 |
+
+没有配置 `backups` 时,OpenViking 继续使用原有单后端模式。
+
+## 写入路径
+
+默认情况下,写入先落到 primary,再复制到 write-enabled backup。
+
+```text
+Client
+ -> OpenViking API
+ -> RAGFS MultiWrite
+ -> primary
+ -> backup1 / backup2 / ...
+```
+
+backup 未配置 `operations` 时默认参与写入。这样可以用最少配置得到冷备能力。
+
+## 同步模式
+
+多写支持两种一致性模式。
+
+| 模式 | 配置值 | 行为 | 适用场景 |
+| --- | --- | --- | --- |
+| 异步多写 | `async` | primary 写成功后立即返回,backup 后台同步 | 低延迟写入、最终一致 |
+| 同步多写 | `sync` | primary 写成功后等待 backup 确认 | 更强写入确认、可接受额外延迟 |
+
+异步模式下,backup 可能在短时间内落后于 primary。同步模式下,可以通过 `write_ack_count` 和 `write_ack_timeout_ms` 控制需要等待多少 backup 确认,以及等待多久。
+
+即使使用同步模式,未确认或超时的 backup 仍会由后台重试修复。
+
+## 读取路径
+
+读取不会默认访问所有 backup。只有显式声明 `read` 操作的 backup 才会进入读路由。
+
+读取顺序如下:
+
+```text
+1. 按 priority 升序访问 read-enabled backup
+2. 回退到 primary
+3. 如果文件被 redirect,则访问 redirect target
+4. 仍未命中则返回 NotFound
+```
+
+这种设计避免冷备节点默认参与读取,降低读到旧数据的风险。
+
+## Redirect
+
+Redirect 表示“某些文件不写入 primary,而是写入指定 backup”。
+
+常见用途:
+
+- 大文件进入对象存储。
+- 特定后缀文件进入专门 backend。
+- 主存储只保存常规内容,特殊文件由其他 backend 保存。
+
+Redirect 策略配置在 primary 上。命中策略后,OpenViking 会把映射记录到内部元数据中。用户执行 `ls()`、`stat()`、`read()` 时仍能看到正常的文件系统视图。
+
+## Exclude
+
+Exclude 表示“某个 backup 不接收匹配的文件”。
+
+常见用途:
+
+- 内存或缓存 backend 不保存大文件。
+- 某个 backup 只保存文本类资源。
+- 某个低成本 backend 排除临时或超大文件。
+
+Exclude 策略配置在 backup 上,只影响该 backup 是否接收写入。
+
+## 内部元数据
+
+多写使用两个内部元数据文件:
+
+| 文件 | 作用 |
+| --- | --- |
+| `.redirect.json` | 记录 redirect 文件对应的目标 backend |
+| `.sync_log.json` | 记录每个文件的同步版本和 backup 确认进度 |
+
+这些文件对普通用户不可见,不会出现在常规列表结果中,也不应通过公开 API 直接读写。
+
+如果 primary 开启静态数据加密,这些内部元数据也会跟随 primary 加密策略写入。
+
+## 加密关系
+
+多写不会改变 OpenViking 的透明加密模型。
+
+规则如下:
+
+- Python 层和公共 API 不感知加密实现。
+- primary 在全局加密开启时必须加密。
+- backup 可以独立决定是否加密。
+- 内部元数据必须走 primary 的加密入口。
+
+这意味着启用多写后,调用方式仍然不变;只需要通过配置决定每个 backend 的加密策略。
+
+## 与 OVPack 的关系
+
+多写只负责“启用之后的新写入”。它不会自动同步启用之前已经存在于 primary 中的历史文件。
+
+如果要迁移存量数据,推荐流程是:
+
+1. 使用 OVPack 或其他受控方式完成全量迁移。
+2. 校验目标 backend 数据。
+3. 启用多写配置。
+4. 后续新增和修改的数据由多写持续复制。
+
+## 限制
+
+- 异步模式下 backup 可能短暂落后。
+- 启用多写前的历史文件需要单独迁移或回填。
+- redirect 文件依赖内部元数据恢复目录视图。
+- 多进程同时写同一 primary 时,需要未来的分布式元数据锁能力。
+- 热点目录会频繁更新内部元数据,可能带来额外写放大。
+
+## 相关文档
+
+- [存储架构](./05-storage.md)
+- [配置指南](../guides/01-configuration.md)
+- [多写存储指南](../guides/13-multi-write-storage.md)
+- [OVPack 导入导出](../guides/09-ovpack.md)
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index ec970b37c..af6a86ff2 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -957,6 +957,8 @@ AST 提取支持:Python、JavaScript/TypeScript、Rust、Go、Java、C/C++。
|------|------|------|--------|
| `backend` | str | `"local"`、`"s3"` 或 `"memory"` | `"local"` |
| `timeout` | float | 请求超时时间(秒) | `10.0` |
+| `backups` | object | 多写存储配置。配置后顶层 `backend` 作为 primary,`backups.items[]` 作为 backup | `null` |
+| `redirects` | array | 多写存储的文件重定向策略。命中后文件写入指定 backup,而不是 primary | `[]` |
| `queuefs` | object | QueueFS 配置。控制 `/queue` 的命名空间模式、后端和运行时参数 | `{ "mode": "shared", "backend": "sqlite", "recover_stale_sec": 0, "busy_timeout_ms": 5000 }` |
| `queue_db_path` | str(可选)| 旧版兼容字段,用于覆盖 QueueFS 的 sqlite 数据库文件路径。已被 `storage.agfs.queuefs.db_path` 取代。未设置时默认为 `{storage.workspace}/_system/queue/queue.db`。适用于 workspace 卷不支持 sqlite 的场景(例如某些网络文件系统) | `null` |
| `s3` | object | S3 backend configuration (when backend is 's3') | - |
@@ -969,6 +971,82 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
> [!WARNING]
> `storage.agfs` 已不再支持 AGFS HTTP client 模式,也无需再配置旧的 HTTP client 入口。当前 AGFS / RAGFS 文件系统访问仅通过 Rust binding(`RAGFSBindingClient`)在进程内完成。这不影响 OpenViking server 的 HTTP API、`ov` CLI,或 `AsyncHTTPClient` / `SyncHTTPClient` 访问 OpenViking 服务端的能力。
+##### 多写存储配置
+
+`storage.agfs.backups` 用于启用多写存储。未配置时,OpenViking 保持单 backend 模式。
+
+```json
+{
+ "storage": {
+ "workspace": "./data",
+ "agfs": {
+ "backend": "local",
+ "redirects": [
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": ["(pdf|ppt|zip)"],
+ "target": ["s3-backup"]
+ }
+ ],
+ "backups": {
+ "sync_type": "async",
+ "items": [
+ {
+ "name": "s3-backup",
+ "backend": "s3",
+ "s3": {
+ "bucket": "openviking-backup",
+ "region": "cn-beijing",
+ "endpoint": "https://tos-s3-cn-beijing.volces.com",
+ "access_key": "your-ak",
+ "secret_key": "your-sk",
+ "prefix": "multi-write"
+ }
+ }
+ ]
+ }
+ }
+ }
+}
+```
+
+`backups` 常用字段:
+
+| 参数 | 类型 | 说明 | 默认值 |
+|------|------|------|--------|
+| `sync_type` | str | 多写同步模式,支持 `"async"` 或 `"sync"` | `"async"` |
+| `write_ack_count` | int | `sync` 模式下返回前需要的 backup 确认数 | 全部 backup |
+| `write_ack_timeout_ms` | int | `sync` 模式下等待 backup 确认的超时时间,单位毫秒 | `null` |
+| `write_concurrency` | int | 异步 backup 写入并发上限 | `null` |
+| `items` | array | backup backend 列表,每个 item 复用普通 backend 配置并增加 `name`、`operations`、`excludes`、`encryption` 等字段 | `[]` |
+
+`redirects` 常用字段:
+
+| 参数 | 类型 | 说明 | 默认值 |
+|------|------|------|--------|
+| `type` | str | 策略类型,支持 `"FileExtensionPolicy"` 或 `"FileOverSizePolicy"` | 必填 |
+| `extensions` | array | `FileExtensionPolicy` 使用的扩展名正则列表,例如 `["(pdf\|ppt)"]` | `[]` |
+| `max_size_mb` | int | `FileOverSizePolicy` 使用的文件大小阈值,单位 MB | `null` |
+| `target` | array | 命中策略后写入的 backup `name` 列表 | 必填 |
+
+按文件大小重定向示例:
+
+```json
+{
+ "type": "FileOverSizePolicy",
+ "max_size_mb": 100,
+ "target": ["s3-backup"]
+}
+```
+
+注意:
+
+- `redirects` 配置在顶层 `storage.agfs`,表示 primary 的重定向策略。
+- `target` 必须引用 `backups.items[]` 中已经定义的 backup `name`。
+- 命中 redirect 的文件仍会通过普通文件系统 API 呈现为可读、可列举的文件。
+
+更多配置示例见 [多写存储指南](./13-multi-write-storage.md)。
+
##### QueueFS 配置
| 参数 | 类型 | 说明 | 默认值 |
diff --git a/docs/zh/guides/08-encryption.md b/docs/zh/guides/08-encryption.md
index 355bc6fa9..15dd2d6ca 100644
--- a/docs/zh/guides/08-encryption.md
+++ b/docs/zh/guides/08-encryption.md
@@ -13,6 +13,19 @@ OpenViking 提供透明的静态数据加密,确保多租户环境下的数据
加密功能的概念说明见 [数据加密](../concepts/10-encryption.md)。
+## 多写存储中的加密
+
+多写存储复用同一套透明加密机制。加密仍在 RAGFS 内部完成,Python SDK、HTTP API 和 CLI 不需要处理加解密。
+
+规则:
+
+- 全局 `encryption.enabled=true` 时,primary backend 必须加密。
+- backup backend 可以通过自己的 `encryption.enabled` 控制是否加密。
+- `.redirect.json` 和 `.sync_log.json` 等多写内部元数据跟随 primary 加密策略。
+- OpenViking 不提供也不需要公开的加解密 API 来操作这些内部文件。
+
+更多多写配置见 [多写存储指南](./13-multi-write-storage.md)。
+
## 快速开始
### 1. 初始化根密钥(Local 模式)
diff --git a/docs/zh/guides/09-ovpack.md b/docs/zh/guides/09-ovpack.md
index eb0fa58ff..e97eee9b4 100644
--- a/docs/zh/guides/09-ovpack.md
+++ b/docs/zh/guides/09-ovpack.md
@@ -24,6 +24,20 @@ checksum,保证包内容没有偏离 manifest;如果攻击者能同时篡改
`temp`、`queue`、`upload`、锁文件、watch control 文件、`.relations.json` 等内部或运行态数据
不属于 OVPack 迁移范围。
+## 与多写存储配合
+
+多写存储只复制启用之后的新写入,不会自动同步启用之前已经存在的历史文件。将已有环境迁移到多写模式时,建议先用 OVPack 完成存量数据迁移,再开启 `storage.agfs.backups`。
+
+推荐流程:
+
+1. 使用 `ov backup` 或 `ov export` 导出现有内容。
+2. 在目标存储环境恢复或导入数据。
+3. 校验目标环境内容和索引状态。
+4. 配置并启用多写存储。
+5. 恢复正常写入,让后续增量由多写机制复制。
+
+更多说明见 [多写存储指南](./13-multi-write-storage.md)。
+
## 快速开始
### 导出和导入资源目录
diff --git a/docs/zh/guides/13-multi-write-storage.md b/docs/zh/guides/13-multi-write-storage.md
new file mode 100644
index 000000000..934a543e6
--- /dev/null
+++ b/docs/zh/guides/13-multi-write-storage.md
@@ -0,0 +1,388 @@
+# 多写存储指南
+
+本指南介绍如何配置 OpenViking 的多写存储能力。多写存储允许一个 primary 后端同时复制写入多个 backup 后端,用于高可用、跨区域副本、读加速和存储迁移。
+
+多写能力位于 RAGFS 内部。OpenViking 的 Python SDK、HTTP API 和 CLI 使用方式保持不变。
+
+## 前置条件
+
+- 已有可用的 `ov.conf`。
+- 已确认 primary backend 可以正常读写。
+- 如果要接入 S3 兼容存储,已准备好 bucket、endpoint 和访问凭据。
+- 如果要迁移已有数据,先完成存量数据迁移,再启用多写。
+
+## 最小配置
+
+下面示例使用本地目录作为 primary,并把写入复制到另一个本地目录。
+
+```json
+{
+ "storage": {
+ "workspace": "./data",
+ "agfs": {
+ "backend": "local",
+ "backups": {
+ "sync_type": "async",
+ "items": [
+ {
+ "name": "local-backup",
+ "backend": "local",
+ "local": {
+ "local_dir": "./data/backup"
+ }
+ }
+ ]
+ }
+ }
+ }
+}
+```
+
+说明:
+
+- 顶层 `backend` 是 primary。
+- `backups.items[]` 是 backup 列表。
+- `name` 是 backup 的稳定身份,后续同步元数据会引用它。
+- `sync_type` 不配置时默认按异步模式理解。
+
+## 多 Backup 配置
+
+可以配置多个 backup。下面示例同时写入本地副本和 S3 兼容对象存储。
+
+```json
+{
+ "storage": {
+ "workspace": "./data",
+ "agfs": {
+ "backend": "local",
+ "backups": {
+ "sync_type": "async",
+ "items": [
+ {
+ "name": "local-az2",
+ "backend": "local",
+ "local": {
+ "local_dir": "./data/local-az2"
+ }
+ },
+ {
+ "name": "object-store",
+ "backend": "s3",
+ "s3": {
+ "bucket": "openviking-backup",
+ "region": "us-east-1",
+ "endpoint": "https://s3.example.com",
+ "access_key": "your-access-key",
+ "secret_key": "your-secret-key",
+ "prefix": "openviking"
+ }
+ }
+ ]
+ }
+ }
+ }
+}
+```
+
+建议:
+
+- `name` 不要使用会频繁变化的机器名或临时编号。
+- backup 的底层路径或 bucket 应避免与 primary 指向同一物理位置。
+- 修改 backup `name` 会影响历史同步元数据的识别,生产环境应谨慎变更。
+
+## 同步模式选择
+
+### 异步模式
+
+异步模式适合大多数场景。
+
+```json
+{
+ "backups": {
+ "sync_type": "async",
+ "items": []
+ }
+}
+```
+
+特点:
+
+- primary 写入成功后立即返回。
+- backup 写入在后台执行。
+- 写入延迟低。
+- backup 可能短暂落后。
+
+适合:
+
+- 写入吞吐优先。
+- backup 主要用于灾备。
+- 可以接受最终一致性。
+
+### 同步模式
+
+同步模式会等待 backup 确认。
+
+```json
+{
+ "backups": {
+ "sync_type": "sync",
+ "write_ack_count": 1,
+ "write_ack_timeout_ms": 5000,
+ "items": []
+ }
+}
+```
+
+参数说明:
+
+| 参数 | 说明 |
+| --- | --- |
+| `write_ack_count` | 写入返回前至少需要多少个 backup 确认 |
+| `write_ack_timeout_ms` | 等待 backup 确认的超时时间,单位毫秒 |
+
+特点:
+
+- 写入确认更强。
+- 写入延迟受 backup 影响。
+- 未确认的 backup 会继续由后台重试修复。
+- primary 已写成功但 backup 未达确认数时,客户端可能收到错误;此时 primary 中可能已经存在数据。
+
+适合:
+
+- 希望尽量减少 primary 与 backup 的确认窗口。
+- backup 延迟可控。
+- 调用方能接受同步写入带来的额外延迟。
+
+## 配置读加速
+
+backup 默认不参与读取。要让 backup 服务读取,需要显式配置 `operations`。
+
+```json
+{
+ "name": "cache-backend",
+ "backend": "memfs",
+ "operations": [
+ {
+ "operation": "read",
+ "priority": 10
+ }
+ ]
+}
+```
+
+读取优先级规则:
+
+- `priority` 越小越优先。
+- 只有声明 `read` 的 backup 才参与读取。
+- primary 始终作为最终兜底。
+- 冷备 backup 不建议配置读能力。
+
+如果一个 backup 只配置了 `read`,没有配置 `write`,它不会接收普通多写复制。只有在你明确知道该 backend 的数据来源时,才应使用这种配置。
+
+## Redirect 配置
+
+Redirect 用于把匹配的文件写入指定 backup,而不是写入 primary。
+
+按扩展名重定向:
+
+```json
+{
+ "storage": {
+ "agfs": {
+ "backend": "local",
+ "redirects": [
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": ["(pdf|ppt|zip)"],
+ "target": ["object-store"]
+ }
+ ],
+ "backups": {
+ "items": [
+ {
+ "name": "object-store",
+ "backend": "s3",
+ "s3": {
+ "bucket": "openviking-large-files",
+ "endpoint": "https://s3.example.com"
+ }
+ }
+ ]
+ }
+ }
+ }
+}
+```
+
+按大小重定向:
+
+```json
+{
+ "type": "FileOverSizePolicy",
+ "max_size_mb": 100,
+ "target": ["object-store"]
+}
+```
+
+注意:
+
+- `target` 必须引用已有 backup 的 `name`。
+- redirect 文件仍会通过普通 API 呈现为可读、可列举、可查询状态。
+- redirect 映射保存在 primary 的内部元数据中。
+
+## Exclude 配置
+
+Exclude 用于让某个 backup 跳过匹配文件。
+
+```json
+{
+ "name": "cache-backend",
+ "backend": "memfs",
+ "excludes": [
+ {
+ "type": "FileOverSizePolicy",
+ "max_size_mb": 50
+ },
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": ["(mp4|zip)"]
+ }
+ ]
+}
+```
+
+常见用法:
+
+- 缓存 backend 排除大文件。
+- 低成本备份排除无需保存的文件类型。
+- 某个 backup 只保存文本或配置类资源。
+
+如果 redirect 的目标 backup 同时 exclude 了该文件,说明配置互相冲突。请优先修正配置,不要依赖系统自动猜测其他目标。
+
+## 加密配置
+
+多写存储复用 OpenViking 的透明静态加密能力。
+
+全局加密开启示例:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ }
+ },
+ "storage": {
+ "workspace": "./data",
+ "agfs": {
+ "backend": "local",
+ "backups": {
+ "items": [
+ {
+ "name": "plain-cache",
+ "backend": "memfs",
+ "encryption": {
+ "enabled": false
+ }
+ },
+ {
+ "name": "encrypted-backup",
+ "backend": "local",
+ "local": {
+ "local_dir": "./data/encrypted-backup"
+ },
+ "encryption": {
+ "enabled": true
+ }
+ }
+ ]
+ }
+ }
+ }
+}
+```
+
+规则:
+
+- 全局 `encryption.enabled=true` 时,primary 必须加密。
+- backup 可以通过 `encryption.enabled` 单独控制是否加密。
+- Python SDK、HTTP API 和 CLI 不需要处理加解密。
+- `.redirect.json` 和 `.sync_log.json` 等内部元数据会跟随 primary 加密策略。
+
+## 存量数据迁移
+
+多写只复制启用之后的新写入,不会自动复制历史文件。
+
+推荐迁移流程:
+
+1. 停止或冻结写入窗口。
+2. 使用 OVPack 或其他受控工具把存量数据迁移到目标 backup。
+3. 校验目标 backend 的数据完整性。
+4. 配置并启用 `storage.agfs.backups`。
+5. 恢复写入。
+6. 观察同步状态和错误日志。
+
+如果无法冻结写入,可以先做一次全量迁移,再短暂停写做增量校验,最后启用多写。
+
+## 验证配置
+
+启动前建议运行:
+
+```bash
+openviking-server doctor
+```
+
+启动后可以用普通文件 API 验证:
+
+```bash
+openviking write viking://resources/multiwrite-check.txt \
+ --content "multi-write check" \
+ --wait
+
+openviking read viking://resources/multiwrite-check.txt
+```
+
+如果使用本地 backup,可以直接检查 backup 目录中是否出现对应文件。生产环境更推荐使用系统健康检查和同步状态命令。
+
+## 常见问题
+
+### 为什么 backup 没有参与读取?
+
+backup 默认只参与写入,不参与读取。需要在 backup 上显式配置:
+
+```json
+{
+ "operations": [
+ {
+ "operation": "read",
+ "priority": 10
+ }
+ ]
+}
+```
+
+### 为什么启用多写后历史文件没有出现在 backup?
+
+多写只处理启用后的新写入。历史文件需要先通过 OVPack、对象存储复制或后续 backfill 能力迁移。
+
+### 异步模式下能否保证立即读到 backup 的最新数据?
+
+不能。异步模式只保证最终一致。需要强读一致时,应让读取回退到 primary,或避免让可能滞后的 backup 参与读路由。
+
+### 内部元数据文件会出现在用户列表里吗?
+
+不会。`.redirect.json` 和 `.sync_log.json` 是内部文件,会被普通目录列表隐藏。
+
+### sync 模式返回失败是否表示 primary 一定没写入?
+
+不是。primary 写成功但 backup 未达到确认数时,客户端可能收到失败。此时 primary 数据可能已经存在,落后的 backup 会由后台重试修复。
+
+## 相关文档
+
+- [多写存储](../concepts/14-multi-write-storage.md)
+- [存储架构](../concepts/05-storage.md)
+- [配置指南](./01-configuration.md)
+- [加密指南](./08-encryption.md)
+- [OVPack 导入导出](./09-ovpack.md)
diff --git a/openviking/pyagfs/__init__.py b/openviking/pyagfs/__init__.py
index 1c2145ae3..7e0a07c37 100644
--- a/openviking/pyagfs/__init__.py
+++ b/openviking/pyagfs/__init__.py
@@ -97,9 +97,18 @@ def _find_ragfs_so():
def _load_rust_binding():
"""Attempt to load the Rust (PyO3) binding client.
- Searches openviking/lib/ for the pre-built native extension first,
- then falls back to a pip-installed ``ragfs_python`` package.
+ Prefers the pip-installed ``ragfs_python`` package (e.g. from maturin develop),
+ then falls back to the vendored native extension in openviking/lib/.
"""
+ # Prefer pip-installed version (handles @rpath correctly)
+ try:
+ from ragfs_python import RAGFSBindingClient as _Rust
+
+ return _Rust, None
+ except ImportError:
+ pass
+
+ # Fallback: vendored .so in openviking/lib/
try:
so_path = _find_ragfs_so()
if so_path:
@@ -107,13 +116,10 @@ def _load_rust_binding():
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod.RAGFSBindingClient, None
-
- # Fallback: maybe ragfs_python was pip-installed (dev environment)
- from ragfs_python import RAGFSBindingClient as _Rust
-
- return _Rust, None
except Exception:
- raise ImportError("Rust binding not available")
+ pass
+
+ raise ImportError("Rust binding not available")
def get_binding_client():
diff --git a/openviking/pyagfs/async_client.py b/openviking/pyagfs/async_client.py
index e23a67700..09e57d43f 100644
--- a/openviking/pyagfs/async_client.py
+++ b/openviking/pyagfs/async_client.py
@@ -45,13 +45,12 @@ def encryption_account_id_from_agfs_path(path: str) -> str:
def ensure_same_encryption_account(src_path: str, dst_path: str) -> None:
- """Reject raw moves/copies that would preserve ciphertext under a different account key."""
+ """Reject AGFS moves/copies across encryption-account domains."""
src_account = encryption_account_id_from_agfs_path(src_path)
dst_account = encryption_account_id_from_agfs_path(dst_path)
if src_account != dst_account:
raise ValueError(
- "cross-account AGFS move/copy is not supported by raw path operations: "
- f"{src_account!r} -> {dst_account!r}"
+ f"cross-account AGFS move/copy is not supported: {src_account!r} -> {dst_account!r}"
)
@@ -126,19 +125,6 @@ class AsyncAGFSClient:
kwargs["stream"] = stream
return await self.run("cat", path, **kwargs, ctx=_fs_ctx_or_default(path, fs_ctx))
- async def read_raw(self, path: str, offset: int = 0, size: int = -1) -> Any:
- """Read raw bytes, bypassing the encryption layer (cp/persist whole-blob copy)."""
- kwargs: Dict[str, Any] = {}
- if offset != 0:
- kwargs["offset"] = offset
- if size != -1:
- kwargs["size"] = size
- return await self.run("read_raw", path, **kwargs)
-
- async def write_raw(self, path: str, data: bytes) -> str:
- """Write raw bytes, bypassing the encryption layer (cp/persist whole-blob copy)."""
- return await self.run("write_raw", path, data)
-
async def write(
self,
path: str,
@@ -193,10 +179,25 @@ class AsyncAGFSClient:
ensure_same_encryption_account(old_path, new_path)
return await self.run("mv", old_path, new_path, ctx=_fs_ctx_or_default(old_path, fs_ctx))
- async def cp(self, src_path: str, dst_path: str, recursive: bool = False) -> Any:
+ async def cp(
+ self,
+ src_path: str,
+ dst_path: str,
+ recursive: bool = False,
+ *,
+ fs_ctx: Dict[str, str] | None = None,
+ ) -> Any:
+ """Copy a path within AGFS while preserving the caller's FsContext."""
from .helpers import cp
- return await asyncio.to_thread(cp, self._client, src_path, dst_path, recursive=recursive)
+ return await asyncio.to_thread(
+ cp,
+ self._client,
+ src_path,
+ dst_path,
+ recursive=recursive,
+ fs_ctx=_fs_ctx_or_default(src_path, fs_ctx),
+ )
async def grep(self, **kwargs: Any) -> Dict[str, Any]:
if "ctx" not in kwargs or kwargs["ctx"] is None:
@@ -222,3 +223,15 @@ class AsyncAGFSClient:
level_limit=level_limit,
ctx=_fs_ctx_or_default(path, fs_ctx),
)
+
+ async def system_sync_status(
+ self, path: str, *, fs_ctx: Dict[str, str] | None = None
+ ) -> Dict[str, Any]:
+ """Return multi-write sync status for a file or directory path."""
+ return await self.run("system_sync_status", path, ctx=_fs_ctx_or_default(path, fs_ctx))
+
+ async def system_sync_retry(
+ self, path: str, *, fs_ctx: Dict[str, str] | None = None
+ ) -> Dict[str, Any]:
+ """Retry pending multi-write sync work for a file or directory path."""
+ return await self.run("system_sync_retry", path, ctx=_fs_ctx_or_default(path, fs_ctx))
diff --git a/openviking/pyagfs/helpers.py b/openviking/pyagfs/helpers.py
index 0683a6559..5483a05f0 100644
--- a/openviking/pyagfs/helpers.py
+++ b/openviking/pyagfs/helpers.py
@@ -8,17 +8,51 @@ This module provides high-level helper functions for common operations:
from collections.abc import Iterator
from pathlib import Path
+from typing import Any
from .async_client import ensure_same_encryption_account
from .protocols import AGFSByteStream, AGFSSyncClientProtocol
+def _call_with_optional_ctx(
+ method: Any, *args: Any, ctx: dict[str, str] | None = None, **kwargs: Any
+) -> Any:
+ """Call a sync AGFS method with ctx when the backend supports it."""
+ if ctx is None:
+ return method(*args, **kwargs)
+ try:
+ return method(*args, ctx=ctx, **kwargs)
+ except TypeError as exc:
+ if "unexpected keyword argument 'ctx'" not in str(exc):
+ raise
+ return method(*args, **kwargs)
+
+
+def _try_copy_within_mount_fast_path(
+ client: AGFSSyncClientProtocol,
+ src: str,
+ dst: str,
+ *,
+ fs_ctx: dict[str, str] | None = None,
+) -> bool:
+ """Attempt a same-mount verbatim copy and return whether the fast-path was used."""
+ copy_within_mount = getattr(client, "copy_within_mount", None)
+ if not callable(copy_within_mount):
+ return False
+
+ result = _call_with_optional_ctx(copy_within_mount, src, dst, ctx=fs_ctx)
+ if isinstance(result, dict):
+ return bool(result.get("performed", False))
+ return bool(result)
+
+
def cp(
client: AGFSSyncClientProtocol,
src: str,
dst: str,
recursive: bool = False,
stream: bool = False,
+ fs_ctx: dict[str, str] | None = None,
) -> None:
"""Copy a file or directory within AGFS.
@@ -39,15 +73,15 @@ def cp(
ensure_same_encryption_account(src, dst)
# Check if source exists and get its type
- src_info = client.stat(src)
+ src_info = _call_with_optional_ctx(client.stat, src, ctx=fs_ctx)
is_dir = src_info.get("isDir", False)
if is_dir:
if not recursive:
raise ValueError(f"Cannot copy directory '{src}' without recursive=True")
- _copy_directory(client, src, dst, stream)
+ _copy_directory(client, src, dst, stream, fs_ctx=fs_ctx)
else:
- _copy_file(client, src, dst, stream)
+ _copy_file(client, src, dst, stream, fs_ctx=fs_ctx)
def upload(
@@ -125,43 +159,58 @@ def download(
# Internal helper functions
-def _copy_file(client: AGFSSyncClientProtocol, src: str, dst: str, stream: bool) -> None:
+def _copy_file(
+ client: AGFSSyncClientProtocol,
+ src: str,
+ dst: str,
+ stream: bool,
+ *,
+ fs_ctx: dict[str, str] | None = None,
+) -> None:
"""Copy a single file within AGFS.
- Uses the raw byte channel (``read_raw``/``write_raw``) when available so a whole blob is moved
- verbatim, bypassing the encryption layer (ciphertext is copied as-is, never decrypted/re-encrypted).
- Falls back to ``cat``/``write`` for clients that do not expose the raw channel.
+ Copy is always expressed in logical AGFS operations so encryption remains
+ transparent to the business layer.
"""
# Ensure parent directory exists
- _ensure_remote_parent_dir(client, dst)
+ _ensure_remote_parent_dir(client, dst, fs_ctx=fs_ctx)
- read_raw = getattr(client, "read_raw", None)
- write_raw = getattr(client, "write_raw", None)
- if callable(read_raw) and callable(write_raw):
- write_raw(dst, read_raw(src))
+ if _try_copy_within_mount_fast_path(client, src, dst, fs_ctx=fs_ctx):
return
if stream:
# The binding client returns bytes or an iterator of bytes, not an
# HTTP response object with iter_content().
- client.write(dst, _iter_file_bytes(client.cat(src, stream=True)))
+ _call_with_optional_ctx(
+ client.write,
+ dst,
+ _iter_file_bytes(_call_with_optional_ctx(client.cat, src, stream=True, ctx=fs_ctx)),
+ ctx=fs_ctx,
+ )
else:
# Read entire file and write
- data = client.cat(src)
- client.write(dst, data)
+ data = _call_with_optional_ctx(client.cat, src, ctx=fs_ctx)
+ _call_with_optional_ctx(client.write, dst, data, ctx=fs_ctx)
-def _copy_directory(client: AGFSSyncClientProtocol, src: str, dst: str, stream: bool) -> None:
+def _copy_directory(
+ client: AGFSSyncClientProtocol,
+ src: str,
+ dst: str,
+ stream: bool,
+ *,
+ fs_ctx: dict[str, str] | None = None,
+) -> None:
"""Recursively copy a directory within AGFS."""
# Create destination directory
try:
- client.mkdir(dst)
+ _call_with_optional_ctx(client.mkdir, dst, ctx=fs_ctx)
except Exception:
# Directory might already exist, continue
pass
# List source directory contents
- items = client.ls(src)
+ items = _call_with_optional_ctx(client.ls, src, ctx=fs_ctx)
for item in items:
item_name = item["name"]
@@ -170,10 +219,10 @@ def _copy_directory(client: AGFSSyncClientProtocol, src: str, dst: str, stream:
if item.get("isDir", False):
# Recursively copy subdirectory
- _copy_directory(client, src_path, dst_path, stream)
+ _copy_directory(client, src_path, dst_path, stream, fs_ctx=fs_ctx)
else:
# Copy file
- _copy_file(client, src_path, dst_path, stream)
+ _copy_file(client, src_path, dst_path, stream, fs_ctx=fs_ctx)
def _upload_file(
@@ -268,22 +317,32 @@ def _download_directory(
_download_file(client, remote_item_path, local_item_path, stream)
-def _ensure_remote_parent_dir(client: AGFSSyncClientProtocol, path: str) -> None:
+def _ensure_remote_parent_dir(
+ client: AGFSSyncClientProtocol,
+ path: str,
+ *,
+ fs_ctx: dict[str, str] | None = None,
+) -> None:
"""Ensure the parent directory exists for a remote path."""
parent = "/".join(path.rstrip("/").split("/")[:-1])
if parent and parent != "/":
# Try to create parent directory (and its parents)
- _ensure_remote_dir_recursive(client, parent)
+ _ensure_remote_dir_recursive(client, parent, fs_ctx=fs_ctx)
-def _ensure_remote_dir_recursive(client: AGFSSyncClientProtocol, path: str) -> None:
+def _ensure_remote_dir_recursive(
+ client: AGFSSyncClientProtocol,
+ path: str,
+ *,
+ fs_ctx: dict[str, str] | None = None,
+) -> None:
"""Recursively ensure a directory exists in AGFS."""
if not path or path == "/":
return
# Check if directory already exists
try:
- info = client.stat(path)
+ info = _call_with_optional_ctx(client.stat, path, ctx=fs_ctx)
if info.get("isDir", False):
return # Directory exists
except Exception:
@@ -293,11 +352,11 @@ def _ensure_remote_dir_recursive(client: AGFSSyncClientProtocol, path: str) -> N
# Ensure parent exists first
parent = "/".join(path.rstrip("/").split("/")[:-1])
if parent and parent != "/":
- _ensure_remote_dir_recursive(client, parent)
+ _ensure_remote_dir_recursive(client, parent, fs_ctx=fs_ctx)
# Create this directory
try:
- client.mkdir(path)
+ _call_with_optional_ctx(client.mkdir, path, ctx=fs_ctx)
except Exception:
# Might already exist due to race condition, ignore
pass
diff --git a/openviking/pyagfs/protocols.py b/openviking/pyagfs/protocols.py
index 04c145e26..b6b6c19c8 100644
--- a/openviking/pyagfs/protocols.py
+++ b/openviking/pyagfs/protocols.py
@@ -78,6 +78,9 @@ class AGFSSyncClientProtocol(Protocol):
def mv(self, old_path: str, new_path: str) -> Dict[str, Any]:
"""Move or rename a path inside AGFS."""
+ def copy_within_mount(self, src_path: str, dst_path: str) -> Dict[str, Any]:
+ """Attempt a same-mount verbatim copy and report whether it was used."""
+
def grep(self, **kwargs: Any) -> Dict[str, Any]:
"""Run a grep-like search through the AGFS backend."""
@@ -89,3 +92,9 @@ class AGFSSyncClientProtocol(Protocol):
level_limit: int | None = None,
) -> list[Dict[str, Any]]:
"""Return a tree view for the given AGFS directory."""
+
+ def system_sync_status(self, path: str) -> Dict[str, Any]:
+ """Return multi-write sync status for a file or directory path."""
+
+ def system_sync_retry(self, path: str) -> Dict[str, Any]:
+ """Retry pending multi-write sync work for a file or directory path."""
diff --git a/openviking/server/api_keys/legacy.py b/openviking/server/api_keys/legacy.py
index 446d95b1f..9b4694f3d 100644
--- a/openviking/server/api_keys/legacy.py
+++ b/openviking/server/api_keys/legacy.py
@@ -13,7 +13,7 @@ from typing import Dict, Optional
from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError
-from openviking.pyagfs import AGFSNotFoundError, AsyncAGFSClient
+from openviking.pyagfs import AGFSAlreadyExistsError, AGFSNotFoundError, AsyncAGFSClient
from openviking.server.api_keys.models import AccountInfo, UserKeyEntry
from openviking.server.identity import ResolvedIdentity, Role
from openviking.storage.viking_fs import VikingFS
@@ -70,6 +70,40 @@ class LegacyAPIKeyManager:
# Prefix index: key_prefix -> list[UserKeyEntry]
self._prefix_index: Dict[str, list[UserKeyEntry]] = {}
+ def _discard_account_state(self, account_id: str) -> None:
+ """Remove an account and its key index entries from in-memory state."""
+ account = self._accounts.pop(account_id, None)
+ if account is None:
+ return
+
+ for user_id, user_info in account.users.items():
+ key_or_hash = user_info.get("key", "")
+ if not key_or_hash:
+ continue
+
+ key_prefix = user_info.get("key_prefix", "")
+ if not key_prefix:
+ key_prefix = self._get_key_prefix(key_or_hash)
+
+ if key_prefix not in self._prefix_index:
+ continue
+
+ self._prefix_index[key_prefix] = [
+ entry
+ for entry in self._prefix_index[key_prefix]
+ if not (entry.account_id == account_id and entry.user_id == user_id)
+ ]
+ if not self._prefix_index[key_prefix]:
+ del self._prefix_index[key_prefix]
+
+ async def _rollback_create_account(self, account_id: str) -> None:
+ """Best-effort rollback for partially persisted account creation."""
+ self._discard_account_state(account_id)
+ try:
+ await self._save_accounts_json()
+ except Exception:
+ logger.exception("Failed to persist rollback for account %s", account_id)
+
async def load(self) -> None:
"""Load accounts and user keys from VikingFS into memory."""
accounts_data = await self._read_json(ACCOUNTS_PATH)
@@ -229,8 +263,12 @@ class LegacyAPIKeyManager:
self._prefix_index[key_prefix] = []
self._prefix_index[key_prefix].append(entry)
- await self._save_accounts_json()
- await self._save_users_json(account_id)
+ try:
+ await self._save_accounts_json()
+ await self._save_users_json(account_id)
+ except Exception:
+ await self._rollback_create_account(account_id)
+ raise
return key
async def delete_account(self, account_id: str) -> None:
@@ -238,25 +276,7 @@ class LegacyAPIKeyManager:
if account_id not in self._accounts:
raise NotFoundError(account_id, "account")
- account = self._accounts.pop(account_id)
- # Remove all keys for this account from prefix index
- for user_info in account.users.values():
- key_or_hash = user_info.get("key", "")
- if key_or_hash:
- # Get key_prefix - if not in user_info, compute from key
- key_prefix = user_info.get("key_prefix", "")
- if not key_prefix:
- key_prefix = self._get_key_prefix(key_or_hash)
-
- if key_prefix in self._prefix_index:
- self._prefix_index[key_prefix] = [
- entry
- for entry in self._prefix_index[key_prefix]
- if entry.account_id != account_id
- ]
- # Remove prefix if index is empty
- if not self._prefix_index[key_prefix]:
- del self._prefix_index[key_prefix]
+ self._discard_account_state(account_id)
await self._save_accounts_json()
@@ -599,18 +619,10 @@ class LegacyAPIKeyManager:
async def _ensure_parent_dirs_async(self, path: str) -> None:
"""Recursively create all parent directories for a file path."""
- # Handle direct AGFS paths
- if path.startswith("/local/"):
- # Extract path parts from /local/{account_id}/_system/...
- parts = path.lstrip("/").split("/")
- if parts:
- # Create directory for each parent path
- for i in range(1, len(parts)):
- parent = "/" + "/".join(parts[:i])
- try:
- await self._async_agfs.mkdir(parent)
- except Exception:
- pass
+ try:
+ await self._async_agfs.ensure_parent_dirs(path)
+ except AGFSAlreadyExistsError:
+ return
async def _save_accounts_json(self) -> None:
"""Persist the global accounts list."""
diff --git a/openviking/server/api_keys/new.py b/openviking/server/api_keys/new.py
index 10cf5f26a..6f97d1dfa 100644
--- a/openviking/server/api_keys/new.py
+++ b/openviking/server/api_keys/new.py
@@ -227,8 +227,12 @@ class NewAPIKeyManager:
self._legacy._prefix_index[key_prefix] = []
self._legacy._prefix_index[key_prefix].append(entry)
- await self._legacy._save_accounts_json()
- await self._legacy._save_users_json(account_id)
+ try:
+ await self._legacy._save_accounts_json()
+ await self._legacy._save_users_json(account_id)
+ except Exception:
+ await self._legacy._rollback_create_account(account_id)
+ raise
return key
async def delete_account(self, account_id: str) -> None:
diff --git a/openviking/server/app.py b/openviking/server/app.py
index 58a2499d1..02558009b 100644
--- a/openviking/server/app.py
+++ b/openviking/server/app.py
@@ -75,6 +75,55 @@ def _on_deferred_init_done(task):
os._exit(1)
+async def _initialize_api_key_manager(
+ app: FastAPI,
+ service: OpenVikingService,
+ config: ServerConfig,
+) -> None:
+ """Initialize API key state before the app serves authenticated requests."""
+ effective_auth_mode = config.get_effective_auth_mode()
+ if config.root_api_key and config.root_api_key != "":
+ api_key_manager = APIKeyManager(
+ root_key=config.root_api_key,
+ viking_fs=service.viking_fs,
+ api_key_hashing_enabled=config.api_key_hashing_enabled,
+ )
+ await api_key_manager.load()
+ app.state.api_key_manager = api_key_manager
+ logger.info(
+ "APIKeyManager initialized with api_key_hashing_enabled=%s",
+ config.api_key_hashing_enabled,
+ )
+ if effective_auth_mode == AuthMode.TRUSTED:
+ logger.info(
+ "Trusted mode enabled: authentication trusts X-OpenViking-Account/User "
+ "headers and requires the configured server API key on each request. "
+ "Only expose this server behind a trusted network boundary or "
+ "identity-injecting gateway."
+ )
+ return
+
+ app.state.api_key_manager = None
+ if effective_auth_mode == AuthMode.TRUSTED:
+ logger.warning(
+ "Trusted mode enabled: authentication uses X-OpenViking-Account/User "
+ "headers without API keys. This is only allowed on localhost. "
+ "Only expose this server behind a trusted network boundary or "
+ "identity-injecting gateway after configuring server.root_api_key."
+ )
+
+
+async def _initialize_runtime_state(
+ app: FastAPI,
+ service: OpenVikingService,
+ config: ServerConfig,
+) -> None:
+ """Initialize service and auth dependencies before traffic is accepted."""
+ await service.initialize()
+ await _initialize_api_key_manager(app, service, config)
+ logger.info("OpenVikingService initialization complete")
+
+
def _format_error_location(loc: object) -> str:
if not isinstance(loc, (list, tuple)):
return "request"
@@ -173,51 +222,8 @@ def create_app(
_configure_session_tool_outputs(service)
async def _deferred_init(service, app, config):
- """Run heavy initialization in background after server starts accepting requests."""
- await service.initialize()
-
- # Initialize APIKeyManager after service (needs VikingFS)
- effective_auth_mode = config.get_effective_auth_mode()
- if config.root_api_key and config.root_api_key != "":
- api_key_manager = APIKeyManager(
- root_key=config.root_api_key,
- viking_fs=service.viking_fs,
- api_key_hashing_enabled=config.api_key_hashing_enabled,
- )
- await api_key_manager.load()
- app.state.api_key_manager = api_key_manager
- logger.info(
- "APIKeyManager initialized with api_key_hashing_enabled=%s",
- config.api_key_hashing_enabled,
- )
- if effective_auth_mode == AuthMode.TRUSTED:
- logger.info(
- "Trusted mode enabled: authentication trusts X-OpenViking-Account/User "
- "headers and requires the configured server API key on each request. "
- "Only expose this server behind a trusted network boundary or "
- "identity-injecting gateway."
- )
- elif effective_auth_mode == AuthMode.TRUSTED:
- app.state.api_key_manager = None
- if config.root_api_key and config.root_api_key != "":
- logger.info(
- "Trusted mode enabled: authentication trusts X-OpenViking-Account/User "
- "headers and requires the configured server API key on each request. "
- "Only expose this server behind a trusted network boundary or "
- "identity-injecting gateway."
- )
- else:
- logger.warning(
- "Trusted mode enabled: authentication uses X-OpenViking-Account/User "
- "headers without API keys. This is only allowed on localhost. "
- "Only expose this server behind a trusted network boundary or "
- "identity-injecting gateway after configuring server.root_api_key."
- )
- else:
- # AuthMode.DEV - logging already handled in validate_server_config
- app.state.api_key_manager = None
-
- logger.info("OpenVikingService initialization complete")
+ """Retained for tests that validate deferred-init callback behavior."""
+ await _initialize_runtime_state(app, service, config)
@asynccontextmanager
async def lifespan(app: FastAPI):
@@ -277,22 +283,11 @@ def create_app(
# Start MCP session manager (must be active before /mcp requests)
from openviking.server.mcp_endpoint import mcp_lifespan
- deferred_task = None
-
async with mcp_lifespan():
- # Start heavy initialization as background task after yield
if service is not None:
- deferred_task = asyncio.create_task(_deferred_init(service, app, config))
- deferred_task.add_done_callback(_on_deferred_init_done)
+ await _initialize_runtime_state(app, service, config)
yield
- # Wait for deferred initialization to complete before shutdown
- if deferred_task and not deferred_task.done():
- try:
- await deferred_task
- except Exception:
- logger.exception("Deferred initialization failed")
-
# Cleanup
from openviking.metrics.global_api import shutdown_metrics_async
from openviking.observability.usage_audit import shutdown_usage_audit
@@ -329,6 +324,7 @@ def create_app(
)
app.state.config = config
+ app.state.api_key_manager = None
set_server_config(config)
# Add CORS middleware
diff --git a/openviking/server/auth.py b/openviking/server/auth.py
index 30826f56b..9f961013b 100644
--- a/openviking/server/auth.py
+++ b/openviking/server/auth.py
@@ -38,6 +38,8 @@ _API_KEY_ROOT_ALLOWED_PREFIXES = (
"/api/v1/admin",
"/api/v1/observer",
"/api/v1/console",
+ "/api/v1/system/backend",
+ "/api/v1/system/sync",
)
diff --git a/openviking/server/routers/system.py b/openviking/server/routers/system.py
index 0b0058ab0..12af14117 100644
--- a/openviking/server/routers/system.py
+++ b/openviking/server/routers/system.py
@@ -9,10 +9,12 @@ from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel
+from openviking.core.path_variables import resolve_path_variables
from openviking.core.uri_validation import validate_viking_uri
-from openviking.server.auth import get_request_context, resolve_identity
+from openviking.pyagfs.exceptions import AGFSInvalidOperationError, AGFSNotSupportedError
+from openviking.server.auth import get_request_context, require_role, resolve_identity
from openviking.server.dependencies import get_service
-from openviking.server.identity import AuthMode, RequestContext
+from openviking.server.identity import AuthMode, RequestContext, Role
from openviking.server.models import Response
from openviking.storage.viking_fs import get_viking_fs
from openviking_cli.utils import get_logger
@@ -22,6 +24,36 @@ logger = get_logger(__name__)
router = APIRouter()
+def _is_ready_check_ok(value) -> bool:
+ """Return whether one readiness check value represents a healthy state."""
+ if isinstance(value, dict):
+ status = value.get("status")
+ if status not in ("ok", "not_configured", "not_supported"):
+ return False
+ nested = value.get("checks")
+ if nested is None:
+ return True
+ return all(_is_ready_check_ok(item) for item in nested.values())
+ return value in ("ok", "not_configured", "not_supported")
+
+
+async def _probe_agfs_readiness() -> dict[str, object]:
+ """Return structured AGFS readiness, including multi-write sync health when available."""
+ viking_fs = get_viking_fs()
+ checks: dict[str, object] = {}
+
+ await viking_fs.ls("viking://", ctx=None)
+ checks["filesystem"] = "ok"
+
+ try:
+ await viking_fs.system_sync_status("viking://", ctx=None)
+ checks["multiwrite_sync"] = "ok"
+ except (AGFSInvalidOperationError, AGFSNotSupportedError):
+ checks["multiwrite_sync"] = "not_supported"
+
+ return {"status": "ok", "checks": checks}
+
+
async def _embedding_probe(embedder) -> str:
"""Quick embedding probe: embed a single token and check for errors."""
from openviking.models.embedder.base import embed_compat
@@ -101,13 +133,11 @@ async def readiness_check(request: Request):
checks = {}
- # 1. AGFS: try to list root
+ # 1. AGFS: probe filesystem access and multi-write sync health
try:
- viking_fs = get_viking_fs()
- await viking_fs.ls("viking://", ctx=None)
- checks["agfs"] = "ok"
+ checks["agfs"] = await _probe_agfs_readiness()
except Exception as e:
- checks["agfs"] = f"error: {e}"
+ checks["agfs"] = {"status": "error", "checks": {"filesystem": f"error: {e}"}}
# 2. VectorDB: health_check()
try:
@@ -164,7 +194,7 @@ async def readiness_check(request: Request):
except Exception as e:
checks["ollama"] = f"error: {e}"
- all_ok = all(v in ("ok", "not_configured") for v in checks.values())
+ all_ok = all(_is_ready_check_ok(v) for v in checks.values())
status_code = 200 if all_ok else 503
return JSONResponse(
status_code=status_code,
@@ -204,6 +234,12 @@ class ConsistencyRequest(BaseModel):
uri: str
+class BackendSyncRequest(BaseModel):
+ """Request model for backend sync status and retry operations."""
+
+ uri: str
+
+
@router.post("/api/v1/system/wait", tags=["system"])
async def wait_processed(
request: WaitRequest,
@@ -228,3 +264,51 @@ async def check_consistency(
ctx=ctx,
)
return Response(status="ok", result=result)
+
+
[email protected]("/api/v1/system/backend/sync-status", tags=["system"])
+async def backend_sync_status(
+ request: BackendSyncRequest,
+ ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),
+):
+ """Return multi-write backend sync status for a Viking URI subtree."""
+ service = get_service()
+ uri = validate_viking_uri(resolve_path_variables(request.uri))
+ result = await service.fs.system_sync_status(uri, ctx=ctx)
+ return Response(status="ok", result=result)
+
+
[email protected]("/api/v1/system/backend/sync-retry", tags=["system"])
+async def backend_sync_retry(
+ request: BackendSyncRequest,
+ ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),
+):
+ """Retry pending multi-write backend sync work for a Viking URI subtree."""
+ service = get_service()
+ uri = validate_viking_uri(resolve_path_variables(request.uri))
+ result = await service.fs.system_sync_retry(uri, ctx=ctx)
+ return Response(status="ok", result=result)
+
+
[email protected]("/api/v1/system/sync/{sync_path:path}", tags=["system"])
+async def admin_sync_status(
+ sync_path: str,
+ ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),
+):
+ """Return multi-write backend sync status for one URI subtree through the admin API."""
+ service = get_service()
+ uri = validate_viking_uri(resolve_path_variables(sync_path))
+ result = await service.fs.system_sync_status(uri, ctx=ctx)
+ return Response(status="ok", result=result)
+
+
[email protected]("/api/v1/system/sync/{sync_path:path}/retry", tags=["system"])
+async def admin_sync_retry(
+ sync_path: str,
+ ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),
+):
+ """Retry pending multi-write backend sync work for one URI subtree through the admin API."""
+ service = get_service()
+ uri = validate_viking_uri(resolve_path_variables(sync_path))
+ result = await service.fs.system_sync_retry(uri, ctx=ctx)
+ return Response(status="ok", result=result)
diff --git a/openviking/server/routers/webdav.py b/openviking/server/routers/webdav.py
index c0944366c..88311e8b4 100644
--- a/openviking/server/routers/webdav.py
+++ b/openviking/server/routers/webdav.py
@@ -18,6 +18,7 @@ from fastapi.responses import Response as FastAPIResponse
from openviking.server.auth import get_request_context
from openviking.server.dependencies import get_service
from openviking.server.identity import RequestContext
+from openviking.storage.internal_names import WEBDAV_RESERVED_FILENAMES
from openviking.utils.time_utils import parse_iso_datetime
from openviking_cli.exceptions import InvalidArgumentError, NotFoundError
from openviking_cli.utils.uri import VikingURI
@@ -25,14 +26,6 @@ from openviking_cli.utils.uri import VikingURI
router = APIRouter(prefix="/webdav/resources", tags=["webdav"])
_DAV_NAMESPACE = "DAV:"
-_RESERVED_FILENAMES = frozenset(
- {
- ".abstract.md",
- ".overview.md",
- ".relations.json",
- ".path.ovlock",
- }
-)
_WEBDAV_METHODS = "OPTIONS, PROPFIND, GET, HEAD, PUT, DELETE, MKCOL, MOVE"
_TEXT_MEDIA_FALLBACK = "text/plain; charset=utf-8"
@@ -72,7 +65,7 @@ def _ensure_exposed_path(resource_path: str) -> None:
if not resource_path:
return
parts = resource_path.split("/")
- if any(part in _RESERVED_FILENAMES for part in parts):
+ if any(part in WEBDAV_RESERVED_FILENAMES for part in parts):
raise NotFoundError(resource_path, "resource")
@@ -206,7 +199,7 @@ def _exposed_child_entries(entries: list[dict[str, Any]]) -> list[dict[str, Any]
name = str(entry.get("name", "") or "")
if not name or name in {".", ".."}:
continue
- if name in _RESERVED_FILENAMES:
+ if name in WEBDAV_RESERVED_FILENAMES:
continue
exposed.append(entry)
return exposed
diff --git a/openviking/service/core.py b/openviking/service/core.py
index 5c8f42e2e..80697ea60 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -14,7 +14,6 @@ from typing import TYPE_CHECKING, Any, Optional
from openviking.core.directories import DirectoryInitializer
from openviking.crypto.config import bootstrap_encryption
from openviking.privacy import UserPrivacyConfigService
-from openviking.pyagfs.exceptions import AGFSNotFoundError
from openviking.resource.watch_scheduler import WatchScheduler
from openviking.server.identity import RequestContext, Role
from openviking.service.debug_service import DebugService
@@ -163,10 +162,6 @@ class OpenVikingService:
# Create RAGFS client using utility
runtime_binding_config = binding_config or RagfsBindingConfig(agfs=config.agfs)
self._agfs_client = create_agfs_client(runtime_binding_config)
- self._probe_storage_shape(
- self._agfs_client,
- encrypted_mode=runtime_binding_config.encryption_enabled(),
- )
# Initialize QueueManager with agfs_client
if self._agfs_client:
@@ -240,25 +235,6 @@ class OpenVikingService:
)
self._data_dir_lock_acquired = True
- def _probe_storage_shape(self, agfs_client: Any, encrypted_mode: bool) -> None:
- """Fail fast if existing system metadata shape conflicts with current encryption mode."""
- try:
- raw = agfs_client.read_raw("/local/_system/accounts.json")
- except AGFSNotFoundError:
- return
-
- is_encrypted = raw.startswith(b"OVE1")
- if encrypted_mode and not is_encrypted:
- raise RuntimeError(
- "Storage shape mismatch: encryption is enabled but existing "
- "remained data is plaintext"
- )
- if not encrypted_mode and is_encrypted:
- raise RuntimeError(
- "Storage shape mismatch: encryption is disabled but existing "
- "remained data is encrypted"
- )
-
@property
def _agfs(self) -> Any:
"""Internal access to AGFS client for APIKeyManager."""
diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.py
index 64a9d8355..c0cb47446 100644
--- a/openviking/service/fs_service.py
+++ b/openviking/service/fs_service.py
@@ -157,7 +157,9 @@ class FSService:
directory_uri = VikingURI(abstract_uri).parent.uri
return directory_uri, abstract_uri
- async def rm(self, uri: str, ctx: RequestContext, recursive: bool = False) -> Optional[Dict[str, Any]]:
+ async def rm(
+ self, uri: str, ctx: RequestContext, recursive: bool = False
+ ) -> Optional[Dict[str, Any]]:
"""Remove resource."""
uri = validate_viking_uri(uri)
viking_fs = self._ensure_initialized()
@@ -199,6 +201,18 @@ class FSService:
uri = validate_viking_uri(uri)
return await viking_fs.stat(uri, ctx=ctx)
+ async def system_sync_status(self, uri: str, ctx: RequestContext) -> Dict[str, Any]:
+ """Return multi-write sync status for one Viking URI subtree."""
+ viking_fs = self._ensure_initialized()
+ uri = validate_viking_uri(uri)
+ return await viking_fs.system_sync_status(uri, ctx=ctx)
+
+ async def system_sync_retry(self, uri: str, ctx: RequestContext) -> Dict[str, Any]:
+ """Retry multi-write sync work for one Viking URI subtree."""
+ viking_fs = self._ensure_initialized()
+ uri = validate_viking_uri(uri)
+ return await viking_fs.system_sync_retry(uri, ctx=ctx)
+
async def read(self, uri: str, ctx: RequestContext, offset: int = 0, limit: int = -1) -> str:
"""Read file content."""
viking_fs = self._ensure_initialized()
diff --git a/openviking/storage/internal_names.py b/openviking/storage/internal_names.py
new file mode 100644
index 000000000..db020d28d
--- /dev/null
+++ b/openviking/storage/internal_names.py
@@ -0,0 +1,33 @@
+"""Shared internal file-name constants for Python storage code."""
+
+from __future__ import annotations
+
+MULTIWRITE_PATH_LOCK_FILE = ".path.ovlock"
+MULTIWRITE_EXACT_LOCK_FILE_PREFIX = ".exact.ovlock."
+MULTIWRITE_REDIRECT_FILE = ".redirect.json"
+MULTIWRITE_SYNC_LOG_FILE = ".sync_log.json"
+
+MULTIWRITE_INTERNAL_FILE_NAMES = frozenset(
+ {
+ MULTIWRITE_PATH_LOCK_FILE,
+ MULTIWRITE_REDIRECT_FILE,
+ MULTIWRITE_SYNC_LOG_FILE,
+ }
+)
+
+STORAGE_INTERNAL_ENTRY_NAMES = frozenset(
+ {
+ "_system",
+ "tasks",
+ *MULTIWRITE_INTERNAL_FILE_NAMES,
+ }
+)
+
+WEBDAV_RESERVED_FILENAMES = frozenset(
+ {
+ ".abstract.md",
+ ".overview.md",
+ ".relations.json",
+ *MULTIWRITE_INTERNAL_FILE_NAMES,
+ }
+)
diff --git a/openviking/storage/transaction/path_lock.py b/openviking/storage/transaction/path_lock.py
index ef45dd20c..12e40ff42 100644
--- a/openviking/storage/transaction/path_lock.py
+++ b/openviking/storage/transaction/path_lock.py
@@ -7,14 +7,18 @@ from typing import Any, Callable, Optional, Tuple
from openviking.pyagfs import AGFSSyncClientProtocol, AsyncAGFSClient
from openviking.pyagfs.async_client import fs_ctx_from_agfs_path
+from openviking.storage.internal_names import (
+ MULTIWRITE_EXACT_LOCK_FILE_PREFIX,
+ MULTIWRITE_PATH_LOCK_FILE,
+)
from openviking.storage.transaction.lock_handle import LockOwner
from openviking_cli.utils.logger import get_logger
logger = get_logger(__name__)
# Lock file name
-LOCK_FILE_NAME = ".path.ovlock"
-EXACT_LOCK_FILE_PREFIX = ".exact.ovlock."
+LOCK_FILE_NAME = MULTIWRITE_PATH_LOCK_FILE
+EXACT_LOCK_FILE_PREFIX = MULTIWRITE_EXACT_LOCK_FILE_PREFIX
# Lock type constants
LOCK_TYPE_EXACT = "E"
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 59f4acb0c..3a46653a5 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -40,6 +40,10 @@ from openviking.resource.watch_storage import is_watch_task_control_uri
from openviking.server.error_mapping import is_not_found_error, map_exception
from openviking.server.identity import RequestContext, Role
from openviking.storage.expr import PathScope
+from openviking.storage.internal_names import (
+ MULTIWRITE_PATH_LOCK_FILE,
+ STORAGE_INTERNAL_ENTRY_NAMES,
+)
from openviking.telemetry import get_current_telemetry
from openviking.utils.time_utils import format_simplified, get_current_timestamp, parse_iso_datetime
from openviking_cli.exceptions import (
@@ -588,7 +592,7 @@ class VikingFS:
# Remove carried lock file from the copy (directory only)
if is_dir:
- carried_lock = new_path.rstrip("/") + "/.path.ovlock"
+ carried_lock = new_path.rstrip("/") + f"/{MULTIWRITE_PATH_LOCK_FILE}"
try:
await self._async_agfs.rm(carried_lock)
except Exception:
@@ -611,6 +615,30 @@ class VikingFS:
await self._async_agfs.rm(old_path, recursive=is_dir)
return {}
+ async def system_sync_status(
+ self, uri: str, ctx: Optional[RequestContext] = None
+ ) -> Dict[str, Any]:
+ """Return multi-write sync status for one Viking URI subtree."""
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ return await self._async_agfs.system_sync_status(
+ path,
+ fs_ctx={"account_id": real_ctx.account_id},
+ )
+
+ async def system_sync_retry(
+ self, uri: str, ctx: Optional[RequestContext] = None
+ ) -> Dict[str, Any]:
+ """Retry multi-write sync for one Viking URI subtree."""
+ self._ensure_mutable_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ return await self._async_agfs.system_sync_retry(
+ path,
+ fs_ctx={"account_id": real_ctx.account_id},
+ )
+
async def _copy_for_mv(
self,
old_uri: str,
@@ -623,7 +651,12 @@ class VikingFS:
) -> None:
"""Copy source to destination for mv without deleting source."""
if is_temp:
- await self._async_agfs.cp(old_path, new_path, recursive=is_dir)
+ await self._async_agfs.cp(
+ old_path,
+ new_path,
+ recursive=is_dir,
+ fs_ctx={"account_id": self._ctx_or_default(ctx).account_id},
+ )
return
if is_dir:
@@ -1481,12 +1514,12 @@ class VikingFS:
"""Check if name would appear in _ls_entries(parent_path).
At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.
- At other levels, uses _INTERNAL_NAMES blacklist.
+ At other levels, uses the shared storage internal-name blacklist.
"""
parts = [p for p in parent_path.strip("/").split("/") if p]
if len(parts) == 2 and parts[0] == "local":
return name in VikingURI.LISTABLE_SCOPES
- return name not in self._INTERNAL_NAMES
+ return name not in STORAGE_INTERNAL_ENTRY_NAMES
def _ancestor_is_filtered(self, entry_path: str, base_path: str) -> bool:
"""Check if any ancestor directory of entry_path would be filtered by _ls_entries.
@@ -1644,7 +1677,6 @@ class VikingFS:
safe_parts = [self._shorten_component(p, self._MAX_FILENAME_BYTES) for p in parts]
return f"/local/{account_id}/{'/'.join(safe_parts)}"
- _INTERNAL_NAMES = {"_system", "tasks", ".path.ovlock"}
_ROOT_PATH = "/local"
async def _ls_entries(
@@ -1653,13 +1685,13 @@ class VikingFS:
"""List directory entries, filtering out internal directories.
At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.
- At other levels, uses _INTERNAL_NAMES blacklist.
+ At other levels, uses the shared storage internal-name blacklist.
"""
entries = await self._async_agfs.ls(path)
parts = [p for p in path.strip("/").split("/") if p]
if len(parts) == 2 and parts[0] == "local":
return [e for e in entries if e.get("name") in VikingURI.LISTABLE_SCOPES]
- return [e for e in entries if e.get("name") not in self._INTERNAL_NAMES]
+ return [e for e in entries if e.get("name") not in STORAGE_INTERNAL_ENTRY_NAMES]
def _path_to_uri(self, path: str, ctx: Optional[RequestContext] = None) -> str:
"""/local/{account}/... -> viking://...
@@ -2321,7 +2353,12 @@ class VikingFS:
src_path = self._uri_to_path(temp_uri, ctx=ctx)
dst_path = self._uri_to_path(target_uri, ctx=ctx)
await self._ensure_parent_dirs(dst_path, ctx=ctx)
- await self._async_agfs.cp(src_path, dst_path, recursive=True)
+ await self._async_agfs.cp(
+ src_path,
+ dst_path,
+ recursive=True,
+ fs_ctx={"account_id": self._ctx_or_default(ctx).account_id},
+ )
async def delete_temp(self, temp_uri: str, ctx: Optional[RequestContext] = None) -> None:
"""Delete temp directory and its contents."""
diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.py
index dff1cd169..04b507f74 100644
--- a/openviking/utils/agfs_utils.py
+++ b/openviking/utils/agfs_utils.py
@@ -8,7 +8,7 @@ import multiprocessing
import os
from dataclasses import dataclass
from pathlib import Path
-from typing import Any, Dict
+from typing import Any, Callable, Dict
from openviking_cli.utils.logger import get_logger
@@ -106,7 +106,9 @@ def _build_queuefs_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str,
return plugin_config
-def _generate_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str, Any]:
+def _generate_plugin_config(
+ agfs_config: Any, data_path: Path, server_encryption_enabled: bool = False
+) -> Dict[str, Any]:
"""Dynamically generate RAGFS plugin configuration based on backend type."""
config = {
"serverinfofs": {
@@ -127,44 +129,204 @@ def _generate_plugin_config(agfs_config: Any, data_path: Path) -> Dict[str, Any]
s3_config = getattr(agfs_config, "s3", None)
vikingfs_path = data_path / "viking"
+ # Check for multi-write configuration
+ backups_config = getattr(agfs_config, "backups", None)
+ redirects_config = getattr(agfs_config, "redirects", None)
+ if redirects_config is not None and backups_config is None:
+ raise ValueError(
+ "redirects requires backups; single-backend mode does not support redirects"
+ )
+
+ # Build primary backend plugin config
+ primary_plugin_config: Dict[str, Any] = {}
+
if backend == "local":
- config["localfs"] = {
- "enabled": True,
- "path": "/local",
- "config": {
- "local_dir": str(vikingfs_path),
- },
+ primary_plugin_config = {
+ "local_dir": str(vikingfs_path),
}
elif backend == "s3" and s3_config:
- s3_plugin_config = {
- "bucket": s3_config.bucket,
- "region": s3_config.region,
- "access_key_id": s3_config.access_key,
- "secret_access_key": s3_config.secret_key,
- "endpoint": s3_config.endpoint,
- "prefix": s3_config.prefix,
- "disable_ssl": not s3_config.use_ssl,
- "use_path_style": s3_config.use_path_style,
- "directory_marker_mode": s3_config.directory_marker_mode.value
- if hasattr(s3_config.directory_marker_mode, "value")
- else s3_config.directory_marker_mode,
- "disable_batch_delete": s3_config.disable_batch_delete,
- "normalize_encoding_chars": s3_config.normalize_encoding_chars,
- }
+ primary_plugin_config = _serialize_s3_plugin_params(s3_config)
- config["s3fs"] = {
- "enabled": True,
- "path": "/local",
- "config": s3_plugin_config,
- }
+ # Build the mount config dict for the primary backend
+ mount_config: Dict[str, Any] = dict(primary_plugin_config)
+
+ # Add multi-write fields if backups are configured
+ if backups_config is not None:
+ # Serialize backups config to dict for FFI JSON passthrough
+ mount_config["backups"] = _serialize_backups_config(backups_config, data_path)
+ mount_config["server_encryption_enabled"] = server_encryption_enabled
+ mount_config["primary_encryption_enabled"] = server_encryption_enabled
+
+ # Serialize redirect policies
+ if redirects_config is not None:
+ mount_config["primary_redirects"] = [
+ _serialize_redirect_policy(p) for p in redirects_config
+ ]
+
+ # Determine the plugin type name for the primary backend
+ if backend == "local":
+ plugin_name = "localfs"
+ elif backend == "s3":
+ plugin_name = "s3fs"
elif backend == "memory":
- config["memfs"] = {
- "enabled": True,
- "path": "/local",
- }
+ plugin_name = "memfs"
+ else:
+ plugin_name = backend
+
+ config[plugin_name] = {
+ "enabled": True,
+ "path": "/local",
+ "config": mount_config,
+ }
+
return config
+def _map_backend_to_plugin_name(backend: str) -> str:
+ """Map user-facing backend name to Rust plugin name."""
+ mapping = {
+ "local": "localfs",
+ "s3": "s3fs",
+ "memory": "memfs",
+ }
+ return mapping.get(backend, backend)
+
+
+def _get_config_value(config: Any, key: str, default: Any = None) -> Any:
+ """Read one value from a dict-like or object-like config."""
+ if isinstance(config, dict):
+ return config.get(key, default)
+ return getattr(config, key, default)
+
+
+def _get_backend_specific_params(item: Any) -> Any:
+ """Return backend-specific nested params from one backup item using the backend name key."""
+ backend_params = _get_config_value(item, "backend_params")
+ if backend_params is not None:
+ return backend_params
+
+ backend_type = _get_config_value(item, "backend")
+ if not isinstance(backend_type, str):
+ return None
+ return _get_config_value(item, backend_type)
+
+
+def _serialize_s3_plugin_params(s3_config: Any) -> Dict[str, Any]:
+ """Serialize user-facing S3 config into Rust s3fs plugin parameters."""
+ directory_marker_mode = _get_config_value(s3_config, "directory_marker_mode")
+ return {
+ "bucket": _get_config_value(s3_config, "bucket"),
+ "region": _get_config_value(s3_config, "region"),
+ "access_key_id": _get_config_value(s3_config, "access_key"),
+ "secret_access_key": _get_config_value(s3_config, "secret_key"),
+ "endpoint": _get_config_value(s3_config, "endpoint"),
+ "prefix": _get_config_value(s3_config, "prefix", ""),
+ "disable_ssl": not _get_config_value(s3_config, "use_ssl", True),
+ "use_path_style": _get_config_value(s3_config, "use_path_style", True),
+ "directory_marker_mode": directory_marker_mode.value
+ if hasattr(directory_marker_mode, "value")
+ else directory_marker_mode,
+ "disable_batch_delete": _get_config_value(s3_config, "disable_batch_delete", False),
+ "normalize_encoding_chars": _get_config_value(
+ s3_config, "normalize_encoding_chars", "?#%+@"
+ ),
+ }
+
+
+def _dump_config_object(config: Any) -> Dict[str, Any]:
+ """Dump a pydantic or namespace-like config object without None values."""
+ if hasattr(config, "model_dump"):
+ return config.model_dump(exclude_none=True)
+ if isinstance(config, dict):
+ return {key: value for key, value in config.items() if value is not None}
+ return {
+ key: value
+ for key, value in vars(config).items()
+ if value is not None and not key.startswith("_")
+ }
+
+
+def _serialize_s3_backup_params(
+ _item: Any, backend_config: Any, _data_path: Path
+) -> Dict[str, Any]:
+ """Serialize one S3 backup item into the Rust s3fs parameter shape."""
+ if backend_config is None:
+ return {}
+ return _serialize_s3_plugin_params(backend_config)
+
+
+def _serialize_local_backup_params(
+ item: Any, backend_config: Any, data_path: Path
+) -> Dict[str, Any]:
+ """Serialize one local backup item and fill the default workspace local_dir."""
+ local_dir = (
+ _get_config_value(backend_config, "local_dir") if backend_config is not None else None
+ )
+ if local_dir is None:
+ local_dir = data_path / "viking" / "_backups" / _get_config_value(item, "name")
+ local_dir_path = Path(local_dir).expanduser()
+ return {"local_dir": str(local_dir_path)}
+
+
+def _serialize_generic_backup_params(
+ _item: Any, backend_config: Any, _data_path: Path
+) -> Dict[str, Any]:
+ """Serialize one backup item with the generic config dumper."""
+ if backend_config is None:
+ return {}
+ return _dump_config_object(backend_config)
+
+
+def _backup_param_serializers() -> Dict[str, Callable[[Any, Any, Path], Dict[str, Any]]]:
+ """Return the serializer registry keyed by user-facing backup backend type."""
+ return {
+ "s3": _serialize_s3_backup_params,
+ "local": _serialize_local_backup_params,
+ }
+
+
+def _serialize_backup_params(item: Any, data_path: Path) -> Dict[str, Any]:
+ """Serialize backend-specific params for one backup item via the serializer registry."""
+ backend_type = _get_config_value(item, "backend")
+ backend_config = _get_backend_specific_params(item)
+ serializer = _backup_param_serializers().get(backend_type, _serialize_generic_backup_params)
+ return serializer(item, backend_config, data_path)
+
+
+def _normalize_backup_item(item: Any, data_path: Path) -> Dict[str, Any]:
+ """Normalize one raw backup item for Rust while preserving unknown future fields."""
+ item_dict = _dump_config_object(item)
+ backend_type = _get_config_value(item, "backend")
+ item_dict["backend"] = _map_backend_to_plugin_name(backend_type)
+ item_dict.pop("backend_params", None)
+ if isinstance(backend_type, str):
+ item_dict.pop(backend_type, None)
+
+ params = _serialize_backup_params(item, data_path)
+ if params:
+ item_dict["params"] = params
+ else:
+ item_dict.pop("params", None)
+
+ return item_dict
+
+
+def _serialize_backups_config(backups_config: Any, data_path: Path) -> Dict[str, Any]:
+ """Serialize raw multi-write backups config with top-level passthrough and normalized items."""
+ result = _dump_config_object(backups_config)
+ result.setdefault("sync_type", "async")
+ result["items"] = [
+ _normalize_backup_item(item, data_path)
+ for item in _get_config_value(backups_config, "items", [])
+ ]
+ return result
+
+
+def _serialize_redirect_policy(policy: Any) -> Dict[str, Any]:
+ """Serialize one raw redirect/exclude policy object to a dict."""
+ return _dump_config_object(policy)
+
+
def create_agfs_client(config: RagfsBindingConfig) -> Any:
"""
Create a RAGFS client based on the provided configuration.
@@ -223,7 +385,12 @@ def mount_agfs_backend(agfs: Any, config: RagfsBindingConfig | Any) -> None:
vikingfs_path.mkdir(parents=True, exist_ok=True)
# 1. Mount standard plugins
- config = _generate_plugin_config(agfs_config, data_path)
+ server_encryption_enabled = (
+ config.encryption_enabled() if isinstance(config, RagfsBindingConfig) else False
+ )
+ config = _generate_plugin_config(
+ agfs_config, data_path, server_encryption_enabled=server_encryption_enabled
+ )
for plugin_name, plugin_config in config.items():
mount_path = plugin_config["path"]
@@ -232,6 +399,12 @@ def mount_agfs_backend(agfs: Any, config: RagfsBindingConfig | Any) -> None:
local_dir = plugin_config["config"]["local_dir"]
os.makedirs(local_dir, exist_ok=True)
logger.debug("[RAGFSUtils] Ensured localfs storage directory exists")
+ for backup_item in plugin_config.get("config", {}).get("backups", {}).get("items", []):
+ if backup_item.get("backend") != "localfs":
+ continue
+ backup_local_dir = backup_item.get("params", {}).get("local_dir")
+ if backup_local_dir:
+ os.makedirs(backup_local_dir, exist_ok=True)
# Ensure queuefs db_path parent directory exists before mounting
if plugin_name == "queuefs" and "db_path" in plugin_config.get("config", {}):
db_path = plugin_config["config"]["db_path"]
@@ -242,7 +415,12 @@ def mount_agfs_backend(agfs: Any, config: RagfsBindingConfig | Any) -> None:
except Exception:
pass
try:
- agfs.mount(plugin_name, mount_path, plugin_config.get("config", {}))
+ cfg = plugin_config.get("config", {})
+ logger.debug(
+ f"[RAGFSUtils] Mounting {plugin_name} at {mount_path} with config keys: {list(cfg.keys()) if isinstance(cfg, dict) else type(cfg)}"
+ )
+ agfs.mount(plugin_name, mount_path, cfg)
logger.debug(f"[RAGFSUtils] Successfully mounted {plugin_name}")
- except Exception:
- logger.error(f"[RAGFSUtils] Failed to mount {plugin_name}")
+ except Exception as e:
+ logger.error(f"[RAGFSUtils] Failed to mount {plugin_name}: {e}")
+ raise
diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.py
index 39617addf..d7652dab8 100644
--- a/openviking_cli/utils/config/agfs_config.py
+++ b/openviking_cli/utils/config/agfs_config.py
@@ -1,7 +1,9 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
+from __future__ import annotations
+
from enum import Enum
-from typing import Any, Optional
+from typing import Any, List, Optional
from pydantic import BaseModel, Field, model_validator
@@ -148,6 +150,11 @@ class QueueFSConfig(BaseModel):
class AGFSConfig(BaseModel):
"""Configuration for RAGFS (Rust-based AGFS)."""
+ name: str = Field(
+ default="primary",
+ description="Logical backend name, globally unique across primary and all backups",
+ )
+
path: Optional[str] = Field(
default=None,
description="[Deprecated in favor of `storage.workspace`] RAGFS data storage path. This will be ignored if `storage.workspace` is set.",
@@ -224,6 +231,14 @@ class AGFSConfig(BaseModel):
# RAGFS will act as a gateway to the specified S3 bucket.
s3: S3Config = Field(default_factory=lambda: S3Config(), description="S3 backend configuration")
+ # Multi-write configuration
+ backups: Optional[dict[str, Any]] = Field(
+ default=None, description="Multi-write backups configuration. None = single backend mode."
+ )
+ redirects: Optional[List[dict[str, Any]]] = Field(
+ default=None, description="Primary redirect policies."
+ )
+
model_config = {"extra": "forbid"}
@model_validator(mode="after")
@@ -271,4 +286,9 @@ class AGFSConfig(BaseModel):
"db_path/queue_db_path will be ignored."
)
+ if self.redirects is not None and self.backups is None:
+ raise ValueError(
+ "redirects requires backups; single-backend mode does not support redirects"
+ )
+
return self
diff --git a/tests/integration/test_encryption_integration.py b/tests/integration/test_encryption_integration.py
index bc3a12458..80aa11b2e 100644
--- a/tests/integration/test_encryption_integration.py
+++ b/tests/integration/test_encryption_integration.py
@@ -307,7 +307,31 @@ class TestVikingFSEncryptionWithAccounts:
except Exception:
return False
- def _check_all_files_encrypted(self, svc, ctx, base_uri: str, print_paths: bool = True) -> None:
+ def _agfs_data_root(self, test_data_dir: Path) -> Path:
+ """Return the mounted localfs root used by integration tests."""
+ return test_data_dir / "viking" / "viking"
+
+ def _backend_file_path(self, svc, ctx, test_data_dir: Path, uri: str) -> Path:
+ """Map one Viking URI to the underlying backend file path."""
+ agfs_path = svc.viking_fs._uri_to_path(uri, ctx=ctx)
+ rel_path = agfs_path.removeprefix("/local/").lstrip("/")
+ return self._agfs_data_root(test_data_dir) / rel_path
+
+ def _backend_file_bytes(self, svc, ctx, test_data_dir: Path, uri: str) -> bytes:
+ """Read the underlying backend bytes for one Viking URI."""
+ file_path = self._backend_file_path(svc, ctx, test_data_dir, uri)
+ assert file_path.exists(), f"Backend file missing for {uri}: {file_path}"
+ return file_path.read_bytes()
+
+ def _assert_uri_encrypted(self, svc, ctx, test_data_dir: Path, uri: str) -> bytes:
+ """Assert one Viking URI is stored as ciphertext on disk."""
+ raw_content = self._backend_file_bytes(svc, ctx, test_data_dir, uri)
+ assert raw_content.startswith(b"OVE1"), f"File not encrypted: {uri}"
+ return raw_content
+
+ def _check_all_files_encrypted(
+ self, svc, ctx, test_data_dir: Path, base_uri: str, print_paths: bool = True
+ ) -> None:
"""
Recursively check if all files in a directory are encrypted.
@@ -317,7 +341,6 @@ class TestVikingFSEncryptionWithAccounts:
base_uri: Base URI to check
print_paths: Whether to print encrypted file paths
"""
- agfs_client = svc._agfs_client
async def _check_recursive(uri: str):
try:
@@ -331,11 +354,7 @@ class TestVikingFSEncryptionWithAccounts:
if ".relations.json" in entry.get("name", ""):
continue
try:
- agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1"), (
- f"File not encrypted: {entry_uri}"
- )
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, entry_uri)
if print_paths:
print(f"[ENCRYPTED] {entry_uri}")
except AssertionError:
@@ -460,10 +479,8 @@ class TestVikingFSEncryptionWithAccounts:
read_content = await svc.viking_fs.read_file(test_uri, ctx=ctx)
assert read_content == test_content, "Read content does not match original content"
- # Verify encryption by reading raw file content directly via AGFS
- agfs_client = svc._agfs_client
- agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
+ # Verify encryption by reading backend file content directly
+ raw_content = self._backend_file_bytes(svc, ctx, data["test_data_dir"], test_uri)
assert raw_content.startswith(b"OVE1"), (
f"File not encrypted, raw content start: {raw_content[:10]}"
)
@@ -548,10 +565,8 @@ This is a test skill for verifying encryption functionality.
assert "Test Skill" in skill_content
# Verify encryption by reading raw file content directly via AGFS
- agfs_client = svc._agfs_client
skill_md_uri = f"{skill_uri}/SKILL.md"
- agfs_path = svc.viking_fs._uri_to_path(skill_md_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
+ raw_content = self._backend_file_bytes(svc, ctx, data["test_data_dir"], skill_md_uri)
assert raw_content.startswith(b"OVE1"), (
f"Skill file not encrypted, raw content start: {raw_content[:10]}"
)
@@ -559,7 +574,9 @@ This is a test skill for verifying encryption functionality.
"Skill file content is plaintext, not encrypted"
)
- async def _check_all_files_encrypted(self, root_uri, ctx, svc, agfs_client, prefix=" "):
+ async def _check_all_files_encrypted(
+ self, root_uri, ctx, svc, test_data_dir: Path, prefix=" "
+ ):
"""
Recursively check if all files are encrypted
@@ -567,17 +584,16 @@ This is a test skill for verifying encryption functionality.
root_uri: Root URI
ctx: Request context
svc: OpenVikingService instance
- agfs_client: AGFS client
+ test_data_dir: Integration test data directory
prefix: Output prefix
"""
try:
# First try to get URI path to see if it's a file
try:
- agfs_path = svc.viking_fs._uri_to_path(root_uri, ctx=ctx)
- if agfs_client.exists(agfs_path) and not agfs_client.isdir(agfs_path):
+ stat_info = await svc.viking_fs.stat(root_uri, ctx=ctx)
+ if not stat_info["isDir"]:
# This is a file, check directly
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1"), f"File not encrypted: {root_uri}"
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, root_uri)
print(f"{prefix}✓ [ENCRYPTED] {root_uri}")
return
except Exception:
@@ -589,13 +605,11 @@ This is a test skill for verifying encryption functionality.
if entry["isDir"]:
# Recursively check subdirectories
await self._check_all_files_encrypted(
- entry["uri"], ctx, svc, agfs_client, prefix + " "
+ entry["uri"], ctx, svc, test_data_dir, prefix + " "
)
else:
# Check file encryption
- agfs_path = svc.viking_fs._uri_to_path(entry["uri"], ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1"), f"File not encrypted: {entry['uri']}"
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, entry["uri"])
print(f"{prefix}✓ [ENCRYPTED] {entry['uri']}")
except Exception as e:
print(f"{prefix}[WARNING] Error checking {root_uri}: {e}")
@@ -610,6 +624,7 @@ This is a test skill for verifying encryption functionality.
data = openviking_service_with_encryption
svc = data["service"]
api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
# ========== Prerequisites ==========
print("\n" + "=" * 80)
@@ -657,10 +672,9 @@ This is a test skill for verifying encryption functionality.
ctx = RequestContext(user=test_user, role=Role.USER)
root_ctx = RequestContext(user=test_user, role=Role.ROOT)
- agfs_client = svc._agfs_client
account_root_uri = f"viking://{test_account_id}"
- await self._check_all_files_encrypted(account_root_uri, ctx, svc, agfs_client)
+ await self._check_all_files_encrypted(account_root_uri, ctx, svc, test_data_dir)
# ========== Execute tests ==========
print("\n" + "=" * 80)
@@ -688,7 +702,7 @@ This is a test skill for verifying encryption functionality.
# Check all files in resources directory are encrypted
print("[1.1] Check files in resources directory are encrypted")
resources_dir_uri = "viking://resources"
- await self._check_all_files_encrypted(resources_dir_uri, ctx, svc, agfs_client)
+ await self._check_all_files_encrypted(resources_dir_uri, ctx, svc, test_data_dir)
# 1.2 Execute various operations to verify returns unencrypted
print("[1.2] Verify various operations return unencrypted content")
@@ -786,7 +800,7 @@ This is a test skill for verifying encryption functionality.
# Check all files in the skill directory are encrypted
print("[2.1] Check files in skill directory are encrypted")
- await self._check_all_files_encrypted(test_skill_dir_uri, root_ctx, svc, agfs_client)
+ await self._check_all_files_encrypted(test_skill_dir_uri, root_ctx, svc, test_data_dir)
# 2.2 Verify various operations return unencrypted
print("[2.2] Verify various operations return unencrypted content")
@@ -881,7 +895,7 @@ This is a test skill for verifying encryption functionality.
# Check all files in user directory are encrypted
print("[3.1] Check files in user directory are encrypted")
user_dir_uri = f"viking://{test_account_id}/user/{test_user_id}"
- await self._check_all_files_encrypted(user_dir_uri, ctx, svc, agfs_client)
+ await self._check_all_files_encrypted(user_dir_uri, ctx, svc, test_data_dir)
# 3.2 Verify various operations return unencrypted
print("[3.2] Verify various operations return unencrypted content")
@@ -953,7 +967,7 @@ This is a test skill for verifying encryption functionality.
# Check session directory files are encrypted
print("[4.1] Check files in session directory are encrypted")
session_dir_uri = f"viking://{test_account_id}/session"
- await self._check_all_files_encrypted(session_dir_uri, ctx, svc, agfs_client)
+ await self._check_all_files_encrypted(session_dir_uri, ctx, svc, test_data_dir)
# 4.2 Verify various operations return unencrypted
print("[4.2] Verify various operations return unencrypted content")
@@ -1015,7 +1029,7 @@ This is a test skill for verifying encryption functionality.
# Check message.jsonl file encryption
print("[4.3] Check messages file encryption")
- await self._check_all_files_encrypted(session_dir_uri, ctx, svc, agfs_client)
+ await self._check_all_files_encrypted(session_dir_uri, ctx, svc, test_data_dir)
# 4.4 Verify read operation returns unencrypted
print("[4.4] Re-read session to verify message")
@@ -1066,8 +1080,9 @@ This is a test skill for verifying encryption functionality.
print("[5.2] Check relation.json file encryption")
try:
relation_file_uri = f"{dir_a_uri}/.relations.json"
- agfs_path = svc.viking_fs._uri_to_path(relation_file_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
+ raw_content = self._backend_file_bytes(
+ svc, ctx, data["test_data_dir"], relation_file_uri
+ )
assert raw_content.startswith(b"OVE1"), (
f"relation.json not encrypted: {relation_file_uri}"
)
@@ -1138,9 +1153,7 @@ This is a test skill for verifying encryption functionality.
assert from_line_3.rstrip("\n") == expected_from_line_3, "Read from offset failed"
# Verify file is encrypted on disk
- agfs_client = svc._agfs_client
- agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
+ raw_content = self._backend_file_bytes(svc, ctx, data["test_data_dir"], test_uri)
assert raw_content.startswith(b"OVE1"), "File should be encrypted"
print("\n" + "=" * 80)
@@ -1186,9 +1199,7 @@ This is a test skill for verifying encryption functionality.
assert from_byte_10.rstrip(b"\n") == expected_from_byte_10, "Read from offset failed"
# Verify file is encrypted on disk
- agfs_client = svc._agfs_client
- agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
+ raw_content = self._backend_file_bytes(svc, ctx, data["test_data_dir"], test_uri)
assert raw_content.startswith(b"OVE1"), "File should be encrypted"
print("\n" + "=" * 80)
@@ -1292,8 +1303,6 @@ class TestAddResourceWithSemanticProcessing:
test_user = UserIdentifier(test_account_id, test_user_id)
ctx = RequestContext(user=test_user, role=Role.USER)
- agfs_client = svc._agfs_client
-
print("[3] Create test resource files (including directory structure)")
test_dir = tmp_path / "test_abstract_dir"
test_dir.mkdir()
@@ -1347,8 +1356,9 @@ This is a file in subdir2.
if ".relations.json" in entry_name:
continue
try:
- agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
+ raw_content = self._backend_file_bytes(
+ svc, ctx, data["test_data_dir"], entry_uri
+ )
assert raw_content.startswith(b"OVE1"), (
f"File not encrypted: {entry_uri}"
)
@@ -1407,8 +1417,9 @@ This is a file in subdir2.
if entry.get("isDir"):
try:
abstract_md_uri = f"{entry['uri']}/.abstract.md"
- agfs_path = svc.viking_fs._uri_to_path(abstract_md_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
+ raw_content = self._backend_file_bytes(
+ svc, ctx, data["test_data_dir"], abstract_md_uri
+ )
assert raw_content.startswith(b"OVE1"), (
f".abstract.md not encrypted: {abstract_md_uri}"
)
diff --git a/tests/integration/test_vault_encryption_integration.py b/tests/integration/test_vault_encryption_integration.py
index c3476ed5c..c080ce933 100644
--- a/tests/integration/test_vault_encryption_integration.py
+++ b/tests/integration/test_vault_encryption_integration.py
@@ -389,8 +389,24 @@ class TestVikingFSEncryptionWithVault:
except Exception:
return False
+ def _agfs_data_root(self, test_data_dir: Path) -> Path:
+ """Return the mounted localfs root used by integration tests."""
+ return test_data_dir / "viking" / "viking"
+
+ def _backend_file_path(self, svc, ctx, test_data_dir: Path, uri: str) -> Path:
+ """Map one Viking URI to the underlying backend file path."""
+ agfs_path = svc.viking_fs._uri_to_path(uri, ctx=ctx)
+ rel_path = agfs_path.removeprefix("/local/").lstrip("/")
+ return self._agfs_data_root(test_data_dir) / rel_path
+
+ def _assert_uri_encrypted(self, svc, ctx, test_data_dir: Path, uri: str) -> None:
+ """Assert one Viking URI is stored as ciphertext on disk."""
+ file_path = self._backend_file_path(svc, ctx, test_data_dir, uri)
+ assert file_path.exists(), f"Backend file missing for {uri}: {file_path}"
+ assert self._is_file_encrypted(file_path), f"File {uri} is not encrypted"
+
async def _check_all_files_encrypted(
- self, svc, ctx, base_uri: str, print_paths: bool = True
+ self, svc, ctx, test_data_dir: Path, base_uri: str, print_paths: bool = True
) -> None:
"""
Recursively check if all files in directory are encrypted.
@@ -401,7 +417,6 @@ class TestVikingFSEncryptionWithVault:
base_uri: Base URI to check
print_paths: Whether to print encrypted file paths
"""
- agfs_client = svc._agfs_client
async def _check_recursive(uri: str):
try:
@@ -411,10 +426,7 @@ class TestVikingFSEncryptionWithVault:
if entry["isDir"]:
await _check_recursive(entry_uri)
else:
- # Use VikingFS's _uri_to_path method to get file path
- agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1"), f"File {entry_uri} is not encrypted"
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, entry_uri)
if print_paths:
print(f"✓ File is encrypted: {entry_uri}")
except Exception as e:
@@ -433,6 +445,7 @@ class TestVikingFSEncryptionWithVault:
data = openviking_service_with_vault_encryption
svc = data["service"]
api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
# Create test account
account_id = "test-account-resources"
@@ -456,10 +469,7 @@ class TestVikingFSEncryptionWithVault:
assert read_content == test_content
# Verify file is encrypted
- agfs_client = svc._agfs_client
- agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1")
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, test_uri)
# Test various operations
resources_dir_uri = f"viking://{account_id}/resources"
@@ -499,6 +509,7 @@ class TestVikingFSEncryptionWithVault:
data = openviking_service_with_vault_encryption
svc = data["service"]
api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
# Create test account
account_id = "test-account-skills"
@@ -535,10 +546,7 @@ This is a test skill for verifying Vault encryption.
assert "Test Skill" in read_content
# Verify file is encrypted
- agfs_client = svc._agfs_client
- agfs_path = svc.viking_fs._uri_to_path(skill_md_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1")
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, skill_md_uri)
# Test various operations
agent_dir_uri = f"viking://{account_id}/agent"
@@ -562,6 +570,7 @@ This is a test skill for verifying Vault encryption.
data = openviking_service_with_vault_encryption
svc = data["service"]
api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
# Create test account
account_id = "test-account-memories"
@@ -592,10 +601,7 @@ This is a test skill for verifying Vault encryption.
assert "User Preferences" in read_content
# Verify file is encrypted
- agfs_client = svc._agfs_client
- agfs_path = svc.viking_fs._uri_to_path(memory_file_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1")
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, memory_file_uri)
@pytest.mark.asyncio
async def test_session_operations_with_encryption(
@@ -605,6 +611,7 @@ This is a test skill for verifying Vault encryption.
data = openviking_service_with_vault_encryption
svc = data["service"]
api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
# Create test account
account_id = "test-account-sessions"
@@ -623,7 +630,9 @@ This is a test skill for verifying Vault encryption.
# Check if session directory files are encrypted
session_dir_uri = f"viking://{account_id}/session"
- await self._check_all_files_encrypted(svc, ctx, session_dir_uri, print_paths=False)
+ await self._check_all_files_encrypted(
+ svc, ctx, test_data_dir, session_dir_uri, print_paths=False
+ )
@pytest.mark.asyncio
async def test_complete_encryption_workflow_with_vault(
@@ -642,7 +651,7 @@ This is a test skill for verifying Vault encryption.
svc = openviking_service_with_vault_encryption["service"]
api_key_manager = openviking_service_with_vault_encryption["api_key_manager"]
- openviking_service_with_vault_encryption["test_data_dir"]
+ test_data_dir = openviking_service_with_vault_encryption["test_data_dir"]
# 1. Precondition: Create random account and user
random_account_id = f"test-account-{uuid.uuid4()}"
@@ -698,7 +707,9 @@ description: Test skill for complete workflow
# Check if all files are encrypted
account_root_uri = f"viking://{random_account_id}"
- await self._check_all_files_encrypted(svc, ctx, account_root_uri, print_paths=True)
+ await self._check_all_files_encrypted(
+ svc, ctx, test_data_dir, account_root_uri, print_paths=True
+ )
# 4. Postcondition: Delete account
print("\n=== Cleanup ===")
diff --git a/tests/integration/test_volcengine_kms_encryption_integration.py b/tests/integration/test_volcengine_kms_encryption_integration.py
index 048d069f7..51b2f7654 100644
--- a/tests/integration/test_volcengine_kms_encryption_integration.py
+++ b/tests/integration/test_volcengine_kms_encryption_integration.py
@@ -395,11 +395,26 @@ class TestVikingFSEncryptionWithVolcengineKMS:
except Exception:
return False
+ def _agfs_data_root(self, test_data_dir: Path) -> Path:
+ """Return the mounted localfs root used by integration tests."""
+ return test_data_dir / "viking" / "viking"
+
+ def _backend_file_path(self, svc, ctx, test_data_dir: Path, uri: str) -> Path:
+ """Map one Viking URI to the underlying backend file path."""
+ agfs_path = svc.viking_fs._uri_to_path(uri, ctx=ctx)
+ rel_path = agfs_path.removeprefix("/local/").lstrip("/")
+ return self._agfs_data_root(test_data_dir) / rel_path
+
+ def _assert_uri_encrypted(self, svc, ctx, test_data_dir: Path, uri: str) -> None:
+ """Assert one Viking URI is stored as ciphertext on disk."""
+ file_path = self._backend_file_path(svc, ctx, test_data_dir, uri)
+ assert file_path.exists(), f"Backend file missing for {uri}: {file_path}"
+ assert self._is_file_encrypted(file_path), f"File {uri} is not encrypted"
+
async def _check_all_files_encrypted(
- self, svc, ctx, base_uri: str, print_paths: bool = True
+ self, svc, ctx, test_data_dir: Path, base_uri: str, print_paths: bool = True
) -> None:
"""Recursively check if all files in directory are encrypted"""
- agfs_client = svc._agfs_client
async def _check_recursive(uri: str):
try:
@@ -409,10 +424,7 @@ class TestVikingFSEncryptionWithVolcengineKMS:
if entry["isDir"]:
await _check_recursive(entry_uri)
else:
- # Use VikingFS's _uri_to_path method to get file path
- agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1"), f"File {entry_uri} is not encrypted"
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, entry_uri)
if print_paths:
print(f"✓ File is encrypted: {entry_uri}")
except Exception as e:
@@ -480,7 +492,9 @@ class TestVikingFSEncryptionWithVolcengineKMS:
assert resources_dir.exists()
- await self._check_all_files_encrypted(svc, ctx, f"viking:///{account_id}/resources")
+ await self._check_all_files_encrypted(
+ svc, ctx, test_data_dir, f"viking:///{account_id}/resources"
+ )
finally:
import os
@@ -495,6 +509,7 @@ class TestVikingFSEncryptionWithVolcengineKMS:
data = openviking_service_with_volcengine_encryption
svc = data["service"]
api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
# Create test account
account_id = "test-account-resources"
@@ -518,10 +533,7 @@ class TestVikingFSEncryptionWithVolcengineKMS:
assert read_content == test_content
# Verify file is encrypted
- agfs_client = svc._agfs_client
- agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1")
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, test_uri)
# Test various operations
resources_dir_uri = f"viking://{account_id}/resources"
@@ -563,6 +575,7 @@ class TestVikingFSEncryptionWithVolcengineKMS:
data = openviking_service_with_volcengine_encryption
svc = data["service"]
api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
# Create test account
account_id = "test-account-skills"
@@ -599,10 +612,7 @@ This is a test skill for verifying Volcengine KMS encryption.
assert "Test Skill" in read_content
# Verify file is encrypted
- agfs_client = svc._agfs_client
- agfs_path = svc.viking_fs._uri_to_path(skill_md_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1")
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, skill_md_uri)
# Test various operations
agent_dir_uri = f"viking://{account_id}/agent"
@@ -626,6 +636,7 @@ This is a test skill for verifying Volcengine KMS encryption.
data = openviking_service_with_volcengine_encryption
svc = data["service"]
api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
# Create test account
account_id = "test-account-memories"
@@ -656,10 +667,7 @@ This is a test skill for verifying Volcengine KMS encryption.
assert "User Preferences" in read_content
# Verify file is encrypted
- agfs_client = svc._agfs_client
- agfs_path = svc.viking_fs._uri_to_path(memory_file_uri, ctx=ctx)
- raw_content = agfs_client.read_raw(agfs_path)
- assert raw_content.startswith(b"OVE1")
+ self._assert_uri_encrypted(svc, ctx, test_data_dir, memory_file_uri)
@pytest.mark.asyncio
async def test_session_operations_with_encryption(
@@ -669,6 +677,7 @@ This is a test skill for verifying Volcengine KMS encryption.
data = openviking_service_with_volcengine_encryption
svc = data["service"]
api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
# Create test account
account_id = "test-account-sessions"
@@ -687,7 +696,9 @@ This is a test skill for verifying Volcengine KMS encryption.
# Check if session directory files are encrypted
session_dir_uri = f"viking://{account_id}/session"
- await self._check_all_files_encrypted(svc, ctx, session_dir_uri, print_paths=False)
+ await self._check_all_files_encrypted(
+ svc, ctx, test_data_dir, session_dir_uri, print_paths=False
+ )
@pytest.mark.asyncio
async def test_complete_encryption_workflow(
@@ -697,6 +708,7 @@ This is a test skill for verifying Volcengine KMS encryption.
data = openviking_service_with_volcengine_encryption
svc = data["service"]
api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
# Generate random account name
import secrets
@@ -765,7 +777,9 @@ description: Test skill for complete workflow
# 5. Verify all files are encrypted
print("\n5. Verifying all files are encrypted...")
account_root_uri = f"viking://{test_account_id}"
- await self._check_all_files_encrypted(svc, ctx, account_root_uri, print_paths=False)
+ await self._check_all_files_encrypted(
+ svc, ctx, test_data_dir, account_root_uri, print_paths=False
+ )
print("✓ All files are encrypted")
print("\n✓ Complete Volcengine KMS encryption workflow test passed!")
diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.py
index 5f50f1672..60fdd7b26 100644
--- a/tests/misc/test_config_validation.py
+++ b/tests/misc/test_config_validation.py
@@ -102,142 +102,269 @@ def test_agfs_s3_normalize_encoding_chars_is_forwarded_to_ragfs_plugin_config():
print(f" Fail: {e}")
-def test_agfs_queuefs_defaults_to_sqlite_backend():
- config = AGFSConfig(path="/tmp/ov-test", backend="local")
+ ("queuefs", "expected"),
+ [
+ (
+ None,
+ {
+ "mode": "shared",
+ "backend": "sqlite",
+ "recover_stale_sec": 0,
+ "busy_timeout_ms": 5000,
+ },
+ ),
+ (
+ {"mode": "worker", "backend": "memory"},
+ {
+ "mode": "worker",
+ "backend": "memory",
+ "recover_stale_sec": 0,
+ "busy_timeout_ms": 5000,
+ },
+ ),
+ ],
+)
+def test_agfs_queuefs_validation_accepts_supported_shapes(queuefs, expected):
+ config_kwargs = {"path": "/tmp/ov-test", "backend": "local"}
+ if queuefs is not None:
+ config_kwargs["queuefs"] = queuefs
+
+ config = AGFSConfig(**config_kwargs)
+
+ assert config.queuefs.mode == expected["mode"]
+ assert config.queuefs.backend == expected["backend"]
+ assert config.queuefs.recover_stale_sec == expected["recover_stale_sec"]
+ assert config.queuefs.busy_timeout_ms == expected["busy_timeout_ms"]
+
+
+ ("queuefs", "match"),
+ [
+ ({"mode": "process"}, "queuefs mode"),
+ ({"backend": "bogus"}, "queuefs"),
+ ({"busy_timeout_ms": -1}, "busy_timeout_ms"),
+ ({"recover_stale_sec": -1}, "recover_stale_sec"),
+ ],
+)
+def test_agfs_queuefs_validation_rejects_invalid_shapes(queuefs, match):
+ with pytest.raises(ValueError, match=match):
+ AGFSConfig(path="/tmp/ov-test", backend="local", queuefs=queuefs)
+
+
+ ("queuefs", "queue_db_path", "expected"),
+ [
+ (
+ {"backend": "memory"},
+ None,
+ {"backend": "memory", "db_path": None},
+ ),
+ (
+ {"backend": "sqlite", "db_path": "/tmp/new-queue.db"},
+ "/tmp/legacy-queue.db",
+ {"backend": "sqlite", "db_path": str(Path("/tmp/new-queue.db").resolve())},
+ ),
+ (
+ None,
+ "/tmp/legacy-queue.db",
+ {"backend": "sqlite", "db_path": str(Path("/tmp/legacy-queue.db").resolve())},
+ ),
+ (
+ None,
+ None,
+ {"backend": "sqlite", "db_path": "/tmp/ov-test/_system/queue/queue.db"},
+ ),
+ (
+ {"backend": "memory", "db_path": "/tmp/new-queue.db"},
+ "/tmp/legacy-queue.db",
+ {"backend": "memory", "db_path": None},
+ ),
+ ],
+)
+def test_generate_plugin_config_materializes_queuefs_paths(queuefs, queue_db_path, expected):
+ config_kwargs = {
+ "path": "/tmp/ov-test",
+ "backend": "local",
+ "queue_db_path": queue_db_path,
+ }
+ if queuefs is not None:
+ config_kwargs["queuefs"] = queuefs
+
+ config = AGFSConfig(**config_kwargs)
+ plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
- assert config.queuefs.mode == "shared"
- assert config.queuefs.backend == "sqlite"
- assert config.queuefs.recover_stale_sec == 0
- assert config.queuefs.busy_timeout_ms == 5000
+ queuefs_config = plugins["queuefs"]["config"]
+ assert queuefs_config["backend"] == expected["backend"]
+ if expected["db_path"] is None:
+ assert "db_path" not in queuefs_config
+ else:
+ assert queuefs_config["db_path"] == expected["db_path"]
-def test_agfs_queuefs_accepts_memory_backend():
+def test_generate_plugin_config_forwards_queuefs_runtime_options():
config = AGFSConfig(
path="/tmp/ov-test",
backend="local",
- queuefs={"mode": "worker", "backend": "memory"},
+ queuefs={
+ "backend": "sqlite3",
+ "recover_stale_sec": 17,
+ "busy_timeout_ms": 1234,
+ },
)
- assert config.queuefs.mode == "worker"
- assert config.queuefs.backend == "memory"
-
-
-def test_agfs_queuefs_rejects_legacy_process_mode():
- with pytest.raises(ValueError, match="queuefs mode"):
- AGFSConfig(
- path="/tmp/ov-test",
- backend="local",
- queuefs={"mode": "process"},
- )
-
+ plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
-def test_agfs_queuefs_rejects_invalid_backend():
- with pytest.raises(ValueError, match="queuefs"):
- AGFSConfig(
- path="/tmp/ov-test",
- backend="local",
- queuefs={"backend": "bogus"},
- )
+ assert plugins["queuefs"]["config"]["backend"] == "sqlite3"
+ assert plugins["queuefs"]["config"]["recover_stale_sec"] == 17
+ assert plugins["queuefs"]["config"]["busy_timeout_ms"] == 1234
-def test_agfs_queuefs_rejects_negative_timeouts():
- with pytest.raises(ValueError, match="busy_timeout_ms"):
+def test_agfs_redirects_require_backups():
+ """Single-backend mode must reject redirect policies during config validation."""
+ with pytest.raises(ValueError, match="redirects requires backups"):
AGFSConfig(
path="/tmp/ov-test",
backend="local",
- queuefs={"busy_timeout_ms": -1},
+ redirects=[
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": ["(md)"],
+ "target": ["s3-backup"],
+ }
+ ],
)
- with pytest.raises(ValueError, match="recover_stale_sec"):
- AGFSConfig(
- path="/tmp/ov-test",
- backend="local",
- queuefs={"recover_stale_sec": -1},
- )
+def test_generate_plugin_config_rejects_redirects_without_backups(tmp_path):
+ """Runtime plugin config generation must also reject redirect-only configs."""
+ config = type(
+ "RedirectOnlyConfig",
+ (),
+ {
+ "backend": "local",
+ "s3": None,
+ "backups": None,
+ "redirects": [
+ type(
+ "RedirectPolicy",
+ (),
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": ["(md)"],
+ "target": ["s3-backup"],
+ },
+ )()
+ ],
+ "queuefs": type(
+ "QueueConfig",
+ (),
+ {
+ "mode": "shared",
+ "backend": "sqlite",
+ "db_path": None,
+ "recover_stale_sec": 0,
+ "busy_timeout_ms": 5000,
+ },
+ )(),
+ "queue_db_path": None,
+ },
+ )()
-def test_generate_plugin_config_uses_queuefs_memory_backend_without_db_path():
- config = AGFSConfig(
- path="/tmp/ov-test",
- backend="local",
- queuefs={"backend": "memory"},
- )
-
- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
-
- assert plugins["queuefs"]["config"]["backend"] == "memory"
- assert "db_path" not in plugins["queuefs"]["config"]
-
-
-def test_generate_plugin_config_uses_new_queuefs_db_path_over_legacy_field():
- config = AGFSConfig(
- path="/tmp/ov-test",
- backend="local",
- queue_db_path="/tmp/legacy-queue.db",
- queuefs={"backend": "sqlite", "db_path": "/tmp/new-queue.db"},
- )
-
- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
-
- assert plugins["queuefs"]["config"]["backend"] == "sqlite"
- assert plugins["queuefs"]["config"]["db_path"] == str(Path("/tmp/new-queue.db").resolve())
-
-
-def test_generate_plugin_config_supports_legacy_queue_db_path():
- config = AGFSConfig(
- path="/tmp/ov-test",
- backend="local",
- queue_db_path="/tmp/legacy-queue.db",
- )
-
- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
-
- assert plugins["queuefs"]["config"]["backend"] == "sqlite"
- assert plugins["queuefs"]["config"]["db_path"] == str(Path("/tmp/legacy-queue.db").resolve())
+ with pytest.raises(ValueError, match="redirects requires backups"):
+ _generate_plugin_config(config, tmp_path)
-def test_generate_plugin_config_uses_workspace_default_queue_db_path():
+def test_generate_plugin_config_passes_multiwrite_encryption_flag(tmp_path):
+ """Multi-write mount config must reflect the real server encryption state."""
config = AGFSConfig(
- path="/tmp/ov-test",
+ path=str(tmp_path),
backend="local",
+ backups={
+ "items": [
+ {
+ "name": "mem-backup",
+ "backend": "memory",
+ }
+ ]
+ },
)
- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
+ plugins = _generate_plugin_config(config, tmp_path, server_encryption_enabled=True)
- assert plugins["queuefs"]["config"]["backend"] == "sqlite"
- assert plugins["queuefs"]["config"]["db_path"] == "/tmp/ov-test/_system/queue/queue.db"
+ mount_config = plugins["localfs"]["config"]
+ assert mount_config["server_encryption_enabled"] is True
+ assert mount_config["primary_encryption_enabled"] is True
-def test_generate_plugin_config_forwards_queuefs_runtime_options():
+def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path):
+ """Plugin config generation should normalize backup params while preserving top-level multi-write fields."""
+ explicit_backup_dir = tmp_path / "backup-local-no-mkdir"
config = AGFSConfig(
- path="/tmp/ov-test",
+ path=str(tmp_path),
backend="local",
- queuefs={
- "backend": "sqlite3",
- "recover_stale_sec": 17,
- "busy_timeout_ms": 1234,
+ backups={
+ "retry_interval_ms": 1234,
+ "retry_backoff_base_ms": 55,
+ "items": [
+ {
+ "name": "local-explicit",
+ "backend": "local",
+ "local": {"local_dir": str(explicit_backup_dir)},
+ },
+ {
+ "name": "local-default",
+ "backend": "local",
+ },
+ {
+ "name": "s3-backup",
+ "backend": "s3",
+ "s3": {
+ "bucket": "backup-bucket",
+ "region": "cn-beijing",
+ "access_key": "test-access-key",
+ "secret_key": "test-secret-key",
+ "endpoint": "https://tos.example.com",
+ "prefix": "backup-prefix",
+ "use_ssl": False,
+ "use_path_style": False,
+ "normalize_encoding_chars": "#?",
+ },
+ },
+ ],
},
)
- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
+ plugins = _generate_plugin_config(config, tmp_path)
- assert plugins["queuefs"]["config"]["backend"] == "sqlite3"
- assert plugins["queuefs"]["config"]["recover_stale_sec"] == 17
- assert plugins["queuefs"]["config"]["busy_timeout_ms"] == 1234
+ mount_backups = plugins["localfs"]["config"]["backups"]
+ assert mount_backups["retry_interval_ms"] == 1234
+ assert mount_backups["retry_backoff_base_ms"] == 55
+ explicit_local, default_local, s3_backup = mount_backups["items"]
+ assert explicit_local["backend"] == "localfs"
+ assert explicit_local["params"]["local_dir"] == str(explicit_backup_dir)
+ assert not explicit_backup_dir.exists()
-def test_generate_plugin_config_ignores_db_paths_in_memory_mode():
- config = AGFSConfig(
- path="/tmp/ov-test",
- backend="local",
- queue_db_path="/tmp/legacy-queue.db",
- queuefs={"backend": "memory", "db_path": "/tmp/new-queue.db"},
+ assert default_local["backend"] == "localfs"
+ assert default_local["params"]["local_dir"] == str(
+ tmp_path / "viking" / "_backups" / "local-default"
)
- plugins = _generate_plugin_config(config, Path("/tmp/ov-test"))
-
- assert plugins["queuefs"]["config"]["backend"] == "memory"
- assert "db_path" not in plugins["queuefs"]["config"]
+ assert s3_backup["backend"] == "s3fs"
+ assert s3_backup["params"] == {
+ "bucket": "backup-bucket",
+ "region": "cn-beijing",
+ "access_key_id": "test-access-key",
+ "secret_access_key": "test-secret-key",
+ "endpoint": "https://tos.example.com",
+ "prefix": "backup-prefix",
+ "disable_ssl": True,
+ "use_path_style": False,
+ "directory_marker_mode": None,
+ "disable_batch_delete": False,
+ "normalize_encoding_chars": "#?",
+ }
class _FakeMountClient:
@@ -251,6 +378,11 @@ class _FakeMountClient:
return None
+class _FailingMountClient(_FakeMountClient):
+ def mount(self, plugin_name, mount_path, config):
+ raise RuntimeError(f"mount failed: {plugin_name}:{mount_path}")
+
+
class _FakeBindingClient:
def __init__(self, *, config):
self.config = config
@@ -298,6 +430,15 @@ def test_mount_agfs_backend_creates_queue_sqlite_dirs_for_sqlite_backend(tmp_pat
assert queuefs_mount[2]["db_path"] == str(queue_db_path.resolve())
+def test_mount_agfs_backend_raises_mount_error(tmp_path):
+ """Mount failures must fail fast instead of being delayed to later filesystem calls."""
+ config = AGFSConfig(path=str(tmp_path), backend="local")
+ client = _FailingMountClient()
+
+ with pytest.raises(RuntimeError, match="mount failed"):
+ mount_agfs_backend(client, config)
+
+
def test_ragfs_binding_config_builds_single_binding_dict_for_local_backend(tmp_path):
agfs_config = AGFSConfig(path=str(tmp_path), backend="local")
diff --git a/tests/server/test_api_key_manager.py b/tests/server/test_api_key_manager.py
index f8820aa6a..982c4fc57 100644
--- a/tests/server/test_api_key_manager.py
+++ b/tests/server/test_api_key_manager.py
@@ -8,6 +8,7 @@ import uuid
import pytest
import pytest_asyncio
+from openviking.pyagfs.exceptions import AGFSNotFoundError
from openviking.server.api_keys import APIKeyManager
from openviking.server.identity import Role
from openviking.service.core import OpenVikingService
@@ -91,6 +92,30 @@ async def test_create_duplicate_account_raises(manager: APIKeyManager):
await manager.create_account(acct, "bob")
+async def test_create_account_rolls_back_when_user_persistence_fails(
+ manager: APIKeyManager, monkeypatch: pytest.MonkeyPatch
+):
+ """create_account should not leave partial in-memory state after a write failure."""
+ acct = _uid()
+ original_save_users_json = manager._legacy._save_users_json
+
+ async def _fail_save_users_json(account_id: str) -> None:
+ if account_id == acct:
+ raise AGFSNotFoundError(account_id)
+ await original_save_users_json(account_id)
+
+ monkeypatch.setattr(manager._legacy, "_save_users_json", _fail_save_users_json)
+
+ with pytest.raises(AGFSNotFoundError):
+ await manager.create_account(acct, "alice")
+
+ assert acct not in manager._legacy._accounts
+
+ monkeypatch.setattr(manager._legacy, "_save_users_json", original_save_users_json)
+ retry_key = await manager.create_account(acct, "alice")
+ assert manager.resolve(retry_key).account_id == acct
+
+
async def test_delete_account(manager: APIKeyManager):
"""Deleting account should invalidate all its user keys."""
acct = _uid()
diff --git a/tests/server/test_api_webdav.py b/tests/server/test_api_webdav.py
index b44b03638..8a10a822d 100644
--- a/tests/server/test_api_webdav.py
+++ b/tests/server/test_api_webdav.py
@@ -7,6 +7,9 @@ import xml.etree.ElementTree as ET
import httpx
+from openviking.server.routers.webdav import _ensure_exposed_path, _exposed_child_entries
+from openviking_cli.exceptions import NotFoundError
+
def _dav_display_names(xml_bytes: bytes) -> list[str]:
root = ET.fromstring(xml_bytes)
@@ -52,6 +55,9 @@ async def test_webdav_propfind_hides_reserved_files_but_keeps_user_dotdirs(clien
assert ".abstract.md" not in names
assert ".overview.md" not in names
assert ".relations.json" not in names
+ assert ".path.ovlock" not in names
+ assert ".sync_log.json" not in names
+ assert ".redirect.json" not in names
async def test_webdav_rejects_direct_access_to_reserved_semantic_files(client_with_resource):
@@ -62,6 +68,36 @@ async def test_webdav_rejects_direct_access_to_reserved_semantic_files(client_wi
assert resp.status_code == 404
+async def test_webdav_rejects_direct_access_to_multiwrite_internal_files(client_with_resource):
+ client, uri = client_with_resource
+ webdav_path = _webdav_path_from_uri(uri)
+
+ for hidden_name in (".path.ovlock", ".sync_log.json", ".redirect.json"):
+ resp = await client.get(f"/webdav/resources/{webdav_path}/{hidden_name}")
+ assert resp.status_code == 404
+
+
+def test_webdav_unit_rejects_direct_access_to_multiwrite_internal_files():
+ for hidden_name in (".path.ovlock", ".sync_log.json", ".redirect.json"):
+ try:
+ _ensure_exposed_path(f"workspace/{hidden_name}")
+ except NotFoundError:
+ continue
+ raise AssertionError(f"{hidden_name} should be rejected")
+
+
+def test_webdav_unit_filters_multiwrite_internal_files_from_children():
+ entries = [
+ {"name": ".obsidian"},
+ {"name": ".path.ovlock"},
+ {"name": ".sync_log.json"},
+ {"name": ".redirect.json"},
+ {"name": "notes.md"},
+ ]
+ filtered = _exposed_child_entries(entries)
+ assert [entry["name"] for entry in filtered] == [".obsidian", "notes.md"]
+
+
async def test_webdav_put_create_get_and_replace_text_file(client: httpx.AsyncClient):
mkcol_resp = await client.request("MKCOL", "/webdav/resources/scratch")
assert mkcol_resp.status_code == 201
diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py
index b54556329..b5c2a866e 100644
--- a/tests/server/test_auth.py
+++ b/tests/server/test_auth.py
@@ -337,6 +337,35 @@ async def test_auth_on_multiple_endpoints(auth_client: httpx.AsyncClient):
assert tenant_resp.status_code == 403
assert tenant_resp.json()["error"]["code"] == "PERMISSION_DENIED"
+
+async def test_admin_sync_route_accepts_root_key(auth_client: httpx.AsyncClient, auth_service):
+ """ROOT keys should be allowed to call the system sync admin route."""
+ calls: list[str] = []
+
+ async def _fake_system_sync_status(uri: str, ctx):
+ calls.append(uri)
+ return {"path": uri, "entry_count": 1}
+
+ auth_service.fs.system_sync_status = _fake_system_sync_status
+
+ resp = await auth_client.get(
+ "/api/v1/system/sync/viking://resources",
+ headers={"X-API-Key": ROOT_KEY},
+ )
+ assert resp.status_code == 200
+ assert resp.json()["result"] == {"path": "viking://resources", "entry_count": 1}
+ assert calls == ["viking://resources"]
+
+
+async def test_admin_sync_route_rejects_user_key(auth_client: httpx.AsyncClient, user_key: str):
+ """Regular user keys must not access the system sync admin route."""
+ resp = await auth_client.get(
+ "/api/v1/system/sync/viking://resources",
+ headers={"X-API-Key": user_key},
+ )
+ assert resp.status_code == 403
+ assert resp.json()["error"]["code"] == "PERMISSION_DENIED"
+
tenant_resp = await auth_client.get(
"/api/v1/fs/ls?uri=viking://",
headers={
diff --git a/tests/server/test_server_health.py b/tests/server/test_server_health.py
index 763bee2a9..f7f1fac3d 100644
--- a/tests/server/test_server_health.py
+++ b/tests/server/test_server_health.py
@@ -5,11 +5,12 @@
import asyncio
import time
+from types import SimpleNamespace
import httpx
import pytest
-from openviking.server.app import _on_deferred_init_done, create_app
+from openviking.server.app import _initialize_runtime_state, _on_deferred_init_done, create_app
from openviking.server.config import ServerConfig
@@ -79,6 +80,84 @@ async def test_system_status(client: httpx.AsyncClient):
assert body["result"]["initialized"] is True
+async def test_backend_sync_status_endpoint(client: httpx.AsyncClient, service):
+ calls: list[str] = []
+
+ async def _fake_system_sync_status(uri: str, ctx):
+ calls.append(uri)
+ assert ctx is not None
+ return {"path": uri, "entry_count": 1}
+
+ service.fs.system_sync_status = _fake_system_sync_status
+
+ resp = await client.post(
+ "/api/v1/system/backend/sync-status",
+ json={"uri": "viking://resources"},
+ )
+ assert resp.status_code == 200
+ body = resp.json()
+ assert body["status"] == "ok"
+ assert body["result"] == {"path": "viking://resources", "entry_count": 1}
+ assert calls == ["viking://resources"]
+
+
+async def test_backend_sync_retry_endpoint(client: httpx.AsyncClient, service):
+ calls: list[str] = []
+
+ async def _fake_system_sync_retry(uri: str, ctx):
+ calls.append(uri)
+ assert ctx is not None
+ return {"path": uri, "retried": 2, "failed": 0}
+
+ service.fs.system_sync_retry = _fake_system_sync_retry
+
+ resp = await client.post(
+ "/api/v1/system/backend/sync-retry",
+ json={"uri": "viking://resources"},
+ )
+ assert resp.status_code == 200
+ body = resp.json()
+ assert body["status"] == "ok"
+ assert body["result"] == {"path": "viking://resources", "retried": 2, "failed": 0}
+ assert calls == ["viking://resources"]
+
+
+async def test_admin_sync_status_route(client: httpx.AsyncClient, service):
+ calls: list[str] = []
+
+ async def _fake_system_sync_status(uri: str, ctx):
+ calls.append(uri)
+ assert ctx is not None
+ return {"path": uri, "entry_count": 3}
+
+ service.fs.system_sync_status = _fake_system_sync_status
+
+ resp = await client.get("/api/v1/system/sync/viking://resources")
+ assert resp.status_code == 200
+ body = resp.json()
+ assert body["status"] == "ok"
+ assert body["result"] == {"path": "viking://resources", "entry_count": 3}
+ assert calls == ["viking://resources"]
+
+
+async def test_admin_sync_retry_route(client: httpx.AsyncClient, service):
+ calls: list[str] = []
+
+ async def _fake_system_sync_retry(uri: str, ctx):
+ calls.append(uri)
+ assert ctx is not None
+ return {"path": uri, "retried": 4, "failed": 1}
+
+ service.fs.system_sync_retry = _fake_system_sync_retry
+
+ resp = await client.post("/api/v1/system/sync/viking://resources/retry")
+ assert resp.status_code == 200
+ body = resp.json()
+ assert body["status"] == "ok"
+ assert body["result"] == {"path": "viking://resources", "retried": 4, "failed": 1}
+ assert calls == ["viking://resources"]
+
+
async def test_process_time_header(client: httpx.AsyncClient):
resp = await client.get("/health")
assert "x-process-time" in resp.headers
@@ -156,9 +235,14 @@ async def test_ready_returns_200_after_initialized(monkeypatch):
"""Ready returns 200 when service is fully initialized and subsystems are healthy."""
class MockVikingFS:
+ """Mock VikingFS for readiness checks."""
+
async def ls(self, path, ctx=None):
return []
+ async def system_sync_status(self, uri, ctx=None):
+ return {"path": uri, "entry_count": 0}
+
def _get_vector_store(self):
class MockVectorStore:
async def health_check(self):
@@ -184,6 +268,9 @@ async def test_ready_returns_200_after_initialized(monkeypatch):
assert resp.status_code == 200
body = resp.json()
assert body["status"] == "ready"
+ assert body["checks"]["agfs"]["status"] == "ok"
+ assert body["checks"]["agfs"]["checks"]["filesystem"] == "ok"
+ assert body["checks"]["agfs"]["checks"]["multiwrite_sync"] == "ok"
async def test_slow_init_does_not_block_health(monkeypatch):
@@ -206,3 +293,37 @@ async def test_slow_init_does_not_block_health(monkeypatch):
assert resp.status_code == 200
# Health responds instantly since it's stateless (no service dependency)
assert elapsed < 0.5
+
+
+async def test_initialize_runtime_state_loads_api_key_manager(monkeypatch):
+ """API key auth must finish manager loading before the app is considered ready."""
+
+ class MockService:
+ def __init__(self):
+ self._initialized = False
+ self.viking_fs = object()
+
+ async def initialize(self):
+ self._initialized = True
+
+ class FakeAPIKeyManager:
+ def __init__(self, root_key, viking_fs, api_key_hashing_enabled):
+ self.root_key = root_key
+ self.viking_fs = viking_fs
+ self.api_key_hashing_enabled = api_key_hashing_enabled
+ self.loaded = False
+
+ async def load(self):
+ self.loaded = True
+
+ monkeypatch.setattr("openviking.server.app.APIKeyManager", FakeAPIKeyManager)
+
+ app = SimpleNamespace(state=SimpleNamespace(api_key_manager=None))
+ service = MockService()
+ config = ServerConfig(root_api_key="root-key-for-test")
+
+ await _initialize_runtime_state(app, service, config)
+
+ assert service._initialized is True
+ assert app.state.api_key_manager is not None
+ assert app.state.api_key_manager.loaded is True
diff --git a/tests/storage/test_viking_fs_tree.py b/tests/storage/test_viking_fs_tree.py
index 11cac777d..ce8b86051 100644
--- a/tests/storage/test_viking_fs_tree.py
+++ b/tests/storage/test_viking_fs_tree.py
@@ -147,10 +147,12 @@ def test_is_name_visible_at_account_root(fs, name, parent_path, expected):
("_system", "/local/test_account/resources", False),
("tasks", "/local/test_account/resources/bar", False),
(".path.ovlock", "/local/test_account/resources", False),
+ (".sync_log.json", "/local/test_account/resources", False),
+ (".redirect.json", "/local/test_account/resources", False),
],
)
def test_is_name_visible_at_non_root(fs, name, parent_path, expected):
- """PY-FLT-004: Non-root _INTERNAL_NAMES blacklist."""
+ """PY-FLT-004: Non-root internal-name blacklist."""
assert fs._is_name_visible_at_path(name, parent_path) == expected
@@ -201,6 +203,18 @@ def test_is_tree_entry_visible_path_ovlock_filtered(monkeypatch, fs):
assert fs._is_tree_entry_visible(entry, "/local/test_account", _default_ctx()) is False
+def test_is_tree_entry_visible_multiwrite_meta_filtered(monkeypatch, fs):
+ """PY-FLT-012: multi-write metadata files are filtered."""
+ patch_visibility(monkeypatch, fs, is_accessible=True)
+ for hidden_name in (".sync_log.json", ".redirect.json"):
+ entry = make_entry(
+ f"/local/test_account/resources/{hidden_name}",
+ hidden_name,
+ is_dir=False,
+ )
+ assert fs._is_tree_entry_visible(entry, "/local/test_account", _default_ctx()) is False
+
+
def test_is_tree_entry_visible_default_ctx(monkeypatch, fs):
"""PY-FLT-010: ctx=None uses default context."""
patch_visibility(monkeypatch, fs, is_accessible=True)