Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(storage): implement multi-write v2 metadata partitioning (#5741)

baojun-zhangff790205a58a
diff --git a/Cargo.lock b/Cargo.lock
index 8b2079660..626c9324f 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4098,6 +4098,7 @@ dependencies = [
"url",
"uuid",
"windows-sys 0.61.2",
+ "xxhash-rust",
]
[[package]]
@@ -6510,6 +6511,12 @@ version = "0.13.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
+[[package]]
+name = "xxhash-rust"
+version = "0.8.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7"
+
[[package]]
name = "xz2"
version = "0.1.7"
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs
index 407823453..2484267ba 100644
--- a/crates/ov_cli/src/client.rs
+++ b/crates/ov_cli/src/client.rs
@@ -510,20 +510,6 @@ impl HttpClient {
self.post("/api/v1/system/consistency", &body).await
}
- pub async fn backend_sync_status(&self, uri: &str) -> Result<serde_json::Value> {
- let body = serde_json::json!({
- "uri": uri,
- });
- self.post("/api/v1/system/backend/sync-status", &body).await
- }
-
- pub async fn backend_sync_retry(&self, uri: &str) -> Result<serde_json::Value> {
- let body = serde_json::json!({
- "uri": uri,
- });
- self.post("/api/v1/system/backend/sync-retry", &body).await
- }
-
/// Download file as raw bytes
pub async fn get_bytes(&self, uri: &str) -> Result<Vec<u8>> {
let url = format!("{}/api/v1/content/download", self.base.base_url);
diff --git a/crates/ov_cli/src/commands/system.rs b/crates/ov_cli/src/commands/system.rs
index 5ad24fff8..037cfae04 100644
--- a/crates/ov_cli/src/commands/system.rs
+++ b/crates/ov_cli/src/commands/system.rs
@@ -81,28 +81,6 @@ pub async fn consistency(
Ok(())
}
-pub async fn backend_sync_status(
- client: &HttpClient,
- uri: &str,
- output_format: OutputFormat,
- compact: bool,
-) -> Result<()> {
- let response: serde_json::Value = client.backend_sync_status(uri).await?;
- output_success(&response, output_format, compact);
- Ok(())
-}
-
-pub async fn backend_sync_retry(
- client: &HttpClient,
- uri: &str,
- output_format: OutputFormat,
- compact: bool,
-) -> Result<()> {
- let response: serde_json::Value = client.backend_sync_retry(uri).await?;
- output_success(&response, output_format, compact);
- Ok(())
-}
-
fn output_consistency_table(response: &serde_json::Value, compact: bool) {
let summary = json!({
"ok": response.get("ok").and_then(|v| v.as_bool()).unwrap_or(false),
diff --git a/crates/ov_cli/src/error_ui.rs b/crates/ov_cli/src/error_ui.rs
index c94397d8f..45db15290 100644
--- a/crates/ov_cli/src/error_ui.rs
+++ b/crates/ov_cli/src/error_ui.rs
@@ -529,12 +529,6 @@ fn system_help_command(program: &str, tokens: &[&str]) -> String {
Some("init-key") => format!("{program} system crypto init-key --help"),
_ => format!("{program} system crypto --help"),
},
- Some("backend") => match tokens.get(3).copied() {
- Some("sync-status" | "sync-retry") => {
- format!("{program} system backend {} --help", tokens[3])
- }
- _ => format!("{program} system backend --help"),
- },
Some("wait" | "status" | "health" | "consistency") => {
format!("{program} system {} --help", tokens[2])
}
diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rs
index 8c927503c..8bf132c1f 100644
--- a/crates/ov_cli/src/handlers.rs
+++ b/crates/ov_cli/src/handlers.rs
@@ -415,7 +415,7 @@ pub async fn handle_restore(
.await
}
-use crate::{SystemBackendCommands, SystemCommands};
+use crate::SystemCommands;
pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> {
let client = ctx.get_client();
@@ -440,16 +440,6 @@ pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> {
commands::system::consistency(&client, &uri, ctx.output_format, ctx.compact).await
}
SystemCommands::Crypto { action } => commands::crypto::handle_crypto(action).await,
- SystemCommands::Backend { action } => match action {
- SystemBackendCommands::SyncStatus { uri } => {
- commands::system::backend_sync_status(&client, &uri, ctx.output_format, ctx.compact)
- .await
- }
- SystemBackendCommands::SyncRetry { uri } => {
- commands::system::backend_sync_retry(&client, &uri, ctx.output_format, ctx.compact)
- .await
- }
- },
}
}
diff --git a/crates/ov_cli/src/help_ui.rs b/crates/ov_cli/src/help_ui.rs
index d386936be..17f4015f1 100644
--- a/crates/ov_cli/src/help_ui.rs
+++ b/crates/ov_cli/src/help_ui.rs
@@ -1340,7 +1340,7 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[
},
CommandHelpSpec {
path: &["system"],
- purpose: "Run server utility, health, consistency, backend sync, and crypto commands.",
+ purpose: "Run server utility, health, consistency, and crypto commands.",
examples: &[
HelpItem {
label: "ov system health",
@@ -1350,34 +1350,12 @@ const COMMAND_HELP_SPECS: &[CommandHelpSpec] = &[
label: "ov system consistency viking://projects/acme",
description: "Check filesystem/vector consistency.",
},
- HelpItem {
- label: "ov system backend sync-status viking://projects/acme",
- description: "Inspect multi-write backend sync lag.",
- },
],
next_steps: &[HelpItem {
label: "ov status",
description: "Use the standard status view.",
}],
},
- CommandHelpSpec {
- path: &["system", "backend"],
- purpose: "Inspect and repair multi-write backend sync state.",
- examples: &[
- HelpItem {
- label: "ov system backend sync-status viking://resources",
- description: "Show pending and acknowledged backend sync state.",
- },
- HelpItem {
- label: "ov system backend sync-retry viking://resources",
- description: "Retry lagging backend sync targets.",
- },
- ],
- next_steps: &[HelpItem {
- label: "ov system backend sync-status <uri>",
- description: "Inspect the subtree again after retry.",
- }],
- },
CommandHelpSpec {
path: &["reindex"],
purpose: "Reindex semantic/vector artifacts for a URI.",
@@ -2992,7 +2970,6 @@ mod tests {
fn curated_help_lists_timeout_for_waiting_commands() {
for args in [
["ov", "add-resource", "--help"],
- ["ov", "add-skill", "--help"],
["ov", "rm", "--help"],
["ov", "write", "--help"],
] {
@@ -3214,7 +3191,7 @@ mod tests {
("admin", "Manage accounts, users, roles, and API keys."),
(
"system",
- "Run server utility, health, consistency, backend sync, and crypto commands.",
+ "Run server utility, health, consistency, and crypto commands.",
),
("observer", "Inspect specific OpenViking server subsystems."),
] {
@@ -3281,20 +3258,6 @@ mod tests {
);
}
- #[test]
- fn supported_nested_curated_help_still_renders() {
- let rendered = strip_ansi(
- &render_command_help_request(&os_args(&["ov", "system", "backend", "--help"]))
- .expect("system backend help should render curated nested help"),
- );
-
- assert!(rendered.contains("ov system backend"));
- assert!(rendered.contains("sync-status"));
- assert!(rendered.contains("sync-retry"));
- assert!(!rendered.contains("help Print this message"));
- assert!(rendered.contains("--sudo"));
- }
-
#[test]
fn prefixed_help_with_positional_value_renders_curated_command_help() {
let rendered = strip_ansi(
diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs
index 30ff8c4d6..00630c3ed 100644
--- a/crates/ov_cli/src/main.rs
+++ b/crates/ov_cli/src/main.rs
@@ -1589,29 +1589,6 @@ enum SystemCommands {
#[command(subcommand)]
action: commands::crypto::CryptoCommands,
},
- /// Backend sync inspection and repair commands
- Backend {
- #[command(subcommand)]
- action: SystemBackendCommands,
- },
-}
-
-#[derive(Subcommand)]
-enum SystemBackendCommands {
- /// Show multi-write backend sync status for a URI subtree
- #[command(name = "sync-status")]
- SyncStatus {
- /// Viking URI to inspect
- #[arg(value_name = "uri")]
- uri: String,
- },
- /// Retry pending multi-write backend sync work for a URI subtree
- #[command(name = "sync-retry")]
- SyncRetry {
- /// Viking URI to repair
- #[arg(value_name = "uri")]
- uri: String,
- },
}
#[derive(Subcommand)]
@@ -2544,14 +2521,6 @@ fn plain_help_misuse(args: &[OsString]) -> Option<PlainHelpMisuse> {
{
path.push(watch_subcommand.to_string());
}
- if command == "system"
- && path.get(1).is_some_and(|token| token == "backend")
- && let Some(backend_subcommand) =
- tokens.get(2).map(|token| canonical_plain_help_token(token))
- && backend_subcommand != "help"
- {
- path.push(backend_subcommand.to_string());
- }
return Some(PlainHelpMisuse {
help_command: prefixed_help_command(&path),
});
@@ -2763,10 +2732,6 @@ fn known_task_command_requires_config(tokens: &[String]) -> bool {
fn known_system_command_requires_config(tokens: &[String]) -> bool {
match tokens.get(1).map(String::as_str) {
Some("wait" | "status" | "health" | "consistency") => true,
- Some("backend") => matches!(
- tokens.get(2).map(String::as_str),
- None | Some("sync-status" | "sync-retry")
- ),
Some("crypto") => matches!(tokens.get(2).map(String::as_str), None | Some("init-key")),
_ => false,
}
@@ -4032,7 +3997,7 @@ mod tests {
};
use crate::config::{Config, DEFAULT_CUSTOM_URL};
use crate::output::OutputFormat;
- use crate::{AdminCommands, SystemBackendCommands, SystemCommands, handlers};
+ use crate::{AdminCommands, handlers};
use clap::{CommandFactory, Parser};
use std::ffi::OsString;
@@ -4382,22 +4347,6 @@ mod tests {
}
}
- #[test]
- fn cli_parses_system_backend_sync_status() {
- let cli = Cli::try_parse_from(["ov", "system", "backend", "sync-status", "viking://a"])
- .expect("system backend sync-status should parse");
-
- match cli.command {
- Commands::System { action } => match action {
- SystemCommands::Backend { action } => match action {
- SystemBackendCommands::SyncStatus { uri } => assert_eq!(uri, "viking://a"),
- _ => panic!("expected backend sync-status command"),
- },
- _ => panic!("expected system backend command"),
- },
- _ => panic!("expected system command"),
- }
- }
#[test]
fn server_commands_require_existing_cli_config() {
@@ -4623,7 +4572,6 @@ mod tests {
&["ov", "config", "validate"],
&["ov", "config", "show"],
&["ov", "system", "consistency"],
- &["ov", "system", "backend", "sync-status"],
&["ov", "admin", "list-accounts"],
] {
assert!(
@@ -4708,9 +4656,6 @@ mod tests {
&["ov", "system", "status"],
&["ov", "system", "health"],
&["ov", "system", "consistency"],
- &["ov", "system", "backend"],
- &["ov", "system", "backend", "sync-status"],
- &["ov", "system", "backend", "sync-retry"],
&["ov", "system", "crypto"],
&["ov", "system", "crypto", "init-key"],
&["ov", "session", "new"],
@@ -5628,10 +5573,6 @@ mod tests {
vec!["ov", "system", "consistency", "help"],
"ov system consistency --help",
),
- (
- vec!["ov", "system", "backend", "sync-status", "help"],
- "ov system backend sync-status --help",
- ),
] {
let misuse = plain_help_misuse(&os_args(&args))
.unwrap_or_else(|| panic!("{args:?} should be plain-help misuse"));
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs
index bab219ebc..b09adc8c8 100644
--- a/crates/ragfs-python/src/lib.rs
+++ b/crates/ragfs-python/src/lib.rs
@@ -956,6 +956,7 @@ fn to_py_err(e: ragfs::core::Error) -> PyErr {
ragfs::core::Error::IsADirectory(_) => new_py_err("AGFSIsADirectoryError", msg),
ragfs::core::Error::DirectoryNotEmpty(_) => new_py_err("AGFSDirectoryNotEmptyError", msg),
ragfs::core::Error::InvalidOperation(_) => new_py_err("AGFSInvalidOperationError", msg),
+ ragfs::core::Error::NotSupported(_) => new_py_err("AGFSNotSupportedError", msg),
ragfs::core::Error::Io(_) => new_py_err("AGFSIoError", msg),
ragfs::core::Error::Plugin(_) => {
// Check if the plugin error message contains known patterns
@@ -1628,7 +1629,11 @@ impl RAGFSBindingClient {
cache_config.stack_config(),
))
.map_err(|error| {
- PyRuntimeError::new_err(format!("Failed to build RAGFS stack: {error}"))
+ if matches!(error, ragfs::core::Error::NotSupported(_)) {
+ to_py_err(error)
+ } else {
+ PyRuntimeError::new_err(format!("Failed to build RAGFS stack: {error}"))
+ }
})?;
// Build the git service from inline config when present; otherwise fall
@@ -1652,7 +1657,7 @@ impl RAGFSBindingClient {
})
}
- /// Stop mounted background services, then close the shared CacheRuntime.
+ /// Stop mounts requiring explicit cleanup, then close the shared CacheRuntime.
fn close(&self, py: Python<'_>) -> PyResult<()> {
let mountable = Arc::clone(&self.mountable);
let cache_runtime = self.cache_runtime.clone();
@@ -2169,7 +2174,6 @@ impl RAGFSBindingClient {
/// - "backups": nested dict matching BackendsConfig schema
/// - "server_encryption_enabled": bool
/// - "primary_encryption_enabled": bool
- /// - "primary_redirects": list of redirect policy dicts
#[pyo3(signature = (fstype, path, config=None))]
fn mount(
&self,
@@ -2433,58 +2437,6 @@ impl RAGFSBindingClient {
})
}
- /// Query multi-write sync status under a file or directory path.
- ///
- /// Args:
- /// path: Absolute AGFS path rooted at the mounted namespace
- /// ctx: Optional FsContext dict (e.g. {"account_id": ...})
- ///
- /// Returns:
- /// A JSON-like dict describing pending and acknowledged backup sync state.
- #[pyo3(signature = (path, ctx=None))]
- fn system_sync_status(
- &self,
- py: Python<'_>,
- path: String,
- ctx: Option<HashMap<String, String>>,
- ) -> PyResult<Py<PyAny>> {
- let fs_ctx = build_fs_context(ctx);
- let mountable = self.mountable.clone();
- let result = self
- .run_scoped(py, fs_ctx, move || async move {
- mountable.system_sync_status(&path).await
- })
- .map_err(to_py_err)?;
-
- Python::attach(|py| serde_json_to_py(py, &result))
- }
-
- /// Manually retry lagging multi-write backup sync operations under a path.
- ///
- /// Args:
- /// path: Absolute AGFS path rooted at the mounted namespace
- /// ctx: Optional FsContext dict (e.g. {"account_id": ...})
- ///
- /// Returns:
- /// A JSON-like dict summarizing retry results per target backend.
- #[pyo3(signature = (path, ctx=None))]
- fn system_sync_retry(
- &self,
- py: Python<'_>,
- path: String,
- ctx: Option<HashMap<String, String>>,
- ) -> PyResult<Py<PyAny>> {
- let fs_ctx = build_fs_context(ctx);
- let mountable = self.mountable.clone();
- let result = self
- .run_scoped(py, fs_ctx, move || async move {
- mountable.system_sync_retry(&path).await
- })
- .map_err(to_py_err)?;
-
- Python::attach(|py| serde_json_to_py(py, &result))
- }
-
/// Calculate file digest (not yet implemented in ragfs).
#[pyo3(signature = (path, algorithm="xxh3"))]
fn digest(&self, path: String, algorithm: &str) -> PyResult<HashMap<String, String>> {
diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.toml
index 787c1ffcf..b941490a7 100644
--- a/crates/ragfs/Cargo.toml
+++ b/crates/ragfs/Cargo.toml
@@ -18,11 +18,6 @@ path = "src/lib.rs"
name = "ragfs-shell"
path = "src/shell/main.rs"
-[[test]]
-name = "cache_wrapper"
-path = "tests/cache_wrapper.rs"
-required-features = ["cache", "test-utils"]
-
[[test]]
name = "cache_runtime"
path = "tests/cache_runtime.rs"
@@ -67,6 +62,7 @@ filelocks = "0.1.0"
# Data structures
radix_trie = "0.2"
+xxhash-rust = { version = "0.8", features = ["xxh3"] }
# Error handling
anyhow = "1.0"
diff --git a/crates/ragfs/src/core/builder.rs b/crates/ragfs/src/core/builder.rs
index 61c47051e..4c5f4b67c 100644
--- a/crates/ragfs/src/core/builder.rs
+++ b/crates/ragfs/src/core/builder.rs
@@ -200,6 +200,8 @@ pub async fn build_configured_stack(
CacheNamespace::new(&cache.cachefs.namespace),
cache.cachefs.policy,
))
+ } else if let Some(runtime) = runtime.as_ref() {
+ Arc::new(MountableFS::with_shared_cache_runtime(runtime.clone()))
} else {
Arc::new(MountableFS::new())
};
diff --git a/crates/ragfs/src/core/errors.rs b/crates/ragfs/src/core/errors.rs
index 9ee84e5c2..43f10ac2c 100644
--- a/crates/ragfs/src/core/errors.rs
+++ b/crates/ragfs/src/core/errors.rs
@@ -56,6 +56,10 @@ pub enum Error {
#[error("invalid operation: {0}")]
InvalidOperation(String),
+ /// Operation is intentionally unsupported by the current protocol.
+ #[error("not supported: {0}")]
+ NotSupported(String),
+
/// I/O error
#[error("I/O error: {0}")]
Io(#[from] io::Error),
@@ -188,6 +192,7 @@ impl Error {
Self::IsADirectory(_) => "is_a_directory",
Self::DirectoryNotEmpty(_) => "directory_not_empty",
Self::InvalidOperation(_) => "invalid_operation",
+ Self::NotSupported(_) => "not_supported",
Self::Io(_) => "io",
Self::Plugin(_) => "plugin",
Self::Config(_) => "config",
@@ -208,6 +213,11 @@ impl Error {
pub fn invalid_operation(msg: impl Into<String>) -> Self {
Self::InvalidOperation(msg.into())
}
+
+ /// Create a NotSupported error.
+ pub fn not_supported(msg: impl Into<String>) -> Self {
+ Self::NotSupported(msg.into())
+ }
}
#[cfg(test)]
diff --git a/crates/ragfs/src/core/internal_names.rs b/crates/ragfs/src/core/internal_names.rs
index dadfc36cd..f73ddd27d 100644
--- a/crates/ragfs/src/core/internal_names.rs
+++ b/crates/ragfs/src/core/internal_names.rs
@@ -1,8 +1,7 @@
//! Static built-in internal names shared across the RAGFS stack.
//!
-//! Every module that references `.path.ovlock`, `.exact.ovlock.*`, `.redirect.json`,
-//! or `.sync_log.json` must import these constants from this single source of truth.
-//! There is no dynamic extension mechanism.
+//! Every module that references `.path.ovlock` or `.exact.ovlock.*` must import
+//! these constants from this single source of truth.
/// Path-lock marker for directory-level locks.
pub const PATH_LOCK_FILE: &str = ".path.ovlock";
@@ -10,21 +9,49 @@ pub const PATH_LOCK_FILE: &str = ".path.ovlock";
/// Prefix for exact (sidecar) lock files: `.exact.ovlock.<safe_name>.<sha1-prefix>`.
pub const EXACT_LOCK_FILE_PREFIX: &str = ".exact.ovlock.";
-/// Multi-write redirect metadata file.
-pub const REDIRECT_FILE: &str = ".redirect.json";
-
-/// Multi-write sync-log metadata file.
-pub const SYNC_LOG_FILE: &str = ".sync_log.json";
-
/// Returns `true` when `name` is a runtime path-lock file (`.path.ovlock` or `.exact.ovlock.*`).
pub fn is_hidden_runtime_lock_name(name: &str) -> bool {
// Reserve the whole prefix, including names not produced by the lock resolver.
name == PATH_LOCK_FILE || name.starts_with(EXACT_LOCK_FILE_PREFIX)
}
-/// Returns `true` when `name` is any hidden internal name (lock files, redirect, sync-log).
+/// Returns `true` when `name` is a hidden runtime lock name.
pub fn is_hidden_internal_name(name: &str) -> bool {
- is_hidden_runtime_lock_name(name) || name == REDIRECT_FILE || name == SYNC_LOG_FILE
+ is_hidden_runtime_lock_name(name)
+}
+
+/// Returns `true` when a logical path belongs to V2 multi-write internals.
+pub fn is_multiwrite_internal_path(path: &str) -> bool {
+ let Some(relative) = path.strip_prefix('/') else {
+ return false;
+ };
+ let components = relative.split('/').collect::<Vec<_>>();
+ if components
+ .last()
+ .is_some_and(|name| is_hidden_internal_name(name))
+ {
+ return true;
+ }
+ if matches!(
+ components.as_slice(),
+ ["local", "_system", ".multiwrite.json"]
+ ) {
+ return true;
+ }
+ if matches!(
+ components.as_slice(),
+ ["local", "_system", name] if name.starts_with(".multiwrite.catch-up.")
+ ) || matches!(components.as_slice(), ["local", "_system", "_system", ..])
+ {
+ return true;
+ }
+ matches!(
+ components.as_slice(),
+ ["local", account, "_system", "partitions", ..] if !account.is_empty()
+ ) || matches!(
+ components.as_slice(),
+ ["local", account, "temp", ..] if !account.is_empty()
+ )
}
#[cfg(test)]
diff --git a/crates/ragfs/src/core/mod.rs b/crates/ragfs/src/core/mod.rs
index fb75b3d52..9e4a9be9d 100644
--- a/crates/ragfs/src/core/mod.rs
+++ b/crates/ragfs/src/core/mod.rs
@@ -23,7 +23,7 @@ pub mod stats_wrapper;
pub mod types;
// Re-export commonly used types
-pub use crate::multibackend::{FsContextResolver, MetaStateStore};
+pub use crate::multibackend::FsContextResolver;
#[cfg(feature = "cache")]
pub use builder::{
build_configured_stack, CacheFsConfig, CacheRuntimeProviderConfig, CacheStackConfig,
@@ -37,13 +37,12 @@ pub use encryption_wrapper::EncryptionWrappedFS;
pub use errors::{Error, Result};
pub use filesystem::FileSystem;
pub use mountable::MountableFS;
-pub use multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS, SyncMode};
+pub use multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS};
pub use plugin::{HealthStatus, PluginRegistry, ServicePlugin};
pub use stats::{FilesystemStats, FsOperation, OperationStats, OperationTimer, StatsCollector};
pub use stats_wrapper::StatsWrappedFS;
pub use types::{
- BackendItemConfig, BackendRole, BackendSyncState, BackendsConfig, ConfigParameter, ConfigValue,
- EncryptionConfig, FileInfo, GlobEntry, GlobPage, GrepMatch, GrepOptions, GrepResult,
- ListSortBy, OperationItemConfig, PluginConfig, RedirectEntry, RedirectMeta, RedirectPolicy,
- SortOrder, SyncLogEntry, SyncLogMeta, SyncOp, SyncType, TreeEntry, WriteFlag,
+ BackendItemConfig, BackendRole, BackendsConfig, ConfigParameter, ConfigValue, EncryptionConfig,
+ FileInfo, GlobEntry, GlobPage, GrepMatch, GrepOptions, GrepResult, ListSortBy, PluginConfig,
+ SortOrder, TreeEntry, WriteFlag,
};
diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs
index 2b7f625ab..32d9c201c 100644
--- a/crates/ragfs/src/core/mountable.rs
+++ b/crates/ragfs/src/core/mountable.rs
@@ -6,7 +6,6 @@
use async_trait::async_trait;
use radix_trie::{Trie, TrieCommon};
-use serde_json::Value;
use std::collections::HashMap;
use std::sync::{Arc, OnceLock};
use std::time::Duration;
@@ -14,10 +13,8 @@ use tokio::sync::RwLock;
use tracing::warn;
use crate::lock::{AutoPathLockAction, PathLockKind, PathLockManager, PathLockRequest};
-use crate::metrics::{
- lock_metrics, merge_metrics, operation_metrics, RagfsMetric, RagfsMetricValue,
-};
-use crate::multibackend::factory::build_multi_write_fs;
+use crate::metrics::{lock_metrics, merge_metrics, operation_metrics, RagfsMetric};
+use crate::multibackend::factory::build_inactive_multi_write_fs;
use crate::multibackend::types::MultiBackendBuildContext;
use crate::plugins::QueueFileSystem;
use crate::shape::validate::ensure_backend_shape;
@@ -88,6 +85,10 @@ pub struct MountableFS {
/// Optional cache configuration shared by mounted filesystems.
#[cfg(feature = "cache")]
cache: Option<MountCacheConfig>,
+
+ /// Optional shared runtime, independent of mount-level CacheFS enablement.
+ #[cfg(feature = "cache")]
+ cache_runtime: Option<Arc<CacheRuntime>>,
}
#[cfg(feature = "cache")]
@@ -169,9 +170,6 @@ impl MountableFS {
if let Some(queuefs) = Self::as_queuefs_ref(mount_info.fs.as_ref()) {
queuefs.shutdown().await?;
}
- if let Some(multiwrite) = Self::as_multiwrite(&mount_info.fs) {
- multiwrite.shutdown().await?;
- }
Ok(())
}
@@ -220,22 +218,6 @@ impl MountableFS {
Self::encryption_handles_pathlock_ref(mount_info.fs.as_ref())
}
- /// Query multi-write sync status for a mounted path.
- pub async fn system_sync_status(&self, path: &str) -> Result<Value> {
- let (mount_info, rel_path) = self.find_mount(path).await?;
- let multiwrite = Self::as_multiwrite(&mount_info.fs)
- .ok_or_else(|| Error::invalid_operation("mounted filesystem is not multi-write"))?;
- multiwrite.system_sync_status(&rel_path).await
- }
-
- /// Manually retry pending multi-write sync operations under a mounted path.
- pub async fn system_sync_retry(&self, path: &str) -> Result<Value> {
- let (mount_info, rel_path) = self.find_mount(path).await?;
- let multiwrite = Self::as_multiwrite(&mount_info.fs)
- .ok_or_else(|| Error::invalid_operation("mounted filesystem is not multi-write"))?;
- multiwrite.system_sync_retry(&rel_path).await
- }
-
/// Create a new MountableFS
pub fn new() -> Self {
Self {
@@ -246,6 +228,22 @@ impl MountableFS {
pathlock_manager: OnceLock::new(),
#[cfg(feature = "cache")]
cache: None,
+ #[cfg(feature = "cache")]
+ cache_runtime: None,
+ }
+ }
+
+ /// Create a MountableFS that shares a runtime without enabling CacheFS.
+ #[cfg(feature = "cache")]
+ pub fn with_shared_cache_runtime(runtime: Arc<CacheRuntime>) -> Self {
+ Self {
+ mounts: Arc::new(RwLock::new(Trie::new())),
+ registry: Arc::new(RwLock::new(HashMap::new())),
+ encryption_root_key: RwLock::new(None),
+ encryption_provider_type: RwLock::new(None),
+ pathlock_manager: OnceLock::new(),
+ cache: None,
+ cache_runtime: Some(runtime),
}
}
@@ -263,10 +261,11 @@ impl MountableFS {
encryption_provider_type: RwLock::new(None),
pathlock_manager: OnceLock::new(),
cache: Some(MountCacheConfig {
- runtime,
+ runtime: runtime.clone(),
namespace,
policy,
}),
+ cache_runtime: Some(runtime),
}
}
@@ -448,9 +447,13 @@ impl MountableFS {
stats: stats_collector,
plugin_name: config.name.clone(),
};
-
+ let mounted_fs = mount_info.fs.clone();
let mut mounts = self.mounts.write().await;
mounts.insert(normalized_path, mount_info);
+ drop(mounts);
+ if let Some(multiwrite) = Self::as_multiwrite(&mounted_fs) {
+ multiwrite.activate();
+ }
Ok(())
}
@@ -471,7 +474,7 @@ impl MountableFS {
let pathlock_manager = self.pathlock_manager.get().cloned().ok_or_else(|| {
Error::config("pathlock manager must be initialized before multi-write mount")
})?;
- build_multi_write_fs(
+ build_inactive_multi_write_fs(
&self.registry,
config,
bc,
@@ -480,6 +483,8 @@ impl MountableFS {
enc_provider_type,
pathlock_manager,
backend_prefix: config.mount_path.clone(),
+ #[cfg(feature = "cache")]
+ cache_runtime: self.cache_runtime.clone(),
},
)
.await
@@ -617,28 +622,7 @@ impl MountableFS {
metrics.extend(crate::metrics::cache_metrics(cache.metrics().snapshot()));
}
if let Some(multiwrite) = Self::as_multiwrite(&mount.fs) {
- metrics.push(RagfsMetric {
- name: "ragfs_multiwrite_background_tasks".into(),
- labels: Default::default(),
- value: RagfsMetricValue::Gauge(multiwrite.background_task_count() as f64),
- });
- let routes = multiwrite.inner.read_route_metrics();
- for (route, key) in [
- ("primary", "primary_hits"),
- ("backup", "backup_hits"),
- ("redirect", "redirect_hits"),
- ("miss", "misses"),
- ] {
- let count = routes[key].as_u64().ok_or_else(|| {
- Error::internal(format!("invalid read-route counter '{key}'"))
- })?;
- metrics.push(RagfsMetric::counter(
- "ragfs_multiwrite_read_routes_total",
- &[("route", route)],
- count,
- 1.0,
- ));
- }
+ metrics.extend(multiwrite.metrics().await?);
}
}
if let Some(manager) = self.pathlock_manager.get() {
@@ -1250,9 +1234,7 @@ impl FileSystem for MountableFS {
mod tests {
use super::*;
use crate::core::BackendItemConfig;
- #[cfg(feature = "cache")]
use crate::core::ConfigValue;
- use crate::core::RedirectPolicy;
use crate::shape::SHAPE_MANIFEST_PATH;
use serde_json::Value;
use std::collections::HashMap;
@@ -1274,29 +1256,66 @@ mod tests {
mount_path: mount_path.to_string(),
params: HashMap::new(),
backups: Some(BackendsConfig {
- sync_type: "async".to_string(),
- write_ack_count: None,
- write_ack_timeout_ms: None,
- write_concurrency: None,
- retry_interval_ms: None,
- retry_backoff_base_ms: None,
- retry_max_retries_per_round: None,
- retry_quarantine_after_failures: None,
- read_probe_cache_ttl_ms: None,
+ namespace: "default".to_string(),
+ initial_partitions: 16,
+ checkpoint_interval_secs: 86_400,
+ provider: "filesystem".to_string(),
items: vec![BackendItemConfig {
name: "backup1".to_string(),
backend: backup_backend.to_string(),
params: Value::Null,
- timeout: None,
encryption: None,
- operations: None,
- excludes: None,
}],
}),
..PluginConfig::default()
}
}
+ /// Build a two-backup LocalFS configuration from three persistent directories.
+ fn local_multiwrite_test_config(
+ primary_dir: &std::path::Path,
+ backup_dirs: [(&str, &std::path::Path); 2],
+ ) -> PluginConfig {
+ let mut config = multiwrite_test_config("localfs", "localfs", "/local");
+ config.params.insert(
+ "local_dir".to_string(),
+ ConfigValue::String(primary_dir.to_string_lossy().into_owned()),
+ );
+ config.backups.as_mut().unwrap().initial_partitions = 1;
+ config.backups.as_mut().unwrap().items = backup_dirs
+ .into_iter()
+ .map(|(name, path)| BackendItemConfig {
+ name: name.to_string(),
+ backend: "localfs".to_string(),
+ params: serde_json::json!({
+ "local_dir": path.to_string_lossy(),
+ }),
+ encryption: None,
+ })
+ .collect();
+ config
+ }
+
+ /// Mount one LocalFS multi-write configuration with the shared lock provider.
+ async fn mount_local_multiwrite_for_test(
+ config: PluginConfig,
+ provider: Arc<dyn crate::lock::PathLockProvider>,
+ ) -> Arc<MountableFS> {
+ use crate::lock::{PathLockConfig, PathLockManager};
+ use crate::plugins::{memfs::MemFileSystem, LocalFSPlugin};
+
+ let mfs = Arc::new(MountableFS::new());
+ mfs.register_plugin(LocalFSPlugin::new()).await;
+ let manager = Arc::new(PathLockManager::new(
+ Arc::new(MemFileSystem::new()),
+ provider,
+ PathLockConfig::default(),
+ ));
+ mfs.set_pathlock_manager(manager).await;
+ mfs.mount(config).await.unwrap();
+ mfs
+ }
+
// Mock filesystem for testing
struct MockFS {
name: String,
@@ -1338,10 +1357,10 @@ mod tests {
}
async fn read(&self, path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {
- if path == SHAPE_MANIFEST_PATH
- || path.ends_with(".redirect.json")
- || path.ends_with(".sync_log.json")
- {
+ if path == "/_system/accounts.json" {
+ return Ok(br#"{"accounts":{}}"#.to_vec());
+ }
+ if path == SHAPE_MANIFEST_PATH || path == "/_system/.multiwrite.json" {
return Err(Error::not_found(path));
}
Ok(self.name.as_bytes().to_vec())
@@ -1373,6 +1392,9 @@ mod tests {
}
async fn stat(&self, path: &str) -> Result<FileInfo> {
+ if path == "/_system/.multiwrite.json" {
+ return Err(Error::not_found(path));
+ }
Ok(FileInfo::new_file(path.to_string(), 0, 0o644))
}
@@ -1561,33 +1583,6 @@ mod tests {
mfs
}
- /// Create a MountableFS backed by the real in-memory plugin.
- async fn mounted_memfs(mount_path: &str) -> MountableFS {
- use crate::plugins::MemFSPlugin;
-
- let mfs = MountableFS::new();
- mfs.register_plugin(MemFSPlugin).await;
- mfs.mount(test_config("memfs", mount_path)).await.unwrap();
- mfs
- }
-
- /// Create a cache-enabled MountableFS backed by the real in-memory plugin.
- #[cfg(feature = "cache")]
- async fn mounted_cached_memfs(namespace: &str, mount_path: &str) -> MountableFS {
- use crate::cache::{CacheNamespace, CachePolicy};
- use crate::cache_runtime::CacheRuntime;
- use crate::plugins::MemFSPlugin;
-
- let mfs = MountableFS::with_cache_runtime(
- CacheRuntime::memory(),
- CacheNamespace::new(namespace),
- CachePolicy::default(),
- );
- mfs.register_plugin(MemFSPlugin).await;
- mfs.mount(test_config("memfs", mount_path)).await.unwrap();
- mfs
- }
-
/// Create a MountableFS with two mounted mock plugins.
async fn mounted_two_mocks() -> MountableFS {
let mfs = MountableFS::new();
@@ -1733,6 +1728,32 @@ mod tests {
);
}
+ /// Verifies service shutdown leaves multi-write workers to process teardown.
+ #[tokio::test]
+ async fn shutdown_does_not_stop_multiwrite_runtime() {
+ use crate::lock::MemoryPathLockProvider;
+
+ let primary = tempfile::tempdir().unwrap();
+ let backup_a = tempfile::tempdir().unwrap();
+ let backup_b = tempfile::tempdir().unwrap();
+ let config = local_multiwrite_test_config(
+ primary.path(),
+ [("backup-a", backup_a.path()), ("backup-b", backup_b.path())],
+ );
+ let mfs =
+ mount_local_multiwrite_for_test(config, Arc::new(MemoryPathLockProvider::new())).await;
+ let mount = mfs.mounts.read().await.get("/local").unwrap().clone();
+
+ mfs.shutdown().await.unwrap();
+
+ assert_eq!(
+ MountableFS::as_multiwrite(&mount.fs)
+ .unwrap()
+ .background_task_count(),
+ 1
+ );
+ }
+
#[cfg(feature = "cache")]
#[tokio::test]
async fn shutdown_reaches_cache_queuefs_through_stats_wrapper() {
@@ -1949,115 +1970,6 @@ mod tests {
);
}
- #[cfg(feature = "cache")]
- #[tokio::test]
- async fn encrypted_multiwrite_mount_does_not_install_plaintext_cache() {
- use crate::cache::{CacheNamespace, CachePolicy};
- use crate::cache_runtime::CacheRuntime;
- use crate::lock::{
- MemoryPathLockProvider, PathLockConfig, PathLockManager, PathLockProvider,
- };
- use crate::plugins::MemFSPlugin;
-
- let mfs = Arc::new(MountableFS::with_cache_runtime(
- CacheRuntime::memory(),
- CacheNamespace::new("encrypted-multiwrite-test"),
- CachePolicy::default(),
- ));
- mfs.register_plugin(MemFSPlugin).await;
- mfs.set_encryption_config(Some([9u8; 32]), Some(1)).await;
- let provider: Arc<dyn PathLockProvider> = Arc::new(MemoryPathLockProvider::new());
- let manager = Arc::new(PathLockManager::new(
- mfs.clone() as Arc<dyn FileSystem>,
- provider,
- PathLockConfig::default(),
- ));
- mfs.set_pathlock_manager(manager).await;
-
- let mut config = multiwrite_test_config("memfs", "memfs", "/local");
- config.server_encryption_enabled = true;
- config.primary_encryption_enabled = true;
- mfs.mount(config).await.unwrap();
-
- let mounts = mfs.mounts.read().await;
- let mount_info = mounts.get("/local").expect("mounted entry should exist");
- let stats = (mount_info.fs.as_ref() as &dyn std::any::Any)
- .downcast_ref::<StatsWrappedFS>()
- .expect("stats wrapper should be outermost");
- assert!(
- (stats.inner_fs().as_ref() as &dyn std::any::Any)
- .downcast_ref::<MultiWriteWrappedFS>()
- .is_some(),
- "encrypted multi-write must not install a plaintext cache outside MultiWriteWrappedFS"
- );
- }
-
- #[cfg(feature = "cache")]
- #[tokio::test]
- async fn cached_unencrypted_multiwrite_keeps_admin_and_copy_fast_paths() {
- use crate::cache::{CacheNamespace, CachePolicy};
- use crate::cache_runtime::CacheRuntime;
- use crate::core::{FsContextInner, FS_CTX};
- use crate::plugins::MemFSPlugin;
-
- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::with_cache_runtime(
- CacheRuntime::memory(),
- CacheNamespace::new("cached-multiwrite-test"),
- CachePolicy::default(),
- )))
- .await;
- mfs.register_plugin(MemFSPlugin).await;
-
- let mut config = multiwrite_test_config("memfs", "memfs", "/local");
- config.backups.as_mut().unwrap().items[0].name = "backup1".to_string();
- mfs.mount(config).await.unwrap();
-
- let ctx = Arc::new(FsContextInner::new("acct"));
- FS_CTX
- .scope(ctx.clone(), async {
- let status = mfs.system_sync_status("/local").await.unwrap();
- assert_eq!(status["path"], "/");
- assert_eq!(status["entry_count"], 0);
-
- let retry = mfs.system_sync_retry("/local").await.unwrap();
- assert_eq!(retry["path"], "/");
- assert_eq!(retry["retried"], 0);
-
- mfs.mkdir("/local/docs", 0o755).await.unwrap();
- mfs.write("/local/docs/src.md", b"copied", 0, WriteFlag::Create)
- .await
- .unwrap();
-
- assert!(mfs
- .copy_within_mount("/local/docs/src.md", "/local/docs/dst.md")
- .await
- .unwrap());
- assert_eq!(
- mfs.read("/local/docs/dst.md", 0, 0).await.unwrap(),
- b"copied"
- );
- })
- .await;
-
- mfs.unmount("/local").await.unwrap();
- assert!(mfs.list_mounts().await.is_empty());
- }
-
- #[tokio::test]
- async fn multiwrite_mount_without_pathlock_returns_config_error() {
- use crate::plugins::MemFSPlugin;
-
- let mfs = MountableFS::new();
- mfs.register_plugin(MemFSPlugin).await;
-
- let error = mfs
- .mount(multiwrite_test_config("memfs", "memfs", "/local"))
- .await
- .unwrap_err();
-
- assert!(matches!(error, Error::Config(_)));
- }
-
#[tokio::test]
async fn test_mount_duplicate_error() {
let mfs = Arc::new(MountableFS::new());
@@ -2346,180 +2258,6 @@ mod tests {
assert_eq!(result[0].rel_path, "a/b/c.txt");
}
- #[tokio::test]
- async fn test_build_multi_write_fs_rejects_primary_encryption_disable() {
- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;
- mfs.register_plugin(MockPlugin::new("primary")).await;
- mfs.register_plugin(MockPlugin::new("backupfs")).await;
- mfs.set_encryption_config(Some([7u8; 32]), Some(1)).await;
-
- let backups = BackendsConfig {
- sync_type: "async".to_string(),
- write_ack_count: None,
- write_ack_timeout_ms: None,
- write_concurrency: None,
- retry_interval_ms: None,
- retry_backoff_base_ms: None,
- retry_max_retries_per_round: None,
- retry_quarantine_after_failures: None,
- read_probe_cache_ttl_ms: None,
- items: vec![BackendItemConfig {
- name: "backup1".to_string(),
- backend: "backupfs".to_string(),
- params: serde_json::Value::Null,
- timeout: None,
- encryption: None,
- operations: None,
- excludes: None,
- }],
- };
- let config = PluginConfig {
- name: "primary".to_string(),
- mount_path: "/local".to_string(),
- params: HashMap::new(),
- backups: Some(backups.clone()),
- server_encryption_enabled: true,
- ..PluginConfig::default()
- };
-
- let result = mfs.build_multi_write_fs(&config, &backups).await;
- assert!(result.is_err());
- assert!(matches!(result.err(), Some(Error::Config(_))));
- }
-
- #[tokio::test]
- async fn encrypted_multiwrite_reports_pathlock_ownership() {
- use crate::core::{FsContextInner, PathLockContext, FS_CTX};
- use crate::lock::{
- MemoryPathLockProvider, PathLockConfig, PathLockManager, PathLockProvider,
- };
- use crate::plugins::MemFSPlugin;
-
- let mfs = Arc::new(MountableFS::new());
- mfs.register_plugin(MemFSPlugin).await;
- mfs.set_encryption_config(Some([7u8; 32]), Some(1)).await;
- let provider: Arc<dyn PathLockProvider> = Arc::new(MemoryPathLockProvider::new());
- let manager = Arc::new(PathLockManager::new(
- mfs.clone() as Arc<dyn FileSystem>,
- provider,
- PathLockConfig::default(),
- ));
- mfs.set_pathlock_manager(manager.clone()).await;
-
- let mut config = multiwrite_test_config("memfs", "memfs", "/local");
- config.server_encryption_enabled = true;
- config.primary_encryption_enabled = true;
- mfs.mount(config).await.unwrap();
-
- assert!(mfs.encryption_handles_pathlock("/local/file.txt").await);
-
- let outer_ctx = Arc::new(FsContextInner::new("tenant"));
- let outer = FS_CTX
- .scope(outer_ctx, async {
- manager
- .acquire_exact("/local/tenant/file.txt", std::time::Duration::ZERO, None)
- .await
- })
- .await
- .unwrap();
- let ctx = Arc::new(FsContextInner::with_pathlock(
- "tenant",
- PathLockContext {
- lease_ref: Some(outer.lease.lease_ref.clone()),
- disable_auto_pathlock: false,
- },
- ));
- FS_CTX
- .scope(ctx, async {
- mfs.write("/local/tenant/file.txt", b"content", 0, WriteFlag::Create)
- .await
- })
- .await
- .unwrap();
- manager.release(&outer).await.unwrap();
- }
-
- #[tokio::test]
- async fn test_build_multi_write_fs_rejects_exclude_target_field() {
- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;
- mfs.register_plugin(MockPlugin::new("primary")).await;
- mfs.register_plugin(MockPlugin::new("backupfs")).await;
-
- let backups = BackendsConfig {
- sync_type: "async".to_string(),
- write_ack_count: None,
- write_ack_timeout_ms: None,
- write_concurrency: None,
- retry_interval_ms: None,
- retry_backoff_base_ms: None,
- retry_max_retries_per_round: None,
- retry_quarantine_after_failures: None,
- read_probe_cache_ttl_ms: None,
- items: vec![BackendItemConfig {
- name: "backup1".to_string(),
- backend: "backupfs".to_string(),
- params: serde_json::Value::Null,
- timeout: None,
- encryption: None,
- operations: None,
- excludes: Some(vec![RedirectPolicy::FileExtensionPolicy {
- extensions: vec!["\\.tmp$".to_string()],
- target: Some(vec!["should-not-exist".to_string()]),
- }]),
- }],
- };
- let config = PluginConfig {
- name: "primary".to_string(),
- mount_path: "/local".to_string(),
- params: HashMap::new(),
- backups: Some(backups.clone()),
- ..PluginConfig::default()
- };
-
- let result = mfs.build_multi_write_fs(&config, &backups).await;
- assert!(result.is_err());
- assert!(matches!(result.err(), Some(Error::Config(_))));
- }
-
- #[tokio::test]
- async fn test_build_multi_write_fs_rejects_reserved_primary_backup_name() {
- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;
- mfs.register_plugin(MockPlugin::new("primary")).await;
- mfs.register_plugin(MockPlugin::new("backupfs")).await;
-
- let backups = BackendsConfig {
- sync_type: "async".to_string(),
- write_ack_count: None,
- write_ack_timeout_ms: None,
- write_concurrency: None,
- retry_interval_ms: None,
- retry_backoff_base_ms: None,
- retry_max_retries_per_round: None,
- retry_quarantine_after_failures: None,
- read_probe_cache_ttl_ms: None,
- items: vec![BackendItemConfig {
- name: "primary".to_string(),
- backend: "backupfs".to_string(),
- params: serde_json::Value::Null,
- timeout: None,
- encryption: None,
- operations: None,
- excludes: None,
- }],
- };
- let config = PluginConfig {
- name: "primary".to_string(),
- mount_path: "/local".to_string(),
- params: HashMap::new(),
- backups: Some(backups.clone()),
- ..PluginConfig::default()
- };
-
- let result = mfs.build_multi_write_fs(&config, &backups).await;
- assert!(result.is_err());
- assert!(matches!(result.err(), Some(Error::Config(_))));
- }
-
#[tokio::test]
async fn test_single_backend_mount_wraps_stats_outside_encryption() {
use crate::lock::{
@@ -2552,83 +2290,4 @@ mod tests {
"single-backend encrypted mount should place encryption under stats"
);
}
-
- #[tokio::test]
- async fn test_multiwrite_mount_wraps_stats_outside_multiwrite() {
- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;
- mfs.register_plugin(MockPlugin::new("primary")).await;
- mfs.register_plugin(MockPlugin::new("backupfs")).await;
-
- mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))
- .await
- .unwrap();
-
- let mounts = mfs.mounts.read().await;
- let mount_info = mounts.get("/local").expect("mounted entry should exist");
- let wrapped = (mount_info.fs.as_ref() as &dyn std::any::Any)
- .downcast_ref::<StatsWrappedFS>()
- .expect("stats wrapper should be outermost");
- assert!(
- (wrapped.inner_fs().as_ref() as &dyn std::any::Any)
- .downcast_ref::<MultiWriteWrappedFS>()
- .is_some(),
- "multi-write mount should place MultiWriteWrappedFS directly under stats"
- );
- }
-
- #[tokio::test]
- async fn test_multiwrite_raw_access_is_rejected() {
- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;
- mfs.register_plugin(MockPlugin::new("primary")).await;
- mfs.register_plugin(MockPlugin::new("backupfs")).await;
-
- mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))
- .await
- .unwrap();
-
- let read_err = mfs.read_raw("/local/data.txt", 0, 0).await.unwrap_err();
- assert!(matches!(read_err, Error::InvalidOperation(_)));
-
- let write_err = mfs
- .write_raw("/local/data.txt", b"raw", WriteFlag::Create)
- .await
- .unwrap_err();
- assert!(matches!(write_err, Error::InvalidOperation(_)));
- }
-
- #[tokio::test]
- async fn test_mountable_multiwrite_admin_smoke() {
- let mfs = with_test_pathlock_manager(Arc::new(MountableFS::new())).await;
- mfs.register_plugin(MockPlugin::new("primary")).await;
- mfs.register_plugin(MockPlugin::new("backupfs")).await;
-
- mfs.mount(multiwrite_test_config("primary", "backupfs", "/local"))
- .await
- .unwrap();
-
- let status = crate::core::FS_CTX
- .scope(
- Arc::new(crate::core::context::FsContextInner::new(
- "acct".to_string(),
- )),
- async { mfs.system_sync_status("/local").await.unwrap() },
- )
- .await;
- assert_eq!(status["path"], "/");
- assert_eq!(status["entry_count"], 0);
- assert_eq!(status["pending_target_count"], 0);
- assert_eq!(status["capabilities"]["multi_instance_safe"], false);
-
- let retry = crate::core::FS_CTX
- .scope(
- Arc::new(crate::core::context::FsContextInner::new(
- "acct".to_string(),
- )),
- async { mfs.system_sync_retry("/local").await.unwrap() },
- )
- .await;
- assert_eq!(retry["path"], "/");
- assert_eq!(retry["retried"], 0);
- assert_eq!(retry["failed"], 0);
- }
}
diff --git a/crates/ragfs/src/core/multibackend_wrapper.rs b/crates/ragfs/src/core/multibackend_wrapper.rs
index c8f8149d8..8d5553a37 100644
--- a/crates/ragfs/src/core/multibackend_wrapper.rs
+++ b/crates/ragfs/src/core/multibackend_wrapper.rs
@@ -1,1415 +1,591 @@
-//! Multi-write wrapper — routes operations across primary and backup backends.
-//!
-//! Implements `MultiWriteWrappedFS` which handles:
-//! - Write fanout to primary + backup backends (sync/async)
-//! - Read routing with priority-based fallback chain
-//! - Redirect policy evaluation
-//! - Exclude policy filtering
-//! - `.redirect.json` / `.sync_log.json` metadata management
+//! Primary filesystem wrapper that submits V2 multi-write metadata events.
-use std::collections::{HashMap, HashSet};
-use std::future::Future;
-use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
+use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
-use futures::stream::{self, StreamExt};
-use regex::Regex;
-use serde_json::Value;
-use tokio::sync::{Mutex, Notify};
-use tracing::warn;
+use uuid::Uuid;
use super::context::{FsContext, FS_CTX};
use super::encryption_wrapper::EncryptionWrappedFS;
use super::errors::{Error, Result};
-use super::filesystem::{
- apply_read_dir_options, normalize_prefix_path, paginate_entries, relative_depth,
- relative_match_file, FileSystem,
-};
+use super::filesystem::{normalize_prefix_path, FileSystem};
use super::types::{
- BackendRole, BackendSyncState, FileInfo, GlobEntry, GlobPage, GrepOptions, GrepResult,
- ListSortBy, OperationItemConfig, RedirectEntry, RedirectPolicy, SortOrder, SyncLogEntry,
- SyncOp, SyncType, TreeEntry, WriteFlag,
-};
-use crate::core::glob::{
- compare_rel_paths, decode_offset_token, encode_offset_token, PreparedGlob,
+ BackendRole, FileInfo, GlobPage, GrepOptions, GrepResult, ListSortBy, SortOrder, TreeEntry,
+ WriteFlag,
};
-use crate::core::internal_names::is_hidden_internal_name;
+use crate::core::internal_names::is_multiwrite_internal_path;
+use crate::metrics::{RagfsMetric, RagfsMetricValue};
+use crate::multibackend::catch_up::CatchUpTarget;
use crate::multibackend::meta::{
- current_required_ctx, file_name, parent_dir, DefaultFsContextResolver, FsContextResolver,
- MetaStateStore, PathSerializer, MULTIWRITE_INTERNAL_NAMES,
+ current_required_ctx, DefaultFsContextResolver, FsContextResolver, MetadataStore,
+ MultiWriteWorker,
};
+use crate::multibackend::migration::{FullDataImporter, ProtocolDetector};
+use crate::multibackend::model::{
+ PartitionState, PartitionsManifest, PendingEvent, PendingEventKind, ProtocolState,
+ ProtocolStatus, ProtocolVersion, SegmentEventType, SegmentManifest,
+};
+use crate::multibackend::provider::{bootstrap_account, MultiWriteProvider};
+use crate::multibackend::runtime::MultiWriteRuntime;
-mod retry;
-mod routing;
-#[cfg(test)]
-mod tests;
-
-/// Default chunk size used when copying file state from primary to backup.
-const DEFAULT_COPY_CHUNK_SIZE: usize = 8 * 1024 * 1024;
-/// Default retry loop interval.
-const DEFAULT_RETRY_INTERVAL: Duration = Duration::from_secs(30);
-/// Default retry backoff base in milliseconds.
-const DEFAULT_RETRY_BACKOFF_BASE_MS: u64 = 1000;
-/// Default maximum retries per file per retry round.
-const DEFAULT_MAX_RETRIES_PER_ROUND: usize = 3;
-/// Default failure threshold before a target is quarantined.
-const DEFAULT_QUARANTINE_AFTER_FAILURES: u32 = 9;
-/// Default wait timeout when shutting down background tasks.
-const DEFAULT_SHUTDOWN_WAIT: Duration = Duration::from_secs(5);
-/// Maximum concurrent metadata reads when merging redirect entries into tree output.
-const DEFAULT_TREE_REDIRECT_META_CONCURRENCY: usize = 32;
-
-#[derive(Debug)]
-struct SyncFanoutTaskResult {
- backend: String,
- result: Result<()>,
-}
-
-#[derive(Clone)]
-enum BackupWriteOp {
- Replay(SyncOp),
- WriteFile {
- data: Arc<Vec<u8>>,
- offset: u64,
- flags: WriteFlag,
- },
- EnsureParentDirs {
- mode: u32,
- },
-}
-
-impl BackupWriteOp {
- /// Apply this backup-side operation under the provided request context.
- async fn apply(
- &self,
- inner: &Inner,
- backup: Arc<dyn FileSystem>,
- path: &str,
- ctx: &FsContext,
- ) -> Result<()> {
- match self {
- Self::Replay(op) => {
- op.replay(inner.primary().backend.clone(), backup, path, ctx)
- .await
- }
- Self::WriteFile {
- data,
- offset,
- flags,
- } => {
- let data = Arc::clone(data);
- let offset = *offset;
- let flags = *flags;
- FS_CTX
- .scope(ctx.clone(), async move {
- backup.ensure_parent_dirs(path, 0o755).await?;
- backup.write(path, data.as_slice(), offset, flags).await?;
- Ok(())
- })
- .await
- }
- Self::EnsureParentDirs { mode } => {
- let mode = *mode;
- FS_CTX
- .scope(ctx.clone(), async move {
- backup.ensure_parent_dirs(path, mode).await
- })
- .await
- }
- }
- }
-}
-
-/// Cloned target data passed to fanout strategies without borrowing `Inner`.
-#[derive(Clone)]
-struct FanoutTarget {
- name: String,
- backend: Arc<dyn FileSystem>,
-}
-
-/// Effective sync work item with resolved target backend names.
-pub(crate) struct SyncWorkEntry {
- pub(crate) file_path: String,
- pub(crate) entry: SyncLogEntry,
- pub(crate) targets: Vec<String>,
-}
-
-#[derive(Clone, Copy)]
-enum ReadRouteSource {
- Backup,
- Primary,
- Redirect,
- Miss,
-}
-
-/// Builder-style sync mode configuration.
-pub enum SyncMode {
- /// Synchronous fanout requiring backup acknowledgement.
- Sync {
- /// Minimum backup acknowledgements required for a successful write.
- ack_count: usize,
- /// Maximum time to wait for backup acknowledgements, in milliseconds.
- timeout_ms: u64,
- },
- /// Asynchronous fanout with background retry.
- Async,
-}
+const COPY_CHUNK_SIZE: usize = 8 * 1024 * 1024;
-/// A backend entry within the multi-write wrapper.
+/// One configured primary or backup filesystem.
pub struct BackendEntry {
- /// Logical name (globally unique)
+ /// Logical backend name.
pub name: String,
- /// Role: Primary or Backup
+ /// Backend role.
pub role: BackendRole,
- /// The backend filesystem handle (may be encrypted)
+ /// Filesystem handle, including encryption when configured.
pub backend: Arc<dyn FileSystem>,
- /// Optional raw backend handle used by primary verbatim copy fast-paths.
+ /// Optional raw filesystem handle.
pub raw_backend: Option<Arc<dyn FileSystem>>,
- /// Operations this backend participates in (only for Backup)
- pub operations: Vec<OperationItemConfig>,
- /// Exclude policies (only for Backup)
- pub excludes: Vec<RedirectPolicy>,
-}
-
-impl BackendEntry {
- /// Check if this backend participates in read operations.
- fn participates_in_read(&self) -> bool {
- self.operations.iter().any(|op| op.operation == "read")
- }
-
- /// Check if this backend participates in write operations.
- /// Backups default to write-enabled when operations is empty.
- fn participates_in_write(&self) -> bool {
- if self.operations.is_empty() {
- true
- } else {
- self.operations.iter().any(|op| op.operation == "write")
- }
- }
-
- /// Get read priority (lower = higher priority). Returns None if not read-enabled.
- fn read_priority(&self) -> Option<u32> {
- self.operations
- .iter()
- .find(|op| op.operation == "read")
- .map(|op| op.priority)
- }
-
- /// Convert this backend entry into a fanout target.
- fn fanout_target(&self) -> FanoutTarget {
- FanoutTarget {
- name: self.name.clone(),
- backend: self.backend.clone(),
- }
- }
-}
-
-/// File policy trait — shared by redirects and excludes.
-pub trait FilePolicy {
- /// Check if this policy matches the given file.
- fn matches(&self, path: &str, size: u64) -> bool;
-}
-
-impl FilePolicy for RedirectPolicy {
- fn matches(&self, path: &str, size: u64) -> bool {
- match self {
- RedirectPolicy::FileOverSizePolicy { max_size_mb, .. } => {
- let max_bytes = max_size_mb * 1024 * 1024;
- size > max_bytes
- }
- RedirectPolicy::FileExtensionPolicy { extensions, .. } => {
- let name = file_name(path);
- extensions.iter().any(|ext_pattern| {
- if let Ok(re) = Regex::new(ext_pattern) {
- re.is_match(name)
- } else {
- name.ends_with(ext_pattern.as_str())
- }
- })
- }
- }
- }
-}
-
-impl SyncOp {
- /// Replay this operation on one backup backend using the original request semantics.
- async fn replay(
- &self,
- primary: Arc<dyn FileSystem>,
- backup: Arc<dyn FileSystem>,
- file_path: &str,
- ctx: &FsContext,
- ) -> Result<()> {
- match self {
- SyncOp::SyncFile { size } => {
- let size = *size;
- copy_current_primary_state(primary, backup, file_path, size, ctx).await
- }
- SyncOp::Create => {
- FS_CTX
- .scope(ctx.clone(), async { backup.create(file_path).await })
- .await
- }
- SyncOp::Mkdir { mode } => {
- let mode = *mode;
- FS_CTX
- .scope(ctx.clone(), async { backup.mkdir(file_path, mode).await })
- .await
- }
- SyncOp::Remove => {
- match FS_CTX
- .scope(ctx.clone(), async { backup.remove(file_path).await })
- .await
- {
- Ok(()) | Err(Error::NotFound(_)) => Ok(()),
- Err(e) => Err(e),
- }
- }
- SyncOp::RemoveAll => {
- match FS_CTX
- .scope(ctx.clone(), async { backup.remove_all(file_path).await })
- .await
- {
- Ok(()) | Err(Error::NotFound(_)) => Ok(()),
- Err(e) => Err(e),
- }
- }
- SyncOp::Rename { to } => {
- let to = to.clone();
- FS_CTX
- .scope(ctx.clone(), async {
- backup.ensure_parent_dirs(&to, 0o755).await?;
- backup.rename(file_path, &to).await
- })
- .await
- }
- SyncOp::Chmod { mode } => {
- let mode = *mode;
- FS_CTX
- .scope(ctx.clone(), async { backup.chmod(file_path, mode).await })
- .await
- }
- }
- }
-}
-
-/// Copy one file between two filesystem handles in bounded-size chunks.
-async fn copy_file_state(
- source: Arc<dyn FileSystem>,
- source_path: &str,
- destination: Arc<dyn FileSystem>,
- destination_path: &str,
- size: u64,
- ctx: &FsContext,
-) -> Result<()> {
- FS_CTX
- .scope(ctx.clone(), async {
- destination
- .ensure_parent_dirs(destination_path, 0o755)
- .await?;
- if size == 0 {
- if destination.exists(destination_path).await {
- return destination.truncate(destination_path, 0).await;
- }
- return destination.create(destination_path).await;
- }
-
- let mut offset = 0u64;
- while offset < size {
- let chunk_len = (size - offset).min(DEFAULT_COPY_CHUNK_SIZE as u64);
- let chunk = source.read(source_path, offset, chunk_len).await?;
- if chunk.is_empty() {
- if offset == 0 {
- if destination.exists(destination_path).await {
- destination.truncate(destination_path, 0).await?;
- } else {
- destination.create(destination_path).await?;
- }
- }
- break;
- }
- let flag = if offset == 0 {
- WriteFlag::Create
- } else {
- WriteFlag::None
- };
- destination
- .write(destination_path, &chunk, offset, flag)
- .await?;
- offset = offset.saturating_add(chunk.len() as u64);
- }
- Ok(())
- })
- .await
-}
-
-/// Copy the current file state from primary to backup in bounded-size chunks.
-async fn copy_current_primary_state(
- primary: Arc<dyn FileSystem>,
- backup: Arc<dyn FileSystem>,
- file_path: &str,
- size: u64,
- ctx: &FsContext,
-) -> Result<()> {
- copy_file_state(primary, file_path, backup, file_path, size, ctx).await
-}
-
-/// Copy one file within the primary raw backend without decrypting and re-encrypting bytes.
-async fn copy_raw_primary_state(
- primary_raw: Arc<dyn FileSystem>,
- source_path: &str,
- destination_path: &str,
- ctx: &FsContext,
-) -> Result<()> {
- let raw_size = FS_CTX
- .scope(ctx.clone(), async {
- let source_info = primary_raw.stat(source_path).await?;
- if source_info.is_dir {
- return Err(Error::IsADirectory(source_path.to_string()));
- }
- Ok::<u64, Error>(source_info.size)
- })
- .await?;
-
- copy_file_state(
- primary_raw.clone(),
- source_path,
- primary_raw,
- destination_path,
- raw_size,
- ctx,
- )
- .await
}
-/// Inner state shared via Arc for async spawn and retry_loop.
pub(crate) struct Inner {
- /// All backend entries (primary at index 0)
- backends: Vec<BackendEntry>,
- /// Index of the primary backend
- primary_idx: usize,
- /// Sync type: Async or Sync.
- sync_type: SyncType,
- /// Minimum backup ack count for sync mode
- write_ack_count: usize,
- /// Timeout for waiting backup ack in sync mode (ms)
- write_ack_timeout_ms: u64,
- /// Semaphore for async write concurrency control
- write_sem: Option<Arc<tokio::sync::Semaphore>>,
- /// Primary redirect policies
- redirects: Vec<RedirectPolicy>,
- /// Metadata store (encrypted via primary_backend)
- pub(crate) meta_store: MetaStateStore,
- /// Per-path serialization queues
- path_queues: PathSerializer,
- /// Directories that currently have outstanding retry work.
- pending_dirs: Mutex<HashSet<String>>,
- /// Retry loop interval.
- retry_interval: Duration,
- /// Base retry backoff in milliseconds.
- pub(crate) retry_backoff_base_ms: u64,
- /// Maximum retry attempts for one target in one round.
- pub(crate) max_retry_per_round: usize,
- /// Failure threshold before quarantining one target.
- quarantine_after_failures: u32,
- /// Number of background tasks currently in flight.
- background_tasks: AtomicUsize,
- /// Notifier fired when background task count reaches zero.
- idle_notify: Notify,
- /// Read route hit metrics.
- read_backup_hits: AtomicU64,
- read_primary_hits: AtomicU64,
- read_redirect_hits: AtomicU64,
- read_misses: AtomicU64,
- /// Cancellation flag for the background retry loop.
- retry_cancelled: AtomicBool,
- /// Wake-up signal used to stop retry_loop promptly on drop.
- retry_shutdown: Notify,
+ primary: BackendEntry,
+ runtime: MultiWriteRuntime,
+ ctx_resolver: Arc<dyn FsContextResolver>,
}
-/// Multi-write wrapped filesystem.
+/// Filesystem wrapper that commits primary operations before queuing V2 events.
+#[derive(Clone)]
pub struct MultiWriteWrappedFS {
pub(crate) inner: Arc<Inner>,
+ metadata_store: Arc<MetadataStore>,
+ metadata_provider: Arc<dyn MultiWriteProvider>,
}
-/// Builder for `MultiWriteWrappedFS`.
+/// Builder for one V2 multi-write wrapper and its flush runtime.
pub struct MultiWriteWrappedFSBuilder {
primary_backend: Arc<dyn FileSystem>,
primary_raw_backend: Option<Arc<dyn FileSystem>>,
backup_entries: Vec<BackendEntry>,
- redirects: Vec<RedirectPolicy>,
- sync_mode: SyncMode,
- write_concurrency: Option<usize>,
- retry_interval: Duration,
- retry_backoff_base_ms: u64,
- max_retry_per_round: usize,
- quarantine_after_failures: u32,
+ metadata_store: Option<Arc<MetadataStore>>,
+ metadata_provider: Option<Arc<dyn MultiWriteProvider>>,
+ initial_partitions: u32,
+ checkpoint_interval: Duration,
ctx_resolver: Arc<dyn FsContextResolver>,
}
impl MultiWriteWrappedFSBuilder {
- /// Attach the raw primary backend so the wrapper can do verbatim primary copies.
+ /// Attach the raw primary backend used by internal operations and physical copies.
pub fn with_primary_raw_backend(mut self, primary_raw_backend: Arc<dyn FileSystem>) -> Self {
self.primary_raw_backend = Some(primary_raw_backend);
self
}
- /// Set backup backend entries on the builder.
+ /// Attach configured backup handles for later catch-up workers.
pub fn with_backups(mut self, backup_entries: Vec<BackendEntry>) -> Self {
self.backup_entries = backup_entries;
self
}
- /// Set redirect policies for the primary backend.
- pub fn with_redirects(mut self, redirects: Vec<RedirectPolicy>) -> Self {
- self.redirects = redirects;
+ /// Attach the metadata store used for account initialization and routing.
+ pub fn with_metadata_store(mut self, metadata_store: Arc<MetadataStore>) -> Self {
+ self.metadata_store = Some(metadata_store);
self
}
- /// Select the sync mode used by write fanout.
- pub fn sync_mode(mut self, sync_mode: SyncMode) -> Self {
- self.sync_mode = sync_mode;
+ /// Attach the provider used by the background flush worker.
+ pub fn with_metadata_provider(
+ mut self,
+ metadata_provider: Arc<dyn MultiWriteProvider>,
+ ) -> Self {
+ self.metadata_provider = Some(metadata_provider);
self
}
- /// Set the maximum number of concurrent async backup writes.
- pub fn write_concurrency(mut self, write_concurrency: Option<usize>) -> Self {
- self.write_concurrency = write_concurrency;
+ /// Set the initial metadata partition count for newly observed accounts.
+ pub fn initial_partitions(mut self, initial_partitions: u32) -> Self {
+ self.initial_partitions = initial_partitions;
self
}
- /// Configure retry loop interval.
- pub fn retry_interval(mut self, retry_interval: Duration) -> Self {
- self.retry_interval = retry_interval;
+ /// Set the interval between checkpoint discovery rounds.
+ pub fn checkpoint_interval(mut self, checkpoint_interval: Duration) -> Self {
+ self.checkpoint_interval = checkpoint_interval;
self
}
- /// Configure retry backoff base duration in milliseconds.
- pub fn retry_backoff_base_ms(mut self, retry_backoff_base_ms: u64) -> Self {
- self.retry_backoff_base_ms = retry_backoff_base_ms;
- self
- }
-
- /// Configure the maximum number of retries per round.
- pub fn max_retry_per_round(mut self, max_retry_per_round: usize) -> Self {
- self.max_retry_per_round = max_retry_per_round.max(1);
- self
- }
-
- /// Configure quarantine threshold for one path/backup pair.
- pub fn quarantine_after_failures(mut self, quarantine_after_failures: u32) -> Self {
- self.quarantine_after_failures = quarantine_after_failures.max(1);
- self
- }
-
- /// Configure the context resolver used by retry and admin background paths.
+ /// Configure the resolver for mount-relative foreground paths.
pub fn ctx_resolver(mut self, ctx_resolver: Arc<dyn FsContextResolver>) -> Self {
self.ctx_resolver = ctx_resolver;
self
}
- /// Build the multi-write wrapper and start the retry loop when needed.
- pub fn build(self) -> Result<MultiWriteWrappedFS> {
- let mut backends = Vec::new();
- backends.push(BackendEntry {
+ /// Build the wrapper after asynchronously starting its V2 runtime.
+ pub async fn build(self) -> Result<MultiWriteWrappedFS> {
+ let fs = self.build_inactive().await?;
+ fs.activate();
+ Ok(fs)
+ }
+
+ /// Build the wrapper while keeping its V2 runtime inactive.
+ pub(crate) async fn build_inactive(self) -> Result<MultiWriteWrappedFS> {
+ let metadata_store = self
+ .metadata_store
+ .ok_or_else(|| Error::config("multi-write metadata store is required"))?;
+ let metadata_provider = self
+ .metadata_provider
+ .ok_or_else(|| Error::config("multi-write metadata provider is required"))?;
+ let backup_names: Vec<String> = self
+ .backup_entries
+ .iter()
+ .map(|entry| entry.name.clone())
+ .collect();
+ let catch_up_targets = self
+ .backup_entries
+ .iter()
+ .map(|entry| CatchUpTarget {
+ backend_id: entry.name.clone(),
+ backend: entry.backend.clone(),
+ })
+ .collect();
+ let protocol = ProtocolDetector::new(metadata_store.clone())
+ .detect()
+ .await?;
+ let configured_backups = backup_names.iter().collect::<BTreeSet<_>>();
+ for account in metadata_store.initialized_accounts().await? {
+ let path = metadata_store.paths().account_manifest(&account)?.0;
+ let manifest: PartitionsManifest = match metadata_store.read_json(&path).await {
+ Ok(manifest) => manifest,
+ Err(Error::NotFound(_)) => continue,
+ Err(error) => return Err(error),
+ };
+ for &partition_id in manifest.partitions.keys() {
+ let path = metadata_store
+ .paths()
+ .segment_manifest(&account, partition_id)?
+ .0;
+ let segments: SegmentManifest = metadata_store.read_json(&path).await?;
+ segments.validate()?;
+ let persisted = segments.backend_states.keys().collect::<BTreeSet<_>>();
+ if persisted != configured_backups {
+ return Err(Error::not_supported(format!(
+ "backup membership changed for account {account}"
+ )));
+ }
+ }
+ bootstrap_account(
+ metadata_provider.as_ref(),
+ metadata_store.as_ref(),
+ &account,
+ &manifest,
+ )
+ .await?;
+ }
+ let runtime = MultiWriteRuntime::prepare(
+ metadata_store.clone(),
+ metadata_provider.clone(),
+ self.initial_partitions,
+ backup_names.clone(),
+ Some(self.primary_backend.clone()),
+ catch_up_targets,
+ self.checkpoint_interval,
+ )
+ .await;
+ let primary = BackendEntry {
name: "primary".to_string(),
role: BackendRole::Primary,
- backend: self.primary_backend.clone(),
- raw_backend: self.primary_raw_backend.clone(),
- operations: Vec::new(),
- excludes: Vec::new(),
- });
- backends.extend(self.backup_entries);
-
- let (sync_type, write_ack_count, write_ack_timeout_ms) = match self.sync_mode {
- SyncMode::Sync {
- ack_count,
- timeout_ms,
- } => (SyncType::Sync, ack_count, timeout_ms),
- SyncMode::Async => (SyncType::Async, usize::MAX, 0),
+ backend: self.primary_backend,
+ raw_backend: self.primary_raw_backend,
};
-
- let write_sem = self
- .write_concurrency
- .filter(|&n| n > 0)
- .map(|n| Arc::new(tokio::sync::Semaphore::new(n)));
-
- let meta_store = MetaStateStore::new(self.primary_backend, self.ctx_resolver);
-
- let inner = Arc::new(Inner {
- backends,
- primary_idx: 0,
- sync_type,
- write_ack_count,
- write_ack_timeout_ms,
- write_sem,
- redirects: self.redirects,
- meta_store,
- path_queues: PathSerializer::new(),
- pending_dirs: Mutex::new(HashSet::new()),
- retry_interval: self.retry_interval,
- retry_backoff_base_ms: self.retry_backoff_base_ms,
- max_retry_per_round: self.max_retry_per_round,
- quarantine_after_failures: self.quarantine_after_failures,
- background_tasks: AtomicUsize::new(0),
- idle_notify: Notify::new(),
- read_backup_hits: AtomicU64::new(0),
- read_primary_hits: AtomicU64::new(0),
- read_redirect_hits: AtomicU64::new(0),
- read_misses: AtomicU64::new(0),
- retry_cancelled: AtomicBool::new(false),
- retry_shutdown: Notify::new(),
- });
-
- // Start retry_loop if there are write-enabled backups.
- if inner.write_backups().next().is_some() {
- inner.background_task_started();
- tokio::spawn(Inner::retry_loop(Arc::clone(&inner)));
+ let fs = MultiWriteWrappedFS {
+ inner: Arc::new(Inner {
+ primary,
+ runtime,
+ ctx_resolver: self.ctx_resolver,
+ }),
+ metadata_store: metadata_store.clone(),
+ metadata_provider: metadata_provider.clone(),
+ };
+ if protocol.status != ProtocolStatus::Stable {
+ let importer_primary: Arc<dyn FileSystem> = Arc::new(fs.clone());
+ let importer = FullDataImporter::new(
+ importer_primary,
+ metadata_store,
+ metadata_provider,
+ self.initial_partitions,
+ backup_names,
+ );
+ fs.inner.runtime.set_import_worker(async move {
+ let result = importer.run().await;
+ if let Err(error) = &result {
+ tracing::error!(error = %error, "multi-write full-data import failed");
+ }
+ result
+ });
}
-
- Ok(MultiWriteWrappedFS { inner })
+ Ok(fs)
}
}
impl MultiWriteWrappedFS {
- /// Return this wrapper's current background task count without scanning metadata.
- pub(crate) fn background_task_count(&self) -> usize {
- self.inner.background_tasks.load(Ordering::SeqCst)
- }
-
- /// Start building a multi-write wrapper from a primary backend.
+ /// Start building a wrapper around the supplied primary filesystem.
pub fn builder(primary_backend: Arc<dyn FileSystem>) -> MultiWriteWrappedFSBuilder {
MultiWriteWrappedFSBuilder {
primary_backend,
primary_raw_backend: None,
backup_entries: Vec::new(),
- redirects: Vec::new(),
- sync_mode: SyncMode::Async,
- write_concurrency: None,
- retry_interval: DEFAULT_RETRY_INTERVAL,
- retry_backoff_base_ms: DEFAULT_RETRY_BACKOFF_BASE_MS,
- max_retry_per_round: DEFAULT_MAX_RETRIES_PER_ROUND,
- quarantine_after_failures: DEFAULT_QUARANTINE_AFTER_FAILURES,
+ metadata_store: None,
+ metadata_provider: None,
+ initial_partitions: 16,
+ checkpoint_interval: Duration::from_secs(86_400),
ctx_resolver: Arc::new(DefaultFsContextResolver),
}
}
+ /// Release prepared runtime work after the wrapper becomes reachable.
+ pub(crate) fn activate(&self) {
+ self.inner.runtime.activate();
+ }
+
/// Return the raw primary backend for mount-level internal-name operations.
pub(crate) fn primary_raw_backend(&self) -> Option<Arc<dyn FileSystem>> {
- self.inner.primary().raw_backend.clone()
+ self.inner.primary.raw_backend.clone()
}
- /// Return whether encrypted backends own dual-path PathLock acquisition.
+ /// Return whether the primary encryption wrapper owns PathLock acquisition.
pub(crate) fn encryption_handles_pathlock(&self) -> bool {
- let any = self.inner.primary().backend.as_ref() as &dyn std::any::Any;
+ let any = self.inner.primary.backend.as_ref() as &dyn std::any::Any;
any.downcast_ref::<EncryptionWrappedFS>().is_some()
}
-}
-
-impl Inner {
- /// Build the per-path/per-backend queue key used by both fanout and retry.
- fn backup_queue_key(path: &str, backup_name: &str) -> String {
- format!("{}\0{}", path, backup_name)
- }
-
- /// Iterate over write-enabled backup entries.
- fn write_backups(&self) -> impl Iterator<Item = &BackendEntry> {
- self.backends[self.primary_idx + 1..]
- .iter()
- .filter(|be| be.participates_in_write())
- }
-
- /// Resolve write-enabled backup targets after applying exclude policies.
- fn write_targets(&self, path: &str, size: u64) -> Vec<FanoutTarget> {
- self.write_backups()
- .filter(|be| !self.is_excluded(be, path, size))
- .map(BackendEntry::fanout_target)
- .collect()
- }
-
- /// Resolve explicitly named backup targets.
- fn named_targets(&self, target_names: &[String]) -> Vec<FanoutTarget> {
- target_names
- .iter()
- .filter_map(|name| self.backup_by_name(name))
- .map(BackendEntry::fanout_target)
- .collect()
- }
-
- /// Execute one backup write synchronously so redirect visibility only appears
- /// after at least one target has durably materialized the file contents.
- async fn write_first_target(
- inner: &Arc<Inner>,
- path: &str,
- target: &FanoutTarget,
- ctx: &FsContext,
- op: BackupWriteOp,
- ) -> Result<()> {
- let queue_key = Self::backup_queue_key(path, &target.name);
- let fs = target.backend.clone();
- let ack_ctx = ctx.clone();
-
- inner
- .path_queues
- .with_path_lock(&queue_key, || async move {
- op.apply(inner, fs, path, ctx).await
- })
- .await?;
-
- inner
- .update_backup_acked_seq(path, &target.name, &ack_ctx)
- .await
- }
-
- /// Resolve effective target backend names for sync/retry work.
- pub(crate) fn target_backend_names(
- &self,
- redirect_meta: &super::types::RedirectMeta,
- file_name: &str,
- file_path: &str,
- sync_entry: &SyncLogEntry,
- ) -> Vec<String> {
- if let Some(redir) = redirect_meta.entries.get(file_name) {
- return redir.targets.clone();
- }
- let policy_size = self.retry_policy_size(sync_entry);
- self.write_backups()
- .filter(|be| !self.is_excluded(be, file_path, policy_size))
- .map(|be| be.name.clone())
- .collect()
- }
- /// Iterate over read-enabled backup entries sorted by priority.
- fn read_backups_sorted(&self) -> Vec<&BackendEntry> {
- let mut read_backups: Vec<&BackendEntry> = self.backends[self.primary_idx + 1..]
- .iter()
- .filter(|be| be.participates_in_read())
- .collect();
- read_backups.sort_by_key(|be| be.read_priority().unwrap_or(u32::MAX));
- read_backups
- }
-
- /// Get the primary backend entry.
- pub(crate) fn primary(&self) -> &BackendEntry {
- &self.backends[self.primary_idx]
- }
-
- /// Get a backup entry by name.
- fn backup_by_name(&self, name: &str) -> Option<&BackendEntry> {
- self.backends.iter().find(|be| be.name == name)
- }
-
- /// Check if a file should be excluded from a backup.
- fn is_excluded(&self, backup: &BackendEntry, path: &str, size: u64) -> bool {
- backup
- .excludes
- .iter()
- .any(|policy| policy.matches(path, size))
- }
-
- /// Check if a file matches any redirect policy.
- fn check_redirect(&self, path: &str, size: u64) -> Option<Vec<String>> {
- for policy in &self.redirects {
- if policy.matches(path, size) {
- let targets = match policy {
- RedirectPolicy::FileOverSizePolicy { target, .. } => target.clone(),
- RedirectPolicy::FileExtensionPolicy { target, .. } => target.clone(),
- };
- return targets;
- }
- }
- None
- }
-
- /// Resolve persisted redirect targets for one already-written file path.
- async fn redirect_targets_for_path(
- &self,
- path: &str,
- ctx: &FsContext,
- ) -> Result<Option<Vec<String>>> {
- if self.redirects.is_empty() {
- return Ok(None);
- }
-
- let dir = parent_dir(path);
- let name = file_name(path);
- let redirect_meta = self.meta_store.get_redirect_meta(&dir, ctx).await?;
- Ok(redirect_meta
- .entries
- .get(name)
- .map(|entry| entry.targets.clone()))
- }
-
- /// Generate and persist the next sequence number.
- async fn next_seq(&self) -> Result<u64> {
- self.meta_store.next_seq().await
- }
-
- /// Resolve a file size for retry-time policy decisions.
- fn retry_policy_size(&self, sync_entry: &SyncLogEntry) -> u64 {
- match &sync_entry.op {
- SyncOp::SyncFile { size } => *size,
- _ => 0,
- }
- }
-
- /// Execute the primary-write, sync-log and backup-fanout pipeline.
- async fn execute_write<R, P, PFut>(
- inner: &Arc<Self>,
- path: String,
- size: u64,
- sync_op: Option<SyncOp>,
- backup_op: Option<BackupWriteOp>,
- primary_fn: P,
- ) -> Result<R>
- where
- P: FnOnce(Arc<dyn FileSystem>) -> PFut + Send,
- PFut: Future<Output = Result<R>> + Send,
- {
- let ctx = current_required_ctx()?;
- inner.invalidate_read_route(&path).await;
-
- let prepared_entry = if let Some(entry) = sync_op {
- let dir = parent_dir(&path);
- let name = file_name(&path).to_string();
- let seq = inner.next_seq().await?;
- inner
- .meta_store
- .update_dir_meta(&dir, &ctx, {
- let name = name.clone();
- move |_redirect, sync_log| {
- sync_log.entries.insert(name, SyncLogEntry::new(seq, entry));
- Ok(())
- }
- })
- .await?;
- Some((dir, name, seq))
- } else {
- None
- };
-
- let result = match FS_CTX
- .scope(ctx.clone(), primary_fn(inner.primary().backend.clone()))
- .await
- {
- Ok(result) => result,
- Err(err) => {
- if let Some((dir, name, seq)) = prepared_entry.as_ref() {
- let _ = inner
- .meta_store
- .update_dir_meta(dir, &ctx, {
- let name = name.clone();
- let seq = *seq;
- move |_redirect, sync_log| {
- let should_remove =
- sync_log.entries.get(&name).is_some_and(|entry| {
- entry.latest_seq == seq && !entry.is_primary_committed()
- });
- if should_remove {
- sync_log.entries.remove(&name);
- }
- Ok(())
- }
- })
- .await;
+ /// Return one while the flush worker is active and zero after it exits.
+ pub(crate) fn background_task_count(&self) -> usize {
+ usize::from(self.inner.runtime.is_running())
+ }
+
+ /// Read and validate one complete V2 multi-write metrics snapshot.
+ pub(crate) async fn metrics(&self) -> Result<Vec<RagfsMetric>> {
+ let protocol_path = self.metadata_store.paths().mount_protocol().0;
+ let protocol: ProtocolState = self.metadata_store.read_json(&protocol_path).await?;
+ protocol.validate()?;
+ let mut lag = BTreeMap::<(String, u32), u64>::new();
+ for account in self.metadata_store.initialized_accounts().await? {
+ let path = self.metadata_store.paths().account_manifest(&account)?.0;
+ let manifest: PartitionsManifest = self.metadata_store.read_json(&path).await?;
+ manifest.validate()?;
+ for (&partition_id, entry) in &manifest.partitions {
+ if entry.state != PartitionState::Stable {
+ continue;
}
- return Err(err);
- }
- };
-
- if let Some((dir, name, seq)) = prepared_entry {
- let mut commit_superseded = false;
- inner
- .meta_store
- .update_dir_meta(&dir, &ctx, |_redirect, sync_log| {
- let entry = sync_log.entries.get_mut(&name).ok_or_else(|| {
- Error::internal(format!(
- "prepared sync log entry missing while committing '{}'",
- name
- ))
- })?;
- if entry.latest_seq != seq {
- warn!(
- path = %path,
- name = %name,
- expected_seq = seq,
- actual_seq = entry.latest_seq,
- "prepared sync log entry was superseded while committing; skip stale backup fanout"
- );
- commit_superseded = true;
- return Ok(());
- }
- entry.mark_primary_committed();
- Ok(())
- })
- .await?;
- if commit_superseded {
- inner.refresh_pending_dir(&dir, &ctx).await?;
- return Ok(result);
- }
- inner.mark_pending_dir(&dir).await;
- }
-
- let dir = parent_dir(&path);
- let fanout_result = if let Some(backup_op) = backup_op {
- Inner::fanout_write(inner, &path, size, ctx.clone(), backup_op).await
- } else {
- Ok(())
- };
- inner.refresh_pending_dir(&dir, &ctx).await?;
- fanout_result?;
- Ok(result)
- }
-
- /// Execute a write that may be redirected away from the primary backend.
- async fn execute_write_with_redirect<P, PFut>(
- inner: &Arc<Self>,
- path: String,
- size: u64,
- sync_op: SyncOp,
- backup_op: BackupWriteOp,
- primary_fn: P,
- ) -> Result<u64>
- where
- P: FnOnce(Arc<dyn FileSystem>) -> PFut + Send,
- PFut: Future<Output = Result<u64>> + Send,
- {
- let ctx = current_required_ctx()?;
- inner.invalidate_read_route(&path).await;
-
- if let Some(targets) = inner.check_redirect(&path, size) {
- let dir = parent_dir(&path);
- let name = file_name(&path).to_string();
- let seq = inner.next_seq().await?;
- let resolved_targets = inner.named_targets(&targets);
- let first_target = resolved_targets.first().cloned().ok_or_else(|| {
- Error::internal(format!(
- "redirect path '{}' resolved no writable targets",
- path
- ))
- })?;
- Inner::write_first_target(inner, &path, &first_target, &ctx, backup_op.clone()).await?;
- let targets_clone = targets.clone();
- let first_target_name = first_target.name.clone();
- inner
- .meta_store
- .update_dir_meta(&dir, &ctx, move |redirect, sync_log| {
- redirect.entries.insert(
- name.clone(),
- RedirectEntry {
- targets: targets_clone.clone(),
- },
- );
- let mut committed = SyncLogEntry::committed(seq, sync_op);
- committed
- .backends
- .insert(first_target_name.clone(), BackendSyncState::acked(seq));
- sync_log.entries.insert(name, committed);
- Ok(())
- })
- .await?;
-
- let remaining_targets: Vec<FanoutTarget> =
- resolved_targets.into_iter().skip(1).collect();
- if !remaining_targets.is_empty() {
- inner.mark_pending_dir(&dir).await;
- Inner::fanout_async(inner, &path, remaining_targets, &ctx, backup_op).await;
- }
- inner.refresh_pending_dir(&dir, &ctx).await?;
- return Ok(size);
- }
-
- Inner::execute_write(
- inner,
- path,
- size,
- Some(sync_op),
- Some(backup_op),
- primary_fn,
- )
- .await
- }
-
- /// Fanout a write operation to all write-enabled backups.
- /// Takes `&Arc<Inner>` so spawned tasks can clone the Arc for acked_seq updates.
- /// `ctx` is required for encrypted backup backends and acked_seq updates.
- async fn fanout_write(
- inner: &Arc<Inner>,
- path: &str,
- size: u64,
- ctx: FsContext,
- op: BackupWriteOp,
- ) -> Result<()> {
- Inner::fanout_targets(inner, path, inner.write_targets(path, size), ctx, op).await
- }
-
- /// Fanout a write operation to explicitly named backup targets (used by redirect path).
- /// Resolves names to BackendEntry references, then delegates to sync/async state machine.
- async fn fanout_write_to_targets(
- inner: &Arc<Inner>,
- path: &str,
- target_names: &[String],
- ctx: FsContext,
- op: BackupWriteOp,
- ) -> Result<()> {
- Inner::fanout_targets(inner, path, inner.named_targets(target_names), ctx, op).await
- }
-
- /// Fanout to already resolved targets using the configured sync mode.
- async fn fanout_targets(
- inner: &Arc<Inner>,
- path: &str,
- targets: Vec<FanoutTarget>,
- ctx: FsContext,
- op: BackupWriteOp,
- ) -> Result<()> {
- if targets.is_empty() {
- if matches!(inner.sync_type, SyncType::Sync)
- && inner.write_ack_count > 0
- && inner.write_backups().next().is_some()
- {
- return Err(Error::SyncWriteQuorum {
- succeeded: 0,
- required: inner.write_ack_count,
- attempted: 0,
- failures: Vec::new(),
- });
- }
- return Ok(());
- }
-
- match inner.sync_type {
- SyncType::Sync => Inner::fanout_sync(inner, path, &targets, &ctx, op).await,
- SyncType::Async => {
- Inner::fanout_async(inner, path, targets, &ctx, op).await;
- Ok(())
- }
- }
- }
-
- /// Synchronous fanout: execute writes in parallel, wait for quorum.
- async fn fanout_sync(
- inner: &Arc<Inner>,
- path: &str,
- targets: &[FanoutTarget],
- ctx: &FsContext,
- op: BackupWriteOp,
- ) -> Result<()> {
- let ack_count = inner.write_ack_count.min(targets.len());
- let timeout = if inner.write_ack_timeout_ms > 0 {
- Some(Duration::from_millis(inner.write_ack_timeout_ms))
- } else {
- None
- };
-
- let path_owned = path.to_string();
- let ctx = Some(ctx.clone());
-
- // Launch parallel tasks for all backup writes.
- let mut handles = Vec::new();
- for target in targets {
- let fs = target.backend.clone();
- let name = target.name.clone();
- let path = path_owned.clone();
- let inner = Arc::clone(inner);
- let ctx = ctx.clone();
- let op_clone = op.clone();
- let queue_key = Self::backup_queue_key(&path, &name);
-
- handles.push(tokio::spawn(async move {
- // Wrap in FS_CTX.scope so encrypted backends can access account_id.
- let exec = inner.path_queues.with_path_lock(&queue_key, || async {
- op_clone
- .apply(&inner, fs, &path, ctx.as_ref().unwrap())
- .await
- });
-
- let result = if let Some(timeout) = timeout {
- match tokio::time::timeout(timeout, exec).await {
- Ok(Ok(())) => Ok(()),
- Ok(Err(e)) => Err(e),
- Err(_) => Err(Error::timeout(format!(
- "backup '{}' timed out waiting for sync acknowledgement",
- name
- ))),
- }
- } else {
- exec.await
+ let scope = crate::multibackend::model::ScopeKey {
+ account_id: account.clone(),
+ partition_id,
+ epoch: manifest.epoch,
};
-
- // Update acked_seq on success.
- if result.is_ok() {
- if let Some(ref ctx) = ctx {
- let _ = inner.update_backup_acked_seq(&path, &name, ctx).await;
- }
- }
-
- SyncFanoutTaskResult {
- backend: name,
- result,
- }
- }));
- }
-
- let results = futures::future::join_all(handles).await;
-
- let mut successes = 0usize;
- let mut failures = Vec::new();
-
- for result in results {
- match result {
- Ok(SyncFanoutTaskResult {
- backend: _,
- result: Ok(()),
- }) => {
- successes += 1;
- }
- Ok(SyncFanoutTaskResult {
- backend,
- result: Err(e),
- }) => {
- failures.push(super::errors::SyncWriteFailureDetail {
- backend,
- kind: e.kind_name().to_string(),
- message: e.to_string(),
- });
- }
- Err(e) => {
- failures.push(super::errors::SyncWriteFailureDetail {
- backend: "<spawn>".to_string(),
- kind: "join_error".to_string(),
- message: e.to_string(),
- });
+ let segments = self.metadata_provider.read_manifest(&scope).await?;
+ segments.validate()?;
+ let tail = segments.next_seq - 1;
+ for (backend, state) in segments.backend_states {
+ *lag.entry((backend, partition_id)).or_default() += tail - state.synced_seq;
}
}
}
-
- if successes >= ack_count {
- Ok(())
- } else {
- Err(Error::SyncWriteQuorum {
- succeeded: successes,
- required: ack_count,
- attempted: targets.len(),
- failures,
- })
- }
- }
-
- /// Asynchronous fanout: spawn background tasks that update acked_seq on completion.
- /// Uses per-path serialization to prevent out-of-order application on backup backends.
- async fn fanout_async(
- inner: &Arc<Inner>,
- path: &str,
- targets: Vec<FanoutTarget>,
- ctx: &FsContext,
- op: BackupWriteOp,
- ) {
- let path_owned = path.to_string();
- let sem = inner.write_sem.clone();
-
- for target in targets {
- let fs = target.backend.clone();
- let name = target.name.clone();
- let path = path_owned.clone();
- let ctx = ctx.clone();
- let sem = sem.clone();
- let inner = Arc::clone(inner);
- let op_clone = op.clone();
- let queue_key = Self::backup_queue_key(&path, &name);
- inner.background_task_started();
-
- tokio::spawn(async move {
- {
- // Per (path, backup) serialization preserves FIFO without blocking other backups.
- let result = inner
- .path_queues
- .with_path_lock(&queue_key, || async {
- let _permit = if let Some(ref sem) = sem {
- sem.acquire().await.ok()
- } else {
- None
- };
-
- op_clone.apply(&inner, fs, &path, &ctx).await
- })
- .await;
-
- // Update acked_seq on successful write.
- if result.is_ok() {
- let _ = inner.update_backup_acked_seq(&path, &name, &ctx).await;
- }
- }
- inner.background_task_finished();
+ let mut metrics = vec![
+ RagfsMetric {
+ name: "ragfs_multiwrite_background_tasks".into(),
+ labels: BTreeMap::new(),
+ value: RagfsMetricValue::Gauge(self.background_task_count() as f64),
+ },
+ RagfsMetric {
+ name: "ragfs_multiwrite_pending_events".into(),
+ labels: BTreeMap::new(),
+ value: RagfsMetricValue::Gauge(self.inner.runtime.pending_event_count() as f64),
+ },
+ ];
+ for (worker, label) in [
+ (MultiWriteWorker::Flush, "flush"),
+ (MultiWriteWorker::Checkpoint, "checkpoint"),
+ (MultiWriteWorker::CatchUp, "catch_up"),
+ (MultiWriteWorker::Gc, "gc"),
+ ] {
+ metrics.push(RagfsMetric::counter(
+ "ragfs_multiwrite_errors_total",
+ &[("worker", label)],
+ self.metadata_store.worker_error_count(worker),
+ 1.0,
+ ));
+ }
+ metrics.push(RagfsMetric {
+ name: "ragfs_multiwrite_protocol_version".into(),
+ labels: BTreeMap::from([("version".into(), "v2".into())]),
+ value: RagfsMetricValue::Gauge(u64::from(
+ protocol.protocol_version == ProtocolVersion::V2,
+ ) as f64),
+ });
+ for (status, value) in [
+ (
+ "migrating",
+ u64::from(protocol.status == ProtocolStatus::Migrating),
+ ),
+ (
+ "stable",
+ u64::from(protocol.status == ProtocolStatus::Stable),
+ ),
+ ] {
+ metrics.push(RagfsMetric {
+ name: "ragfs_multiwrite_protocol_status".into(),
+ labels: BTreeMap::from([("status".into(), status.into())]),
+ value: RagfsMetricValue::Gauge(value as f64),
});
}
+ metrics.extend(
+ lag.into_iter()
+ .map(|((backend, partition), value)| RagfsMetric {
+ name: "ragfs_multiwrite_backend_lag_events".into(),
+ labels: BTreeMap::from([
+ ("backend".into(), backend),
+ ("partition".into(), partition.to_string()),
+ ]),
+ value: RagfsMetricValue::Gauge(value as f64),
+ }),
+ );
+ Ok(metrics)
+ }
+
+ /// Resolve one account context and reject disagreement between path and task.
+ fn resolve_context(&self, path: &str) -> Result<FsContext> {
+ let path_ctx = self.inner.ctx_resolver.resolve(path)?;
+ match current_required_ctx() {
+ Ok(ctx) if !ctx.account_id().trim().is_empty() => {
+ if ctx.account_id() != path_ctx.account_id() {
+ return Err(Error::invalid_path(format!(
+ "path account '{}' conflicts with context account '{}'",
+ path_ctx.account_id(),
+ ctx.account_id()
+ )));
+ }
+ Ok(ctx)
+ }
+ _ => Ok(path_ctx),
+ }
}
- /// Record rename metadata and migrate redirect state when needed.
- async fn record_rename_meta(
+ /// Build one canonical V2 event from a foreground mount-relative path.
+ fn pending_event(
&self,
- old_path: &str,
- new_path: &str,
- ctx: &FsContext,
- ) -> Result<()> {
- let source_dir = parent_dir(old_path);
- let target_dir = parent_dir(new_path);
- let old_name = file_name(old_path).to_string();
- let new_name = file_name(new_path).to_string();
- let seq = self.next_seq().await?;
- let rename_op = SyncOp::Rename {
- to: new_path.to_string(),
- };
-
- if source_dir == target_dir {
- self.meta_store
- .update_dir_meta(&source_dir, ctx, move |redirect, sync_log| {
- sync_log
- .entries
- .insert(old_name.clone(), SyncLogEntry::committed(seq, rename_op));
- if let Some(redirect_entry) = redirect.entries.remove(&old_name) {
- redirect.entries.insert(new_name, redirect_entry);
- }
- Ok(())
- })
- .await
- } else {
- self.meta_store
- .update_dual_dir_meta(
- &source_dir,
- &target_dir,
- ctx,
- move |src_redirect, src_sync_log, tgt_redirect, _tgt_sync_log| {
- src_sync_log
- .entries
- .insert(old_name.clone(), SyncLogEntry::committed(seq, rename_op));
- if let Some(redirect_entry) = src_redirect.entries.remove(&old_name) {
- tgt_redirect.entries.insert(new_name, redirect_entry);
- }
- Ok(())
- },
- )
- .await
+ path: &str,
+ kind: SegmentEventType,
+ destination_path: Option<&str>,
+ ) -> Result<Option<PendingEvent>> {
+ let ctx = self.resolve_context(path)?;
+ let account_id = ctx.account_id().to_string();
+ let mut path = logical_path(&account_id, path)?;
+ let mut destination_path = destination_path
+ .map(|destination| logical_path(&account_id, destination))
+ .transpose()?;
+ let mut kind = kind;
+ let source_internal = is_multiwrite_internal_path(&path);
+ let destination_internal = destination_path
+ .as_deref()
+ .is_some_and(is_multiwrite_internal_path);
+ match (kind, source_internal, destination_internal) {
+ (SegmentEventType::MoveTree, true, false) => {
+ path = destination_path
+ .take()
+ .ok_or_else(|| Error::invalid_operation("move event requires a destination"))?;
+ kind = SegmentEventType::RemoveTree;
+ }
+ (SegmentEventType::MoveTree, false, true) => {
+ kind = SegmentEventType::RemoveTree;
+ destination_path = None;
+ }
+ (_, true, _) | (_, _, true) => return Ok(None),
+ _ => {}
}
+ Ok(Some(PendingEvent {
+ operation_id: Uuid::new_v4(),
+ account_id,
+ kind: PendingEventKind::Data(kind),
+ path,
+ destination_path,
+ route_hint: None,
+ }))
}
-}
-
-/// Return true when `path` falls under `exclude_path` (including itself).
-fn is_excluded_grep_path(path: &str, exclude_path: Option<&str>) -> bool {
- let Some(exclude_path) = exclude_path.map(normalize_prefix_path) else {
- return false;
- };
- let normalized_path = normalize_prefix_path(path);
- normalized_path == exclude_path
- || normalized_path
- .strip_prefix(&exclude_path)
- .is_some_and(|suffix| suffix.starts_with('/'))
-}
-
-// ── FileSystem trait implementation ──
-
-impl Drop for MultiWriteWrappedFS {
- /// Signal retry_loop to exit when the wrapper is unmounted or dropped.
- fn drop(&mut self) {
- self.inner.retry_cancelled.store(true, Ordering::SeqCst);
- self.inner.retry_shutdown.notify_waiters();
- }
-}
-impl MultiWriteWrappedFS {
- /// Stop background retry work and wait for in-flight async fanout to drain.
- pub async fn shutdown(&self) -> Result<()> {
- self.inner.retry_cancelled.store(true, Ordering::SeqCst);
- self.inner.retry_shutdown.notify_waiters();
- self.inner.wait_idle(DEFAULT_SHUTDOWN_WAIT).await
+ /// Submit one prebuilt event after its primary operation has succeeded.
+ fn submit_event(&self, event: Option<PendingEvent>) {
+ if let Some(event) = event {
+ self.inner.runtime.submit(event);
+ }
}
- /// Copy one primary-resident file to a new path while preserving physical primary bytes.
+ /// Copy raw primary bytes to another primary path and queue one write event.
pub async fn copy_within_primary(&self, src_path: &str, dst_path: &str) -> Result<bool> {
if normalize_prefix_path(src_path) == normalize_prefix_path(dst_path) {
return Ok(true);
}
-
- let ctx = current_required_ctx()?;
- let inner = &self.inner;
- let source_path = src_path.to_string();
- let destination_path = dst_path.to_string();
-
- let source_exists_on_primary = FS_CTX
- .scope(ctx.clone(), async {
- inner.primary().backend.exists(&source_path).await
- })
- .await;
- if !source_exists_on_primary {
+ let Some(primary_raw) = self.inner.primary.raw_backend.clone() else {
return Ok(false);
- }
-
- let source_size = FS_CTX
- .scope(ctx.clone(), async {
- let source_info = inner.primary().backend.stat(&source_path).await?;
- if source_info.is_dir {
- return Err(Error::IsADirectory(source_path.clone()));
- }
- Ok::<u64, Error>(source_info.size)
- })
- .await?;
-
- if inner
- .check_redirect(&destination_path, source_size)
- .is_some()
- {
+ };
+ if !primary_raw.exists(src_path).await {
return Ok(false);
}
-
- let Some(primary_raw_backend) = inner.primary().raw_backend.clone() else {
- return Ok(false);
- };
-
- let primary_destination = destination_path.clone();
- Inner::execute_write(
- inner,
- destination_path,
- source_size,
- Some(SyncOp::SyncFile { size: source_size }),
- Some(BackupWriteOp::Replay(SyncOp::SyncFile {
- size: source_size,
- })),
- move |_ignored_fs| {
- let primary_raw_backend = primary_raw_backend.clone();
- let primary_source = source_path.clone();
- let primary_destination = primary_destination.clone();
- let primary_ctx = ctx.clone();
- async move {
- copy_raw_primary_state(
- primary_raw_backend,
- &primary_source,
- &primary_destination,
- &primary_ctx,
- )
- .await?;
- Ok(())
- }
- },
- )
- .await?;
-
+ let ctx = self.resolve_context(src_path)?;
+ logical_path(ctx.account_id(), dst_path)?;
+ let event = self.pending_event(dst_path, SegmentEventType::Write, None)?;
+ copy_raw_primary_state(primary_raw, src_path, dst_path, &ctx).await?;
+ self.submit_event(event);
Ok(true)
}
+}
- /// Execute one non-redirecting write-like operation through the shared multi-write pipeline.
- async fn execute_simple_write<R, P, PFut>(
- &self,
- path: &str,
- size: u64,
- sync_op: Option<SyncOp>,
- primary_fn: P,
- ) -> Result<R>
- where
- R: Send + 'static,
- P: FnOnce(Arc<dyn FileSystem>, String) -> PFut + Send,
- PFut: Future<Output = Result<R>> + Send,
+/// Convert a wrapper path into one canonical account-owned logical path.
+fn logical_path(account_id: &str, path: &str) -> Result<String> {
+ let normalized = normalize_prefix_path(path);
+ let logical = if normalized.starts_with('/') {
+ format!("/local{normalized}")
+ } else {
+ format!("/local/{normalized}")
+ };
+ let account_root = format!("/local/{account_id}");
+ if logical != account_root
+ && !logical
+ .strip_prefix(&account_root)
+ .is_some_and(|suffix| suffix.starts_with('/'))
{
- let path_owned = path.to_string();
- let primary_path = path_owned.clone();
- let backup_op = sync_op.clone().map(BackupWriteOp::Replay);
- Inner::execute_write(
- &self.inner,
- path_owned,
- size,
- sync_op,
- backup_op,
- move |fs| primary_fn(fs, primary_path),
- )
+ return Err(Error::invalid_path(format!(
+ "path does not belong to account '{account_id}': {logical}"
+ )));
+ }
+ Ok(logical)
+}
+
+/// Copy one complete raw file in bounded chunks under the request context.
+async fn copy_raw_primary_state(
+ primary_raw: Arc<dyn FileSystem>,
+ source_path: &str,
+ destination_path: &str,
+ ctx: &FsContext,
+) -> Result<()> {
+ FS_CTX
+ .scope(ctx.clone(), async {
+ let source = primary_raw.stat(source_path).await?;
+ if source.is_dir {
+ return Err(Error::IsADirectory(source_path.to_string()));
+ }
+ primary_raw
+ .ensure_parent_dirs(destination_path, 0o755)
+ .await?;
+ if source.size == 0 {
+ if primary_raw.exists(destination_path).await {
+ return primary_raw.truncate(destination_path, 0).await;
+ }
+ return primary_raw.create(destination_path).await;
+ }
+ let mut offset = 0;
+ while offset < source.size {
+ let size = (source.size - offset).min(COPY_CHUNK_SIZE as u64);
+ let chunk = primary_raw.read(source_path, offset, size).await?;
+ if chunk.is_empty() {
+ return Err(Error::internal(format!(
+ "short read while copying raw primary file: {source_path}"
+ )));
+ }
+ let flag = if offset == 0 {
+ WriteFlag::Create
+ } else {
+ WriteFlag::None
+ };
+ primary_raw
+ .write(destination_path, &chunk, offset, flag)
+ .await?;
+ offset += chunk.len() as u64;
+ }
+ Ok(())
+ })
.await
+}
+
+/// Build the V2 event specs required to represent one completed rename.
+fn rename_event_specs<'a>(
+ source: &FileInfo,
+ old_path: &'a str,
+ new_path: &'a str,
+) -> Vec<(SegmentEventType, &'a str, Option<&'a str>)> {
+ if source.is_dir {
+ vec![(SegmentEventType::MoveTree, old_path, Some(new_path))]
+ } else {
+ vec![
+ (SegmentEventType::Write, new_path, None),
+ (SegmentEventType::Remove, old_path, None),
+ ]
}
}
#[async_trait]
impl FileSystem for MultiWriteWrappedFS {
+ /// Create a primary file and queue a write event after success.
async fn create(&self, path: &str) -> Result<()> {
- if is_hidden_internal_name(file_name(path)) {
- return self.inner.primary().backend.create(path).await;
- }
- self.execute_simple_write(path, 0, Some(SyncOp::Create), |fs, path| async move {
- fs.create(&path).await
- })
- .await
+ let event = self.pending_event(path, SegmentEventType::Write, None)?;
+ self.inner.primary.backend.create(path).await?;
+ self.submit_event(event);
+ Ok(())
}
+ /// Create a primary directory without producing a file-state event.
async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {
- self.execute_simple_write(
- path,
- 0,
- Some(SyncOp::Mkdir { mode }),
- move |fs, path| async move { fs.mkdir(&path, mode).await },
- )
- .await
+ self.inner.primary.backend.mkdir(path, mode).await
}
+ /// Remove a primary file and queue a remove event after success.
async fn remove(&self, path: &str) -> Result<()> {
- if is_hidden_internal_name(file_name(path)) {
- return self.inner.primary().backend.remove(path).await;
- }
- self.execute_simple_write(path, 0, Some(SyncOp::Remove), |fs, path| async move {
- fs.remove(&path).await
- })
- .await
+ let event = self.pending_event(path, SegmentEventType::Remove, None)?;
+ self.inner.primary.backend.remove(path).await?;
+ self.submit_event(event);
+ Ok(())
}
+ /// Remove a primary tree and queue a directory removal event after success.
async fn remove_all(&self, path: &str) -> Result<()> {
- if is_hidden_internal_name(file_name(path)) {
- return self.inner.primary().backend.remove_all(path).await;
+ let event = self
+ .pending_event(path, SegmentEventType::RemoveTree, None)?
+ .map(|mut event| {
+ if event.path == format!("/local/{}", event.account_id) {
+ event.kind = PendingEventKind::DeleteAccount;
+ }
+ event
+ });
+ let is_delete_account = event
+ .as_ref()
+ .is_some_and(|event| matches!(event.kind, PendingEventKind::DeleteAccount));
+ match self.inner.primary.backend.remove_all(path).await {
+ Ok(()) => {
+ self.submit_event(event);
+ Ok(())
+ }
+ Err(error @ Error::NotFound(_)) if is_delete_account => {
+ self.submit_event(event);
+ Err(error)
+ }
+ Err(error) => Err(error),
}
- self.execute_simple_write(path, 0, Some(SyncOp::RemoveAll), |fs, path| async move {
- fs.remove_all(&path).await
- })
- .await
}
+ /// Read bytes directly from the primary backend.
async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {
- if is_hidden_internal_name(file_name(path)) {
- return self.inner.primary().backend.read(path, offset, size).await;
- }
- if let Some(fs) = self.inner.resolve_read_backend(path).await {
- return fs.read(path, offset, size).await;
- }
- Err(Error::not_found(path))
+ self.inner.primary.backend.read(path, offset, size).await
}
+ /// Write primary bytes and queue a write event after success.
async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {
- if is_hidden_internal_name(file_name(path)) {
- return self
- .inner
- .primary()
- .backend
- .write(path, data, offset, flags)
- .await;
- }
- let inner = &self.inner;
- let data_len = data.len() as u64;
- let path_owned = path.to_string();
- let backup_op = BackupWriteOp::WriteFile {
- data: Arc::new(data.to_vec()),
- offset,
- flags,
- };
- let d = data.to_vec();
- let primary_path = path_owned.clone();
- Inner::execute_write_with_redirect(
- inner,
- path_owned,
- data_len,
- SyncOp::SyncFile { size: data_len },
- backup_op,
- move |fs: Arc<dyn FileSystem>| async move {
- fs.write(&primary_path, &d, offset, flags).await
- },
- )
- .await
+ let event = self.pending_event(path, SegmentEventType::Write, None)?;
+ let written = self
+ .inner
+ .primary
+ .backend
+ .write(path, data, offset, flags)
+ .await?;
+ self.submit_event(event);
+ Ok(written)
}
+ /// List one directory directly from the primary backend.
async fn read_dir(
&self,
path: &str,
@@ -1418,379 +594,89 @@ impl FileSystem for MultiWriteWrappedFS {
sort_by: Option<ListSortBy>,
sort_order: Option<SortOrder>,
) -> Result<Vec<FileInfo>> {
- let inner = &self.inner;
- let mut entries = inner
- .primary()
+ self.inner
+ .primary
.backend
- .read_dir(path, None, None, None, None)
- .await?;
-
- // Filter multi-write internal names.
- entries.retain(|e| !MULTIWRITE_INTERNAL_NAMES.contains(&e.name.as_str()));
-
- if inner.redirects.is_empty() {
- return Ok(apply_read_dir_options(
- entries, offset, limit, sort_by, sort_order,
- ));
- }
-
- // Merge redirect entries so users can see redirected files in listings.
- let ctx =
- current_required_ctx().or_else(|_| inner.meta_store.ctx_resolver().resolve(path))?;
-
- if let Ok(redirect_meta) = inner.meta_store.get_redirect_meta(path, &ctx).await {
- for (name, redirect_entry) in &redirect_meta.entries {
- if !entries.iter().any(|e| &e.name == name) {
- let virtual_path = if path == "/" {
- format!("/{}", name)
- } else {
- format!("{}/{}", path.trim_end_matches('/'), name)
- };
- entries.push(
- inner
- .redirect_file_info(&virtual_path, name, redirect_entry)
- .await,
- );
- }
- }
- }
+ .read_dir(path, offset, limit, sort_by, sort_order)
+ .await
+ }
- Ok(apply_read_dir_options(
- entries, offset, limit, sort_by, sort_order,
- ))
+ /// List internal entries directly from the primary backend.
+ async fn read_internal_dir(&self, path: &str) -> Result<Vec<FileInfo>> {
+ self.inner.primary.backend.read_internal_dir(path).await
}
+ /// Read metadata directly from the primary backend.
async fn stat(&self, path: &str) -> Result<FileInfo> {
- if is_hidden_internal_name(file_name(path)) {
- return self.inner.primary().backend.stat(path).await;
- }
- if let Some(fs) = self.inner.resolve_read_backend(path).await {
- return fs.stat(path).await;
- }
- Err(Error::not_found(path))
+ self.inner.primary.backend.stat(path).await
}
+ /// Rename a primary path and queue file or tree events after success.
async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {
- if is_hidden_internal_name(file_name(old_path))
- || is_hidden_internal_name(file_name(new_path))
- {
- return self
- .inner
- .primary()
- .backend
- .rename(old_path, new_path)
- .await;
- }
- let ctx = current_required_ctx()?;
- let inner = &self.inner;
- let old_owned = old_path.to_string();
- let new_owned = new_path.to_string();
- let source_dir = parent_dir(&old_owned);
- let redirect_targets = inner.redirect_targets_for_path(&old_owned, &ctx).await?;
- inner.invalidate_read_route(&old_owned).await;
- inner.invalidate_read_route(&new_owned).await;
-
- let primary_has_old = inner.primary().backend.exists(&old_owned).await;
- if primary_has_old {
- FS_CTX
- .scope(ctx.clone(), async {
- inner.primary().backend.rename(&old_owned, &new_owned).await
- })
- .await?;
- } else if redirect_targets.is_none() {
- return Err(Error::NotFound(old_owned));
- }
-
- inner
- .record_rename_meta(&old_owned, &new_owned, &ctx)
+ let old_ctx = self.resolve_context(old_path)?;
+ let new_ctx = self.resolve_context(new_path)?;
+ if old_ctx.account_id() != new_ctx.account_id() {
+ return Err(Error::invalid_path(format!(
+ "rename paths cross accounts: '{}' -> '{}'",
+ old_path, new_path
+ )));
+ }
+ let source = self.inner.primary.backend.stat(old_path).await?;
+ let events = rename_event_specs(&source, old_path, new_path)
+ .into_iter()
+ .map(|(kind, path, destination)| self.pending_event(path, kind, destination))
+ .collect::<Result<Vec<_>>>()?;
+ self.inner
+ .primary
+ .backend
+ .rename(old_path, new_path)
.await?;
- inner.mark_pending_dir(&source_dir).await;
-
- let fanout_path = old_owned.clone();
- let backup_op = BackupWriteOp::Replay(SyncOp::Rename {
- to: new_owned.clone(),
- });
- if let Some(targets) = redirect_targets {
- Inner::fanout_write_to_targets(inner, &fanout_path, &targets, ctx.clone(), backup_op)
- .await?;
- } else {
- Inner::fanout_write(inner, &fanout_path, 0, ctx.clone(), backup_op).await?;
+ for event in events {
+ self.submit_event(event);
}
- inner.refresh_pending_dir(&source_dir, &ctx).await?;
-
Ok(())
}
+ /// Change primary permissions and queue a write event after success.
async fn chmod(&self, path: &str, mode: u32) -> Result<()> {
- if is_hidden_internal_name(file_name(path)) {
- return self.inner.primary().backend.chmod(path, mode).await;
- }
- self.execute_simple_write(
- path,
- 0,
- Some(SyncOp::Chmod { mode }),
- move |fs, path| async move { fs.chmod(&path, mode).await },
- )
- .await
+ let event = self.pending_event(path, SegmentEventType::Write, None)?;
+ self.inner.primary.backend.chmod(path, mode).await?;
+ self.submit_event(event);
+ Ok(())
}
+ /// Truncate a primary file and queue a write event after success.
async fn truncate(&self, path: &str, size: u64) -> Result<()> {
- if is_hidden_internal_name(file_name(path)) {
- return self.inner.primary().backend.truncate(path, size).await;
- }
- self.execute_simple_write(
- path,
- size,
- Some(SyncOp::SyncFile { size }),
- move |fs, path| async move { fs.truncate(&path, size).await },
- )
- .await
+ let event = self.pending_event(path, SegmentEventType::Write, None)?;
+ self.inner.primary.backend.truncate(path, size).await?;
+ self.submit_event(event);
+ Ok(())
}
+ /// Create primary parent directories without producing a file-state event.
async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {
- let path_owned = path.to_string();
- let primary_path = path_owned.clone();
- Inner::execute_write(
- &self.inner,
- path_owned,
- 0,
- None,
- Some(BackupWriteOp::EnsureParentDirs { mode }),
- move |fs| async move { fs.ensure_parent_dirs(&primary_path, mode).await },
- )
- .await
+ self.inner
+ .primary
+ .backend
+ .ensure_parent_dirs(path, mode)
+ .await
}
+ /// Search content directly on the primary backend.
async fn grep(
&self,
path: &str,
pattern: &str,
options: GrepOptions<'_>,
) -> Result<GrepResult> {
- let inner = &self.inner;
- let path_owned = path.to_string();
- let pattern_owned = pattern.to_string();
- let exclude_owned = options.exclude_path.map(str::to_string);
- let options = GrepOptions {
- exclude_path: exclude_owned.as_deref(),
- ..options
- };
- let recursive = options.recursive;
- let node_limit = options.node_limit;
- let level_limit = options.level_limit;
-
- let mut result = inner
- .primary()
- .backend
- .grep(&path_owned, &pattern_owned, options)
- .await?;
-
- // Filter out multi-write internal metadata files from grep results.
- result.matches.retain(|m| {
- let file_name = m.file.rsplit('/').next().unwrap_or(&m.file);
- !MULTIWRITE_INTERNAL_NAMES.contains(&file_name)
- });
- result.count = result.matches.len();
-
- if inner.redirects.is_empty() {
- return Ok(result);
- }
-
- // For redirect files, also grep in target backends.
- let ctx = current_required_ctx()
- .or_else(|_| inner.meta_store.ctx_resolver().resolve(&path_owned))?;
-
- let search_dir = if inner
- .primary()
+ self.inner
+ .primary
.backend
- .stat(&path_owned)
+ .grep(path, pattern, options)
.await
- .map(|s| s.is_dir)
- .unwrap_or(false)
- {
- path_owned.clone()
- } else {
- parent_dir(&path_owned)
- };
-
- if recursive && search_dir == path_owned {
- let redirect_entries = self
- .tree_directory(
- &path_owned,
- true,
- None,
- level_limit,
- None,
- None,
- None,
- false,
- )
- .await?;
- for entry in redirect_entries {
- if node_limit.is_some_and(|limit| result.count >= limit) {
- break;
- }
- if entry.info.is_dir
- || entry.extra.get("redirect").and_then(Value::as_bool) != Some(true)
- {
- continue;
- }
- if is_excluded_grep_path(&entry.path, exclude_owned.as_deref()) {
- continue;
- }
- let Some(read_backend) = inner.resolve_read_backend(&entry.path).await else {
- continue;
- };
- let rel_path = relative_match_file(&path_owned, &entry.path);
- let target_result = match read_backend
- .grep(
- &entry.path,
- &pattern_owned,
- GrepOptions {
- recursive: false,
- node_limit: node_limit.map(|limit| limit.saturating_sub(result.count)),
- exclude_path: None,
- level_limit: None,
- ..options
- },
- )
- .await
- {
- Ok(found) => found,
- Err(_) => continue,
- };
- for mut m in target_result.matches {
- if node_limit.is_some_and(|limit| result.count >= limit) {
- break;
- }
- m.file = rel_path.clone();
- result.matches.push(m);
- result.count += 1;
- }
- }
- return Ok(result);
- }
-
- if let Ok(redirect_meta) = inner.meta_store.get_redirect_meta(&search_dir, &ctx).await {
- for (name, redirect_entry) in &redirect_meta.entries {
- for target_name in &redirect_entry.targets {
- if let Some(be) = inner.backup_by_name(target_name) {
- let redirect_path = if search_dir == "/" {
- format!("/{}", name)
- } else {
- format!("{}/{}", search_dir, name)
- };
- if let Ok(target_result) = be
- .backend
- .grep(
- &redirect_path,
- &pattern_owned,
- GrepOptions {
- recursive: false,
- exclude_path: None,
- level_limit: None,
- ..options
- },
- )
- .await
- {
- let rel_path = relative_match_file(&path_owned, &redirect_path);
- for mut m in target_result.matches {
- if node_limit.is_some_and(|limit| result.count >= limit) {
- break;
- }
- m.file = rel_path.clone();
- result.matches.push(m);
- result.count += 1;
- }
- }
- }
- }
- }
- }
-
- Ok(result)
- }
-
- async fn glob_directory(
- &self,
- path: &str,
- pattern: &str,
- show_hidden: bool,
- page_size: Option<usize>,
- level_limit: Option<usize>,
- continuation_token: Option<String>,
- ) -> Result<GlobPage> {
- if self.inner.redirects.is_empty() {
- return self
- .inner
- .primary()
- .backend
- .glob_directory(
- path,
- pattern,
- show_hidden,
- page_size,
- level_limit,
- continuation_token,
- )
- .await;
- }
-
- let matcher = PreparedGlob::new(pattern)?;
- if matches!(page_size, Some(0)) {
- return Err(Error::invalid_operation("page_size must be positive"));
- }
-
- let entries = self
- .tree_directory(
- path,
- show_hidden,
- None,
- level_limit,
- None,
- None,
- None,
- false,
- )
- .await?;
-
- let mut matched = Vec::new();
- for entry in entries {
- if matcher.is_match(&entry.rel_path) {
- matched.push(GlobEntry {
- path: entry.path,
- rel_path: entry.rel_path,
- name: entry.info.name,
- is_dir: entry.info.is_dir,
- });
- }
- }
- matched.sort_by(|left, right| compare_rel_paths(&left.rel_path, &right.rel_path));
-
- let start = decode_offset_token(
- continuation_token.as_deref(),
- path,
- pattern,
- show_hidden,
- level_limit,
- )?;
- if start > matched.len() {
- return Err(Error::invalid_operation("continuation token out of range"));
- }
- let end = page_size
- .map(|limit| start.saturating_add(limit))
- .unwrap_or(matched.len())
- .min(matched.len());
- let next_token = (end < matched.len())
- .then(|| encode_offset_token(end, path, pattern, show_hidden, level_limit));
-
- Ok(GlobPage {
- entries: matched[start..end].to_vec(),
- next_token,
- })
}
+ /// Traverse a tree directly on the primary backend.
async fn tree_directory(
&self,
path: &str,
@@ -1802,130 +688,43 @@ impl FileSystem for MultiWriteWrappedFS {
sort_order: Option<SortOrder>,
directories_only: bool,
) -> Result<Vec<TreeEntry>> {
- let base = normalize_prefix_path(path);
- if sort_by.is_some() {
- let mut entries = Vec::new();
- let traversal_limit = node_limit.map(|limit| offset.unwrap_or(0).saturating_add(limit));
- self.tree_directory_internal(
- &base,
- &base,
+ self.inner
+ .primary
+ .backend
+ .tree_directory(
+ path,
show_hidden,
- traversal_limit,
+ node_limit,
level_limit,
+ offset,
sort_by,
sort_order,
directories_only,
- &mut entries,
)
- .await?;
- return Ok(paginate_entries(entries, offset, node_limit));
- }
+ .await
+ }
- let mut entries = self
- .inner
- .primary()
+ /// Match paths directly on the primary backend.
+ async fn glob_directory(
+ &self,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ page_size: Option<usize>,
+ level_limit: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> Result<GlobPage> {
+ self.inner
+ .primary
.backend
- .tree_directory(
+ .glob_directory(
path,
+ pattern,
show_hidden,
- None,
+ page_size,
level_limit,
- None,
- None,
- None,
- directories_only,
+ continuation_token,
)
- .await?;
-
- entries.retain(|e| {
- let name = file_name(&e.path);
- !MULTIWRITE_INTERNAL_NAMES.contains(&name) && (!directories_only || e.info.is_dir)
- });
-
- if self.inner.redirects.is_empty() {
- return Ok(paginate_entries(entries, offset, node_limit));
- }
-
- let ctx = current_required_ctx()
- .or_else(|_| self.inner.meta_store.ctx_resolver().resolve(&base))?;
- let mut seen_paths: HashSet<String> = entries.iter().map(|e| e.path.clone()).collect();
- let mut dir_paths = Vec::new();
- let mut queued_dirs: HashSet<String> = HashSet::new();
- if !matches!(level_limit, Some(0)) {
- queued_dirs.insert(base.clone());
- dir_paths.push(base.clone());
- }
- for entry in &entries {
- if entry.info.is_dir {
- let dir = normalize_prefix_path(&entry.path);
- let may_emit_redirect_children = match level_limit {
- Some(limit) => {
- let rel = relative_match_file(&base, &dir);
- relative_depth(&rel) < limit
- }
- None => true,
- };
- if may_emit_redirect_children && queued_dirs.insert(dir.clone()) {
- dir_paths.push(dir);
- }
- }
- }
-
- let inner = Arc::clone(&self.inner);
- let mut redirect_results = stream::iter(dir_paths)
- .map(|dir| {
- let inner = Arc::clone(&inner);
- let ctx = ctx.clone();
- async move {
- let redirect_meta = inner.meta_store.get_redirect_meta(&dir, &ctx).await.ok();
- (dir, redirect_meta)
- }
- })
- .buffered(DEFAULT_TREE_REDIRECT_META_CONCURRENCY);
-
- while let Some((dir, redirect_meta)) = redirect_results.next().await {
- let Some(redirect_meta) = redirect_meta else {
- continue;
- };
- for (name, redirect_entry) in redirect_meta.entries {
- let virtual_path = if dir == "/" {
- format!("/{}", name)
- } else {
- format!("{}/{}", dir, name)
- };
- if seen_paths.contains(&virtual_path) {
- continue;
- }
- let rel_path = if base == "/" {
- virtual_path.trim_start_matches('/').to_string()
- } else {
- virtual_path
- .strip_prefix(&base)
- .unwrap_or(&virtual_path)
- .trim_start_matches('/')
- .to_string()
- };
- if level_limit.is_some_and(|limit| relative_depth(&rel_path) > limit) {
- continue;
- }
- let mut extra = HashMap::new();
- extra.insert("redirect".to_string(), Value::Bool(true));
- entries.push(TreeEntry {
- path: virtual_path.clone(),
- rel_path,
- info: self
- .inner
- .redirect_file_info(&virtual_path, &name, &redirect_entry)
- .await,
- extra,
- });
- seen_paths.insert(virtual_path);
- }
- }
-
- if directories_only {
- entries.retain(|entry| entry.info.is_dir);
- }
- Ok(paginate_entries(entries, offset, node_limit))
+ .await
}
}
diff --git a/crates/ragfs/src/core/multibackend_wrapper/retry.rs b/crates/ragfs/src/core/multibackend_wrapper/retry.rs
deleted file mode 100644
index c4581bf22..000000000
--- a/crates/ragfs/src/core/multibackend_wrapper/retry.rs
+++ /dev/null
@@ -1,270 +0,0 @@
-use std::time::{Duration, Instant};
-
-use rand::Rng;
-use tracing::warn;
-
-use super::*;
-
-impl Inner {
- /// Mark one directory as pending retry work.
- pub(super) async fn mark_pending_dir(&self, dir: &str) {
- self.pending_dirs.lock().await.insert(dir.to_string());
- }
-
- /// Remove one directory from the retry set.
- async fn clear_pending_dir(&self, dir: &str) {
- self.pending_dirs.lock().await.remove(dir);
- }
-
- /// Snapshot the current set of pending retry directories.
- pub(crate) async fn pending_dirs_snapshot(&self) -> Vec<String> {
- self.pending_dirs.lock().await.iter().cloned().collect()
- }
-
- /// Recompute whether one directory still has pending retry work.
- pub(super) async fn refresh_pending_dir(&self, dir: &str, ctx: &FsContext) -> Result<()> {
- if self.sync_work_in_dir(dir, ctx).await?.iter().any(|work| {
- work.entry.is_primary_committed()
- && work.targets.iter().any(|target| {
- !work.entry.is_in_sync(target) && !work.entry.is_quarantined(target)
- })
- }) {
- self.mark_pending_dir(dir).await;
- } else {
- self.clear_pending_dir(dir).await;
- }
- Ok(())
- }
-
- /// Collect sync work entries for one metadata directory.
- pub(crate) async fn sync_work_in_dir(
- &self,
- dir: &str,
- ctx: &FsContext,
- ) -> Result<Vec<SyncWorkEntry>> {
- let sync_log = self.meta_store.get_sync_log_meta(dir, ctx).await?;
- if sync_log.entries.is_empty() {
- return Ok(Vec::new());
- }
-
- let redirect_meta = self.meta_store.get_redirect_meta(dir, ctx).await?;
- Ok(sync_log
- .entries
- .into_iter()
- .map(|(file_name, entry)| {
- let file_path = if dir == "/" {
- format!("/{}", file_name)
- } else {
- format!("{}/{}", dir, file_name)
- };
- let targets =
- self.target_backend_names(&redirect_meta, &file_name, &file_path, &entry);
- SyncWorkEntry {
- file_path,
- entry,
- targets,
- }
- })
- .collect())
- }
-
- /// Record a started background task.
- pub(super) fn background_task_started(&self) {
- self.background_tasks.fetch_add(1, Ordering::SeqCst);
- }
-
- /// Record a finished background task and wake idle waiters if needed.
- pub(super) fn background_task_finished(&self) {
- if self.background_tasks.fetch_sub(1, Ordering::SeqCst) == 1 {
- self.idle_notify.notify_waiters();
- }
- }
-
- /// Wait for all background tasks to drain.
- pub(super) async fn wait_idle(&self, timeout: Duration) -> Result<()> {
- let deadline = Instant::now() + timeout;
- loop {
- if self.background_tasks.load(Ordering::SeqCst) == 0 {
- return Ok(());
- }
- let now = Instant::now();
- if now >= deadline {
- return Err(Error::timeout(
- "timed out while waiting multi-write background tasks to drain",
- ));
- }
- let wait = deadline.saturating_duration_since(now);
- tokio::time::timeout(wait, self.idle_notify.notified())
- .await
- .map_err(|_| {
- Error::timeout("timed out while waiting multi-write background tasks to drain")
- })?;
- }
- }
-
- /// Update the acked_seq for a backup in the sync log.
- pub(super) async fn update_backup_acked_seq(
- &self,
- path: &str,
- backup_name: &str,
- ctx: &FsContext,
- ) -> Result<()> {
- self.update_backend_state(path, backup_name, ctx, |state, latest_seq| {
- state.mark_acked(latest_seq);
- })
- .await?;
- self.refresh_pending_dir(&parent_dir(path), ctx).await
- }
-
- /// Record a replay failure and quarantine the target after repeated failures.
- pub(crate) async fn record_backup_retry_failure(
- &self,
- path: &str,
- backup_name: &str,
- ctx: &FsContext,
- ) -> Result<()> {
- self.update_backend_state(path, backup_name, ctx, |state, _latest_seq| {
- state.mark_retry_failed(self.quarantine_after_failures);
- })
- .await?;
- self.refresh_pending_dir(&parent_dir(path), ctx).await
- }
-
- /// Update per-backend sync state for a file entry.
- async fn update_backend_state<F>(
- &self,
- path: &str,
- backup_name: &str,
- ctx: &FsContext,
- update: F,
- ) -> Result<()>
- where
- F: FnOnce(&mut BackendSyncState, u64) + Send,
- {
- let dir = parent_dir(path);
- let backup_name = backup_name.to_string();
- let name = file_name(path).to_string();
- self.meta_store
- .update_dir_meta(&dir, ctx, move |_redirect, sync_log| {
- if let Some(entry) = sync_log.entries.get_mut(&name) {
- let state = entry.backends.entry(backup_name).or_default();
- update(state, entry.latest_seq);
- }
- Ok(())
- })
- .await
- }
-
- /// Replay a single operation on a lagging backup.
- pub(crate) async fn replay_operation(
- &self,
- file_path: &str,
- backup_name: &str,
- ctx: &FsContext,
- ) -> Result<()> {
- let queue_key = Self::backup_queue_key(file_path, backup_name);
- self.path_queues
- .with_path_lock(&queue_key, || async {
- let dir = parent_dir(file_path);
- let name = file_name(file_path).to_string();
- let sync_log = self.meta_store.get_sync_log_meta(&dir, ctx).await?;
- let entry = sync_log
- .entries
- .get(&name)
- .ok_or_else(|| Error::not_found(file_path))?
- .clone();
- let backup = match self.backup_by_name(backup_name) {
- Some(b) => b,
- None => {
- return Err(Error::internal(format!(
- "backup '{}' not found",
- backup_name
- )))
- }
- };
-
- entry
- .op
- .replay(
- self.primary().backend.clone(),
- backup.backend.clone(),
- file_path,
- ctx,
- )
- .await?;
-
- self.update_backup_acked_seq(file_path, backup_name, ctx)
- .await?;
-
- Ok(())
- })
- .await
- }
-
- /// Retry all pending backup work for one directory and keep metadata errors visible.
- pub(crate) async fn retry_pending_dir(&self, dir: &str) -> Result<()> {
- let ctx = self.meta_store.ctx_resolver().resolve(dir)?;
-
- for work in self.sync_work_in_dir(dir, &ctx).await? {
- if !work.entry.is_primary_committed() {
- continue;
- }
-
- for backup_name in &work.targets {
- if work.entry.is_in_sync(backup_name) || work.entry.is_quarantined(backup_name) {
- continue;
- }
-
- let mut success = false;
- for attempt in 0..self.max_retry_per_round {
- if self.retry_cancelled.load(Ordering::SeqCst) {
- return Ok(());
- }
- if self
- .replay_operation(&work.file_path, backup_name, &ctx)
- .await
- .is_ok()
- {
- success = true;
- break;
- }
- let base_ms = self.retry_backoff_base_ms.saturating_mul(1u64 << attempt);
- let jitter_ms = rand::thread_rng().gen_range(0..=50);
- tokio::time::sleep(Duration::from_millis(base_ms + jitter_ms)).await;
- }
-
- if !success {
- self.record_backup_retry_failure(&work.file_path, backup_name, &ctx)
- .await?;
- }
- }
- }
-
- self.refresh_pending_dir(dir, &ctx).await
- }
-
- /// Background retry loop: periodically scans sync_log for lagging backups and replays.
- pub(super) async fn retry_loop(inner: Arc<Inner>) {
- loop {
- if inner.retry_cancelled.load(Ordering::SeqCst) {
- break;
- }
-
- tokio::select! {
- _ = tokio::time::sleep(inner.retry_interval) => {}
- _ = inner.retry_shutdown.notified() => break,
- }
- let dirs = inner.pending_dirs_snapshot().await;
- for dir in dirs {
- if let Err(err) = inner.retry_pending_dir(&dir).await {
- warn!(
- dir = %dir,
- error = %err,
- "multi-write retry skipped one pending directory due to metadata or replay error"
- );
- }
- }
- }
- inner.background_task_finished();
- }
-}
diff --git a/crates/ragfs/src/core/multibackend_wrapper/routing.rs b/crates/ragfs/src/core/multibackend_wrapper/routing.rs
deleted file mode 100644
index 1dff3f491..000000000
--- a/crates/ragfs/src/core/multibackend_wrapper/routing.rs
+++ /dev/null
@@ -1,125 +0,0 @@
-use std::sync::Arc;
-
-use serde_json::{json, Value};
-
-use super::*;
-use crate::multibackend::meta::{current_required_ctx, file_name, parent_dir};
-
-impl Inner {
- /// Keep write-side call sites explicit even though read routing is uncached.
- pub(super) async fn invalidate_read_route(&self, _path: &str) {}
-
- /// Record read-route counters in one place so hot paths stay explicit.
- fn record_read_route(&self, source: ReadRouteSource) {
- match source {
- ReadRouteSource::Backup => {
- self.read_backup_hits.fetch_add(1, Ordering::Relaxed);
- }
- ReadRouteSource::Primary => {
- self.read_primary_hits.fetch_add(1, Ordering::Relaxed);
- }
- ReadRouteSource::Redirect => {
- self.read_redirect_hits.fetch_add(1, Ordering::Relaxed);
- }
- ReadRouteSource::Miss => {
- self.read_misses.fetch_add(1, Ordering::Relaxed);
- }
- }
- }
-
- /// Export read-route metrics for operational introspection.
- pub(crate) fn read_route_metrics(&self) -> Value {
- json!({
- "cache_hits": 0,
- "backup_hits": self.read_backup_hits.load(Ordering::Relaxed),
- "primary_hits": self.read_primary_hits.load(Ordering::Relaxed),
- "redirect_hits": self.read_redirect_hits.load(Ordering::Relaxed),
- "misses": self.read_misses.load(Ordering::Relaxed),
- })
- }
-
- /// Stat the first reachable redirect target and return user-visible metadata.
- pub(super) async fn redirect_file_info(
- &self,
- path: &str,
- name: &str,
- redirect_entry: &RedirectEntry,
- ) -> FileInfo {
- for target_name in &redirect_entry.targets {
- if let Some(be) = self.backup_by_name(target_name) {
- if let Ok(mut info) = be.backend.stat(path).await {
- info.name = name.to_string();
- return info;
- }
- }
- }
- FileInfo::new_file(name.to_string(), 0, 0o644)
- }
-
- /// Resolve the read backend for a path using the fallback chain.
- pub(super) async fn resolve_read_backend(&self, path: &str) -> Option<Arc<dyn FileSystem>> {
- let normalized = normalize_prefix_path(path);
- let read_backups = self.read_backups_sorted();
- let backup_exists = futures::future::join_all(read_backups.iter().map(|backup| async {
- (
- backup.name.clone(),
- backup.backend.clone(),
- backup.backend.exists(&normalized).await,
- )
- }))
- .await;
- for (_name, backend, exists) in backup_exists {
- if exists {
- self.record_read_route(ReadRouteSource::Backup);
- return Some(backend);
- }
- }
-
- if self.primary().backend.exists(&normalized).await {
- self.record_read_route(ReadRouteSource::Primary);
- return Some(self.primary().backend.clone());
- }
-
- if self.redirects.is_empty() {
- self.record_read_route(ReadRouteSource::Miss);
- return None;
- }
-
- let dir = parent_dir(&normalized);
- let name = file_name(&normalized).to_string();
- let ctx = current_required_ctx()
- .or_else(|_| self.meta_store.ctx_resolver().resolve(&dir))
- .ok()?;
- if let Ok(redirect_meta) = self.meta_store.get_redirect_meta(&dir, &ctx).await {
- if let Some(entry) = redirect_meta.entries.get(&name) {
- let redirect_targets: Vec<(String, Arc<dyn FileSystem>)> = entry
- .targets
- .iter()
- .filter_map(|target_name| {
- self.backup_by_name(target_name)
- .map(|be| (be.name.clone(), be.backend.clone()))
- })
- .collect();
- let redirect_exists = futures::future::join_all(redirect_targets.iter().map(
- |(target_name, backend)| async {
- (
- target_name.clone(),
- backend.clone(),
- backend.exists(&normalized).await,
- )
- },
- ))
- .await;
- for (_target_name, backend, exists) in redirect_exists {
- if exists {
- self.record_read_route(ReadRouteSource::Redirect);
- return Some(backend);
- }
- }
- }
- }
-
- self.record_read_route(ReadRouteSource::Miss);
- None
- }
-}
diff --git a/crates/ragfs/src/core/multibackend_wrapper/tests.rs b/crates/ragfs/src/core/multibackend_wrapper/tests.rs
deleted file mode 100644
index 1a6752d88..000000000
--- a/crates/ragfs/src/core/multibackend_wrapper/tests.rs
+++ /dev/null
@@ -1,939 +0,0 @@
-use std::sync::Arc;
-
-use serde_json::Value;
-
-use super::*;
-use crate::core::context::FsContextInner;
-use crate::plugins::memfs::MemFileSystem;
-
-/// Create the shared test context used by multi-write tests.
-fn test_ctx() -> FsContext {
- Arc::new(FsContextInner::new("acct".to_string()))
-}
-
-/// Create a sync multi-write filesystem with one memfs backup.
-fn test_multiwrite_fs(redirects: Vec<RedirectPolicy>) -> MultiWriteWrappedFS {
- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let builder = MultiWriteWrappedFS::builder(primary.clone())
- .with_primary_raw_backend(primary)
- .with_backups(vec![BackendEntry {
- name: "backup1".to_string(),
- role: BackendRole::Backup,
- backend: backup,
- raw_backend: None,
- operations: Vec::new(),
- excludes: Vec::new(),
- }])
- .sync_mode(SyncMode::Sync {
- ack_count: 1,
- timeout_ms: 0,
- });
-
- if redirects.is_empty() {
- builder
- } else {
- builder.with_redirects(redirects)
- }
- .build()
- .unwrap()
-}
-
-#[test]
-fn test_file_policy_over_size() {
- let policy = RedirectPolicy::FileOverSizePolicy {
- max_size_mb: 1,
- target: Some(vec!["backup1".to_string()]),
- };
- assert!(policy.matches("/a/big.bin", 2 * 1024 * 1024));
- assert!(!policy.matches("/a/small.txt", 512));
-}
-
-#[test]
-fn test_file_policy_extension() {
- let policy = RedirectPolicy::FileExtensionPolicy {
- extensions: vec!["(pdf|ppt)".to_string()],
- target: Some(vec!["backup1".to_string()]),
- };
- assert!(policy.matches("/a/doc.pdf", 0));
- assert!(policy.matches("/a/slides.ppt", 0));
- assert!(!policy.matches("/a/text.txt", 0));
-}
-
-#[tokio::test]
-async fn test_read_dir_redirect_entries_use_target_stat() {
- let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {
- extensions: vec!["\\.pdf$".to_string()],
- target: Some(vec!["backup1".to_string()]),
- }]);
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx, async {
- fs.ensure_parent_dirs("/local/acct/docs/report.pdf", 0o755)
- .await?;
- fs.write(
- "/local/acct/docs/report.pdf",
- b"pdf body",
- 0,
- WriteFlag::Create,
- )
- .await?;
-
- let entries = fs
- .read_dir("/local/acct/docs", None, None, None, None)
- .await?;
- let report = entries
- .iter()
- .find(|entry| entry.name == "report.pdf")
- .expect("redirected file should be visible in read_dir");
- assert_eq!(report.size, 8);
- assert_eq!(report.mode, 0o644);
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-
- let no_redirect_fs = test_multiwrite_fs(Vec::new());
- let no_redirect_ctx = test_ctx();
-
- FS_CTX
- .scope(no_redirect_ctx.clone(), async {
- no_redirect_fs
- .ensure_parent_dirs("/local/acct/docs/real.txt", 0o755)
- .await?;
- no_redirect_fs
- .write(
- "/local/acct/docs/real.txt",
- b"real body",
- 0,
- WriteFlag::Create,
- )
- .await?;
- no_redirect_fs
- .inner
- .meta_store
- .update_dir_meta(
- "/local/acct/docs",
- &no_redirect_ctx,
- |redirect, _sync_log| {
- redirect.entries.insert(
- "ghost.pdf".to_string(),
- RedirectEntry {
- targets: vec!["backup1".to_string()],
- },
- );
- Ok(())
- },
- )
- .await?;
- let backup = no_redirect_fs
- .inner
- .backup_by_name("backup1")
- .expect("backup exists")
- .backend
- .clone();
- backup
- .ensure_parent_dirs("/local/acct/docs/ghost.pdf", 0o755)
- .await?;
- backup
- .write(
- "/local/acct/docs/ghost.pdf",
- b"GHOST_MARKER\n",
- 0,
- WriteFlag::Create,
- )
- .await?;
-
- let entries = no_redirect_fs
- .read_dir("/local/acct/docs", None, None, None, None)
- .await?;
- let names: Vec<&str> = entries.iter().map(|entry| entry.name.as_str()).collect();
- assert_eq!(names, vec!["real.txt"]);
-
- let tree = no_redirect_fs
- .tree_directory(
- "/local/acct",
- false,
- None,
- Some(3),
- None,
- None,
- None,
- false,
- )
- .await?;
- let paths: Vec<&str> = tree.iter().map(|entry| entry.path.as_str()).collect();
- assert!(!paths.contains(&"/local/acct/docs/ghost.pdf"));
-
- let grep = no_redirect_fs
- .grep(
- "/local/acct/docs",
- "GHOST_MARKER",
- GrepOptions {
- level_limit: Some(1),
- ..Default::default()
- },
- )
- .await?;
- assert_eq!(grep.count, 0);
-
- let err = no_redirect_fs
- .rename("/local/acct/docs/ghost.pdf", "/local/acct/docs/renamed.pdf")
- .await
- .expect_err("stale redirect metadata is ignored when redirects are disabled");
- assert!(matches!(err, Error::NotFound(_)));
- assert!(backup.exists("/local/acct/docs/ghost.pdf").await);
- assert!(!backup.exists("/local/acct/docs/renamed.pdf").await);
-
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_recursive_grep_finds_nested_redirected_files() {
- let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {
- extensions: vec!["\\.md$".to_string()],
- target: Some(vec!["backup1".to_string()]),
- }]);
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx, async {
- fs.ensure_parent_dirs("/local/acct/resources/doc-1/page.md", 0o755)
- .await?;
- fs.write(
- "/local/acct/resources/doc-1/page.md",
- b"MULTIWRITE_TEST_MARKER\n",
- 0,
- WriteFlag::Create,
- )
- .await?;
-
- let shallow = fs
- .tree_directory(
- "/local/acct/resources",
- false,
- None,
- Some(1),
- None,
- None,
- None,
- false,
- )
- .await?;
- let shallow_paths: Vec<&str> =
- shallow.iter().map(|entry| entry.path.as_str()).collect();
- assert_eq!(shallow_paths, vec!["/local/acct/resources/doc-1"]);
-
- let root_only = fs
- .tree_directory(
- "/local/acct/resources",
- false,
- None,
- Some(0),
- None,
- None,
- None,
- false,
- )
- .await?;
- assert!(root_only.is_empty());
-
- let deep = fs
- .tree_directory(
- "/local/acct/resources",
- false,
- None,
- Some(2),
- None,
- None,
- None,
- false,
- )
- .await?;
- let deep_paths: Vec<&str> = deep.iter().map(|entry| entry.path.as_str()).collect();
- assert_eq!(
- deep_paths,
- vec![
- "/local/acct/resources/doc-1",
- "/local/acct/resources/doc-1/page.md",
- ]
- );
-
- let capped = fs
- .tree_directory(
- "/local/acct/resources",
- false,
- Some(1),
- Some(2),
- None,
- None,
- None,
- false,
- )
- .await?;
- let capped_paths: Vec<&str> = capped.iter().map(|entry| entry.path.as_str()).collect();
- assert_eq!(capped_paths, vec!["/local/acct/resources/doc-1"]);
-
- let result = fs
- .grep(
- "/local/acct/resources",
- "MULTIWRITE_TEST_MARKER",
- GrepOptions {
- recursive: true,
- level_limit: Some(5),
- ..Default::default()
- },
- )
- .await?;
- assert_eq!(result.count, 1);
- assert_eq!(result.matches.len(), 1);
- assert_eq!(result.matches[0].file, "doc-1/page.md");
- assert_eq!(result.matches[0].line, 1);
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_redirect_write_marks_first_target_acked_before_returning() {
- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup1: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup2: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup1_handle = backup1.clone();
- let fs = MultiWriteWrappedFS::builder(primary.clone())
- .with_primary_raw_backend(primary)
- .with_backups(vec![
- BackendEntry {
- name: "backup1".to_string(),
- role: BackendRole::Backup,
- backend: backup1,
- raw_backend: None,
- operations: Vec::new(),
- excludes: Vec::new(),
- },
- BackendEntry {
- name: "backup2".to_string(),
- role: BackendRole::Backup,
- backend: backup2,
- raw_backend: None,
- operations: Vec::new(),
- excludes: Vec::new(),
- },
- ])
- .with_redirects(vec![RedirectPolicy::FileExtensionPolicy {
- extensions: vec!["\\.pdf$".to_string()],
- target: Some(vec!["backup1".to_string(), "backup2".to_string()]),
- }])
- .sync_mode(SyncMode::Async)
- .build()
- .unwrap();
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx.clone(), async {
- fs.ensure_parent_dirs("/local/acct/docs/report.pdf", 0o755)
- .await?;
- fs.write(
- "/local/acct/docs/report.pdf",
- b"pdf body",
- 0,
- WriteFlag::Create,
- )
- .await?;
-
- let redirect_meta = fs
- .inner
- .meta_store
- .get_redirect_meta("/local/acct/docs", &ctx)
- .await?;
- let entry = redirect_meta
- .entries
- .get("report.pdf")
- .expect("redirect metadata should only appear after one target succeeds");
- assert_eq!(
- entry.targets,
- vec!["backup1".to_string(), "backup2".to_string()]
- );
-
- let sync_log = fs
- .inner
- .meta_store
- .get_sync_log_meta("/local/acct/docs", &ctx)
- .await?;
- let log_entry = sync_log
- .entries
- .get("report.pdf")
- .expect("redirected file should be tracked in sync log");
- assert!(log_entry.is_primary_committed());
- assert!(
- log_entry.is_in_sync("backup1"),
- "the first redirect target must be acknowledged before returning"
- );
- assert!(
- backup1_handle.exists("/local/acct/docs/report.pdf").await,
- "the first redirect target must already contain the file"
- );
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_redirect_write_does_not_publish_metadata_when_first_target_fails() {
- let fs = test_multiwrite_fs(vec![RedirectPolicy::FileExtensionPolicy {
- extensions: vec!["\\.pdf$".to_string()],
- target: Some(vec!["missing-backup".to_string()]),
- }]);
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx.clone(), async {
- let err = fs
- .write(
- "/local/acct/docs/fail.pdf",
- b"content",
- 0,
- WriteFlag::Create,
- )
- .await
- .expect_err("redirect write should fail when the first target is not durable");
-
- assert!(
- err.to_string().contains("resolved no writable targets"),
- "unexpected error: {err}"
- );
-
- let redirect_meta = fs
- .inner
- .meta_store
- .get_redirect_meta("/local/acct/docs", &ctx)
- .await?;
- assert!(
- !redirect_meta.entries.contains_key("fail.pdf"),
- "redirect metadata must not be published before the first target succeeds"
- );
-
- let sync_log = fs
- .inner
- .meta_store
- .get_sync_log_meta("/local/acct/docs", &ctx)
- .await?;
- assert!(
- !sync_log.entries.contains_key("fail.pdf"),
- "sync log must not record a committed redirect write before durability"
- );
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_cross_dir_rename_does_not_create_target_write_log() {
- let fs = test_multiwrite_fs(Vec::new());
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx.clone(), async {
- fs.ensure_parent_dirs("/local/acct/src/file.txt", 0o755)
- .await?;
- fs.ensure_parent_dirs("/local/acct/dst/file.txt", 0o755)
- .await?;
- fs.write(
- "/local/acct/src/file.txt",
- b"rename me",
- 0,
- WriteFlag::Create,
- )
- .await?;
- fs.rename("/local/acct/src/file.txt", "/local/acct/dst/file.txt")
- .await?;
-
- let target_log = fs
- .inner
- .meta_store
- .get_sync_log_meta("/local/acct/dst", &ctx)
- .await?;
- assert!(
- !target_log.entries.contains_key("file.txt"),
- "target directory must not record a fake write for cross-dir rename"
- );
- let source_log = fs
- .inner
- .meta_store
- .get_sync_log_meta("/local/acct/src", &ctx)
- .await?;
- let rename_entry = source_log
- .entries
- .get("file.txt")
- .expect("source directory should record rename command");
- assert!(matches!(
- &rename_entry.op,
- SyncOp::Rename { to } if to == "/local/acct/dst/file.txt"
- ));
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_rename_redirected_file_renames_original_redirect_target() {
- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup_handle = backup.clone();
- let fs = MultiWriteWrappedFS::builder(primary.clone())
- .with_primary_raw_backend(primary)
- .with_backups(vec![BackendEntry {
- name: "backup1".to_string(),
- role: BackendRole::Backup,
- backend: backup,
- raw_backend: None,
- operations: vec![OperationItemConfig {
- operation: "read".to_string(),
- priority: 1,
- }],
- excludes: Vec::new(),
- }])
- .with_redirects(vec![RedirectPolicy::FileExtensionPolicy {
- extensions: vec!["\\.pdf$".to_string()],
- target: Some(vec!["backup1".to_string()]),
- }])
- .sync_mode(SyncMode::Sync {
- ack_count: 1,
- timeout_ms: 0,
- })
- .build()
- .unwrap();
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx, async {
- fs.ensure_parent_dirs("/local/acct/src/a.pdf", 0o755)
- .await?;
- fs.ensure_parent_dirs("/local/acct/dst/b.pdf", 0o755)
- .await?;
- fs.write(
- "/local/acct/src/a.pdf",
- b"pdf payload",
- 0,
- WriteFlag::Create,
- )
- .await?;
- assert!(backup_handle.exists("/local/acct/src/a.pdf").await);
-
- fs.rename("/local/acct/src/a.pdf", "/local/acct/dst/b.pdf")
- .await?;
-
- assert!(
- backup_handle.exists("/local/acct/dst/b.pdf").await,
- "redirect target backend should receive rename fanout"
- );
- assert!(
- !backup_handle.exists("/local/acct/src/a.pdf").await,
- "redirect target backend should not retain the old path"
- );
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_copy_within_primary_preserves_primary_bytes_and_fanouts_backups() {
- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup_handle = backup.clone();
- let fs = MultiWriteWrappedFS::builder(primary.clone())
- .with_primary_raw_backend(primary)
- .with_backups(vec![BackendEntry {
- name: "backup1".to_string(),
- role: BackendRole::Backup,
- backend: backup,
- raw_backend: None,
- operations: Vec::new(),
- excludes: Vec::new(),
- }])
- .sync_mode(SyncMode::Sync {
- ack_count: 1,
- timeout_ms: 0,
- })
- .build()
- .unwrap();
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx.clone(), async {
- fs.ensure_parent_dirs("/local/acct/docs/src.bin", 0o755)
- .await?;
- fs.write("/local/acct/docs/src.bin", b"payload", 0, WriteFlag::Create)
- .await?;
-
- let copied = fs
- .copy_within_primary("/local/acct/docs/src.bin", "/local/acct/docs/dst.bin")
- .await?;
- assert!(
- copied,
- "copy_within_primary should use the primary fast-path"
- );
-
- assert_eq!(
- fs.inner
- .primary()
- .backend
- .read("/local/acct/docs/dst.bin", 0, 0)
- .await?,
- b"payload"
- );
- assert_eq!(
- backup_handle.read("/local/acct/docs/dst.bin", 0, 0).await?,
- b"payload"
- );
-
- let sync_log = fs
- .inner
- .meta_store
- .get_sync_log_meta("/local/acct/docs", &ctx)
- .await?;
- let entry = sync_log
- .entries
- .get("dst.bin")
- .expect("copied file should be tracked for backup fanout");
- assert!(entry.is_primary_committed());
- assert!(matches!(entry.op, SyncOp::SyncFile { size } if size == 7));
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_sync_log_write_entry_does_not_embed_large_payload() {
- let fs = test_multiwrite_fs(Vec::new());
- let ctx = test_ctx();
- let payload = vec![b'x'; 256 * 1024];
-
- FS_CTX
- .scope(ctx.clone(), async {
- fs.ensure_parent_dirs("/local/acct/docs/large.bin", 0o755)
- .await?;
- fs.write("/local/acct/docs/large.bin", &payload, 0, WriteFlag::Create)
- .await?;
-
- let sync_log = fs
- .inner
- .meta_store
- .get_sync_log_meta("/local/acct/docs", &ctx)
- .await?;
- let encoded = serde_json::to_vec(&sync_log)?;
- assert!(
- encoded.len() < 16 * 1024,
- "sync log should stay metadata-sized, got {} bytes",
- encoded.len()
- );
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_sync_log_truncate_entry_does_not_embed_snapshot() {
- let fs = test_multiwrite_fs(Vec::new());
- let ctx = test_ctx();
- let payload = vec![b'y'; 256 * 1024];
-
- FS_CTX
- .scope(ctx.clone(), async {
- fs.ensure_parent_dirs("/local/acct/docs/large.bin", 0o755)
- .await?;
- fs.write("/local/acct/docs/large.bin", &payload, 0, WriteFlag::Create)
- .await?;
- fs.truncate("/local/acct/docs/large.bin", 128).await?;
-
- let sync_log = fs
- .inner
- .meta_store
- .get_sync_log_meta("/local/acct/docs", &ctx)
- .await?;
- let encoded = serde_json::to_vec(&sync_log)?;
- assert!(
- encoded.len() < 16 * 1024,
- "truncate sync log should stay metadata-sized, got {} bytes",
- encoded.len()
- );
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_fanout_sync_returns_structured_quorum_error() {
- let fs = test_multiwrite_fs(Vec::new());
- let ctx = test_ctx();
- let targets = fs.inner.write_targets("/local/acct/docs/fail.txt", 0);
-
- let err = Inner::fanout_sync(
- &fs.inner,
- "/local/acct/docs/fail.txt",
- &targets,
- &ctx,
- BackupWriteOp::Replay(SyncOp::SyncFile { size: 1 }),
- )
- .await
- .expect_err("sync fanout should surface structured quorum failure");
-
- match err {
- Error::SyncWriteQuorum {
- succeeded,
- required,
- attempted,
- failures,
- } => {
- assert_eq!(succeeded, 0);
- assert_eq!(required, 1);
- assert_eq!(attempted, 1);
- assert_eq!(failures.len(), 1);
- assert_eq!(failures[0].backend, "backup1");
- assert_eq!(failures[0].kind, "not_found");
- assert_eq!(failures[0].message, "not found: /local/acct/docs/fail.txt");
- }
- other => panic!("unexpected error: {other}"),
- }
-}
-
-#[tokio::test]
-async fn test_sync_write_errors_when_required_ack_has_no_targets() {
- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let fs = MultiWriteWrappedFS::builder(primary.clone())
- .with_primary_raw_backend(primary)
- .with_backups(vec![BackendEntry {
- name: "backup1".to_string(),
- role: BackendRole::Backup,
- backend: backup,
- raw_backend: None,
- operations: Vec::new(),
- excludes: vec![RedirectPolicy::FileExtensionPolicy {
- extensions: vec!["\\.txt$".to_string()],
- target: None,
- }],
- }])
- .sync_mode(SyncMode::Sync {
- ack_count: 1,
- timeout_ms: 0,
- })
- .build()
- .unwrap();
- let ctx = test_ctx();
-
- let err = FS_CTX
- .scope(ctx, async {
- fs.ensure_parent_dirs("/local/acct/docs/no-target.txt", 0o755)
- .await?;
- fs.write(
- "/local/acct/docs/no-target.txt",
- b"payload",
- 0,
- WriteFlag::Create,
- )
- .await
- })
- .await
- .expect_err("sync mode must not silently pass when all backups are excluded");
-
- match err {
- Error::SyncWriteQuorum {
- succeeded,
- required,
- attempted,
- failures,
- } => {
- assert_eq!(succeeded, 0);
- assert_eq!(required, 1);
- assert_eq!(attempted, 0);
- assert!(failures.is_empty());
- }
- other => panic!("unexpected error: {other}"),
- }
-}
-
-#[tokio::test]
-async fn test_read_route_metrics_capture_backup_fallbacks() {
- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let backup_handle = backup.clone();
- let fs = MultiWriteWrappedFS::builder(primary)
- .with_backups(vec![BackendEntry {
- name: "backup1".to_string(),
- role: BackendRole::Backup,
- backend: backup,
- raw_backend: None,
- operations: vec![OperationItemConfig {
- operation: "read".to_string(),
- priority: 1,
- }],
- excludes: Vec::new(),
- }])
- .build()
- .unwrap();
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx, async {
- backup_handle
- .ensure_parent_dirs("/local/acct/hot/cache.txt", 0o755)
- .await?;
- backup_handle
- .write("/local/acct/hot/cache.txt", b"hot", 0, WriteFlag::Create)
- .await?;
-
- assert_eq!(fs.read("/local/acct/hot/cache.txt", 0, 0).await?, b"hot");
- assert_eq!(fs.read("/local/acct/hot/cache.txt", 0, 0).await?, b"hot");
-
- let metrics = fs.inner.read_route_metrics();
- assert_eq!(metrics.get("backup_hits").and_then(Value::as_u64), Some(2));
- assert_eq!(metrics.get("cache_hits").and_then(Value::as_u64), Some(0));
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_shutdown_drains_background_retry_loop() {
- let fs = test_multiwrite_fs(Vec::new());
- fs.shutdown().await.unwrap();
-}
-
-#[tokio::test]
-async fn test_system_sync_status_reports_corrupt_redirect_meta() {
- let fs = test_multiwrite_fs(Vec::new());
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx.clone(), async {
- fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)
- .await?;
- fs.write("/local/acct/docs/a.txt", b"hello", 0, WriteFlag::Create)
- .await?;
- fs.inner
- .primary()
- .backend
- .write(
- "/local/acct/docs/.redirect.json",
- b"{not-json",
- 0,
- WriteFlag::Create,
- )
- .await?;
-
- let err = fs
- .system_sync_status("/local/acct/docs/a.txt")
- .await
- .expect_err("corrupt redirect metadata must be reported");
- assert!(
- err.to_string().contains("redirect")
- || err.to_string().contains("json")
- || err.to_string().contains("serialization"),
- "unexpected error: {err}"
- );
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_retry_pending_dir_reports_corrupt_redirect_meta() {
- let fs = test_multiwrite_fs(Vec::new());
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx.clone(), async {
- fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)
- .await?;
- fs.write("/local/acct/docs/a.txt", b"hello", 0, WriteFlag::Create)
- .await?;
- fs.inner
- .meta_store
- .update_dir_meta("/local/acct/docs", &ctx, |_redirect, sync_log| {
- sync_log
- .entries
- .insert("a.txt".to_string(), SyncLogEntry::new(1, SyncOp::Create));
- Ok(())
- })
- .await?;
- fs.inner
- .primary()
- .backend
- .write(
- "/local/acct/docs/.redirect.json",
- b"{not-json",
- 0,
- WriteFlag::Create,
- )
- .await?;
-
- let err = fs
- .inner
- .retry_pending_dir("/local/acct/docs")
- .await
- .expect_err("corrupt redirect metadata must be reported to retry logic");
- assert!(
- err.to_string().contains("redirect")
- || err.to_string().contains("json")
- || err.to_string().contains("serialization"),
- "unexpected error: {err}"
- );
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
-
-#[tokio::test]
-async fn test_uncommitted_sync_log_entry_is_visible_but_not_retried() {
- let fs = test_multiwrite_fs(Vec::new());
- let ctx = test_ctx();
-
- FS_CTX
- .scope(ctx.clone(), async {
- fs.ensure_parent_dirs("/local/acct/docs/a.txt", 0o755)
- .await?;
- fs.inner
- .meta_store
- .update_dir_meta("/local/acct/docs", &ctx, |_redirect, sync_log| {
- sync_log
- .entries
- .insert("a.txt".to_string(), SyncLogEntry::new(1, SyncOp::Create));
- Ok(())
- })
- .await?;
-
- let status = fs.system_sync_status("/local/acct/docs").await?;
- assert_eq!(status["entry_count"], 1);
- assert_eq!(status["pending_target_count"], 0);
- assert_eq!(status["entries"][0]["primary_committed"], false);
- assert_eq!(
- status["entries"][0]["targets"][0]["primary_committed"],
- false
- );
-
- let retry = fs.system_sync_retry("/local/acct/docs").await?;
- assert_eq!(retry["retried"], 0);
- assert_eq!(retry["failed"], 0);
- assert_eq!(retry["skipped"], 1);
- assert_eq!(retry["results"][0]["status"], "awaiting_primary_commit");
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-}
diff --git a/crates/ragfs/src/core/types.rs b/crates/ragfs/src/core/types.rs
index fff95a76a..9038d4abf 100644
--- a/crates/ragfs/src/core/types.rs
+++ b/crates/ragfs/src/core/types.rs
@@ -397,10 +397,6 @@ pub struct PluginConfig {
/// Primary encryption enabled (follows global, not independently configurable)
#[serde(default)]
pub primary_encryption_enabled: bool,
-
- /// Primary redirect policies
- #[serde(default)]
- pub primary_redirects: Vec<RedirectPolicy>,
}
impl Default for PluginConfig {
@@ -413,7 +409,6 @@ impl Default for PluginConfig {
backups: None,
server_encryption_enabled: false,
primary_encryption_enabled: false,
- primary_redirects: Vec::new(),
}
}
}
@@ -444,8 +439,6 @@ impl PluginConfig {
Self::take_bool_with_default("server_encryption_enabled", &mut params, false)?;
let primary_encryption_enabled =
Self::take_bool_with_default("primary_encryption_enabled", &mut params, false)?;
- let primary_redirects =
- Self::take_json_with_default("primary_redirects", &mut params, Vec::new())?;
Ok(Self {
name: name.into(),
@@ -454,7 +447,6 @@ impl PluginConfig {
backups,
server_encryption_enabled,
primary_encryption_enabled,
- primary_redirects,
})
}
@@ -496,15 +488,6 @@ impl PluginConfig {
}
}
- /// Remove one optional JSON config value and fall back to the provided default.
- fn take_json_with_default<T: DeserializeOwned>(
- field_name: &str,
- params: &mut HashMap<String, ConfigValue>,
- default: T,
- ) -> crate::core::Result<T> {
- Ok(Self::take_optional_json(field_name, params)?.unwrap_or(default))
- }
-
/// Return a human-readable config value kind for error messages.
fn config_value_kind(value: &ConfigValue) -> &'static str {
match value {
@@ -576,32 +559,41 @@ impl ConfigValue {
/// Multi-write backends container configuration
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BackendsConfig {
- /// Sync type: "sync" or "async", default async
- #[serde(default = "default_sync_type")]
- pub sync_type: String,
- /// Minimum backup ack count for sync mode
- pub write_ack_count: Option<usize>,
- /// Timeout for waiting backup ack in sync mode (ms)
- pub write_ack_timeout_ms: Option<u64>,
- /// Max concurrent async writes
- pub write_concurrency: Option<usize>,
- /// Retry loop interval in milliseconds
- pub retry_interval_ms: Option<u64>,
- /// Base backoff in milliseconds for retry attempts
- pub retry_backoff_base_ms: Option<u64>,
- /// Maximum retry attempts per file/target in one round
- pub retry_max_retries_per_round: Option<usize>,
- /// Failure threshold before quarantining one file/target pair
- pub retry_quarantine_after_failures: Option<u32>,
- /// Deprecated no-op retained for config compatibility.
- #[serde(default)]
- pub read_probe_cache_ttl_ms: Option<u64>,
+ /// Cache key namespace used by cache-backed metadata.
+ #[serde(default = "default_backups_namespace")]
+ pub namespace: String,
+ /// Initial number of metadata partitions.
+ #[serde(default = "default_initial_partitions")]
+ pub initial_partitions: u32,
+ /// Checkpoint interval in seconds.
+ #[serde(default = "default_checkpoint_interval_secs")]
+ pub checkpoint_interval_secs: u64,
+ /// Metadata storage provider.
+ #[serde(default = "default_backups_provider")]
+ pub provider: String,
/// Backup items
+ #[serde(default)]
pub items: Vec<BackendItemConfig>,
}
-fn default_sync_type() -> String {
- "async".to_string()
+/// Return the default cache-backed metadata namespace.
+fn default_backups_namespace() -> String {
+ "default".to_string()
+}
+
+/// Return the default metadata partition count.
+fn default_initial_partitions() -> u32 {
+ 16
+}
+
+/// Return the default checkpoint interval in seconds.
+fn default_checkpoint_interval_secs() -> u64 {
+ 86_400
+}
+
+/// Return the default metadata provider.
+fn default_backups_provider() -> String {
+ "filesystem".to_string()
}
/// Single backup backend item configuration
@@ -614,23 +606,8 @@ pub struct BackendItemConfig {
/// Plugin-specific params (nested JSON)
#[serde(default)]
pub params: serde_json::Value,
- /// Timeout in seconds
- pub timeout: Option<u64>,
/// Encryption config for this backup
pub encryption: Option<EncryptionConfig>,
- /// Operations this backup participates in
- pub operations: Option<Vec<OperationItemConfig>>,
- /// Exclude policies
- pub excludes: Option<Vec<RedirectPolicy>>,
-}
-
-/// Per-operation priority config
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct OperationItemConfig {
- /// Operation type: "read" | "write"
- pub operation: String,
- /// Priority (smaller = higher priority)
- pub priority: u32,
}
/// Encryption on/off config for a backend
@@ -640,211 +617,6 @@ pub struct EncryptionConfig {
pub enabled: bool,
}
-/// Redirect / exclude policy (shared trait)
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(tag = "type")]
-pub enum RedirectPolicy {
- /// Redirect/exclude files exceeding a size threshold
- #[serde(rename = "FileOverSizePolicy")]
- FileOverSizePolicy {
- /// Max file size in MB
- max_size_mb: u64,
- /// Target backend names (redirect only)
- target: Option<Vec<String>>,
- },
- /// Redirect/exclude files matching extension patterns
- #[serde(rename = "FileExtensionPolicy")]
- FileExtensionPolicy {
- /// Regex patterns for file extensions
- extensions: Vec<String>,
- /// Target backend names (redirect only)
- target: Option<Vec<String>>,
- },
-}
-
-/// Strongly typed multi-write operation stored in `.sync_log.json`.
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-#[serde(tag = "type")]
-pub enum SyncOp {
- /// Create an empty file.
- Create,
- /// Synchronize file content by copying the current primary state to the backup.
- SyncFile {
- /// File size used by retry/exclude policy decisions.
- size: u64,
- },
- /// Create a directory.
- Mkdir {
- /// Directory mode.
- mode: u32,
- },
- /// Change file mode.
- Chmod {
- /// File mode.
- mode: u32,
- },
- /// Remove one file.
- Remove,
- /// Remove a file tree.
- RemoveAll,
- /// Rename to another path.
- Rename {
- /// Rename target path.
- to: String,
- },
-}
-
-/// Per-backend sync state
-#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
-pub struct BackendSyncState {
- /// Acknowledged sequence number
- pub acked_seq: u64,
- /// Consecutive retry failures for the current operation.
- #[serde(default)]
- pub retry_failures: u32,
- /// Whether this backend/path pair is quarantined for manual intervention.
- #[serde(default)]
- pub quarantined: bool,
-}
-
-impl BackendSyncState {
- /// Create an acknowledged backend sync state.
- pub fn acked(acked_seq: u64) -> Self {
- Self {
- acked_seq,
- retry_failures: 0,
- quarantined: false,
- }
- }
-
- /// Mark this backend as acknowledged and clear retry state.
- pub fn mark_acked(&mut self, acked_seq: u64) {
- self.acked_seq = acked_seq;
- self.retry_failures = 0;
- self.quarantined = false;
- }
-
- /// Record one retry failure and quarantine after the configured threshold.
- pub fn mark_retry_failed(&mut self, quarantine_after_failures: u32) {
- self.retry_failures = self.retry_failures.saturating_add(1);
- if self.retry_failures >= quarantine_after_failures {
- self.quarantined = true;
- }
- }
-}
-
-/// Sync log entry for a single file path.
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct SyncLogEntry {
- /// Latest monotonic sequence number.
- pub latest_seq: u64,
- /// Whether the primary backend has durably completed this operation.
- #[serde(default = "default_primary_committed")]
- pub primary_committed: bool,
- /// Strongly typed operation payload.
- pub op: SyncOp,
- /// Per-backend acked sequence numbers.
- pub backends: std::collections::HashMap<String, BackendSyncState>,
-}
-
-fn default_primary_committed() -> bool {
- true
-}
-
-impl SyncLogEntry {
- /// Create a new prepared sync log entry for a sequenced operation.
- pub fn new(latest_seq: u64, op: SyncOp) -> Self {
- Self {
- latest_seq,
- primary_committed: false,
- op,
- backends: std::collections::HashMap::new(),
- }
- }
-
- /// Create a committed sync log entry for an operation already applied on primary.
- pub fn committed(latest_seq: u64, op: SyncOp) -> Self {
- Self {
- latest_seq,
- primary_committed: true,
- op,
- backends: std::collections::HashMap::new(),
- }
- }
-
- /// Mark this entry as committed on primary.
- pub fn mark_primary_committed(&mut self) {
- self.primary_committed = true;
- }
-
- /// Return whether this entry is committed on primary.
- pub fn is_primary_committed(&self) -> bool {
- self.primary_committed
- }
-
- /// Return the backend sync state if present.
- pub fn backend_state(&self, backend_name: &str) -> Option<&BackendSyncState> {
- self.backends.get(backend_name)
- }
-
- /// Return the acknowledged sequence for a backend, or 0 if unknown.
- pub fn acked_seq(&self, backend_name: &str) -> u64 {
- self.backend_state(backend_name)
- .map(|state| state.acked_seq)
- .unwrap_or(0)
- }
-
- /// Return whether the backend is quarantined.
- pub fn is_quarantined(&self, backend_name: &str) -> bool {
- self.backend_state(backend_name)
- .map(|state| state.quarantined)
- .unwrap_or(false)
- }
-
- /// Return whether the backend has acknowledged the latest operation.
- pub fn is_in_sync(&self, backend_name: &str) -> bool {
- self.acked_seq(backend_name) >= self.latest_seq
- }
-}
-
-/// Sync log file content
-#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)]
-pub struct SyncLogMeta {
- /// File entries keyed by file name (current directory)
- pub entries: std::collections::HashMap<String, SyncLogEntry>,
-}
-
-/// Redirect metadata file content
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct RedirectMeta {
- /// Schema version
- #[serde(default = "default_redirect_version")]
- pub version: u32,
- /// Redirect entries keyed by file name
- #[serde(default)]
- pub entries: std::collections::HashMap<String, RedirectEntry>,
-}
-
-fn default_redirect_version() -> u32 {
- 1
-}
-
-impl Default for RedirectMeta {
- fn default() -> Self {
- Self {
- version: 1,
- entries: HashMap::new(),
- }
- }
-}
-
-/// Single redirect entry
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct RedirectEntry {
- /// Target backend names
- pub targets: Vec<String>,
-}
-
/// Backend role
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BackendRole {
@@ -854,15 +626,6 @@ pub enum BackendRole {
Backup,
}
-/// Sync type enum
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum SyncType {
- /// Synchronous: wait for backup ack before returning
- Sync,
- /// Asynchronous: return after primary write, sync in background
- Async,
-}
-
/// Custom serde module for SystemTime
mod systemtime_serde {
use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -922,21 +685,6 @@ mod tests {
assert_eq!(param.param_type, "string");
}
- #[test]
- fn test_sync_log_entry_legacy_payload_defaults_primary_committed_true() {
- let entry: SyncLogEntry = serde_json::from_value(serde_json::json!({
- "latest_seq": 7,
- "op": {
- "type": "Create"
- },
- "backends": {}
- }))
- .unwrap();
-
- assert!(entry.is_primary_committed());
- assert_eq!(entry.latest_seq, 7);
- }
-
#[test]
fn test_plugin_config_from_raw_parts_handles_multiwrite_shapes() {
let mut valid_params = HashMap::new();
@@ -958,22 +706,10 @@ mod tests {
"primary_encryption_enabled".to_string(),
ConfigValue::Bool(true),
);
- valid_params.insert(
- "primary_redirects".to_string(),
- ConfigValue::Json(serde_json::json!([
- {
- "type": "FileExtensionPolicy",
- "extensions": [".bin"],
- "target": ["backup1"]
- }
- ])),
- );
-
let config = PluginConfig::from_raw_parts("localfs", "/local", valid_params).unwrap();
assert!(config.backups.is_some());
assert!(config.server_encryption_enabled);
assert!(config.primary_encryption_enabled);
- assert_eq!(config.primary_redirects.len(), 1);
assert_eq!(
config.params.get("root_path"),
Some(&ConfigValue::String("/tmp/data".to_string()))
@@ -981,7 +717,6 @@ mod tests {
assert!(!config.params.contains_key("backups"));
assert!(!config.params.contains_key("server_encryption_enabled"));
assert!(!config.params.contains_key("primary_encryption_enabled"));
- assert!(!config.params.contains_key("primary_redirects"));
for (params, expected_message) in [
(
diff --git a/crates/ragfs/src/git/service.rs b/crates/ragfs/src/git/service.rs
index 896e7941b..0f1548ae6 100644
--- a/crates/ragfs/src/git/service.rs
+++ b/crates/ragfs/src/git/service.rs
@@ -6635,7 +6635,9 @@ mod fast_path1_tests {
)
.unwrap();
- let stack = build_default_stack(RagfsConfig::default()).await.unwrap();
+ let mut ragfs_config = RagfsConfig::default();
+ ragfs_config.pathlock.provider = "memory".to_string();
+ let stack = build_default_stack(ragfs_config).await.unwrap();
let mut params = HashMap::new();
params.insert(
"local_dir".to_string(),
@@ -6648,25 +6650,17 @@ mod fast_path1_tests {
mount_path: "/local".to_string(),
params,
backups: Some(BackendsConfig {
- sync_type: "async".to_string(),
- write_ack_count: None,
- write_ack_timeout_ms: None,
- write_concurrency: None,
- retry_interval_ms: None,
- retry_backoff_base_ms: None,
- retry_max_retries_per_round: None,
- retry_quarantine_after_failures: None,
- read_probe_cache_ttl_ms: None,
+ namespace: "default".to_string(),
+ initial_partitions: 16,
+ checkpoint_interval_secs: 86_400,
+ provider: "filesystem".to_string(),
items: vec![BackendItemConfig {
name: "backup1".to_string(),
backend: "localfs".to_string(),
params: serde_json::json!({
"local_dir": backup_dir.path().to_str().unwrap(),
}),
- timeout: None,
encryption: None,
- operations: None,
- excludes: None,
}],
}),
..PluginConfig::default()
diff --git a/crates/ragfs/src/lock/manager.rs b/crates/ragfs/src/lock/manager.rs
index 668263a61..cef2bdb7e 100644
--- a/crates/ragfs/src/lock/manager.rs
+++ b/crates/ragfs/src/lock/manager.rs
@@ -875,7 +875,7 @@ impl PathLockManager {
}
if waiting.0.is_none() {
- info!(owner_id = %owner_id, requests = ?sorted, timeout_ms = timeout.as_millis() as u64, error = %err, "pathlock acquire batch entered wait state");
+ debug!(owner_id = %owner_id, requests = ?sorted, timeout_ms = timeout.as_millis() as u64, error = %err, "pathlock acquire batch entered wait state");
waiting.0 = Some(&self.waiting_lock_count);
self.waiting_lock_count
.fetch_add(1, AtomicOrdering::Relaxed);
diff --git a/crates/ragfs/src/multibackend/admin.rs b/crates/ragfs/src/multibackend/admin.rs
deleted file mode 100644
index d0450a1b3..000000000
--- a/crates/ragfs/src/multibackend/admin.rs
+++ /dev/null
@@ -1,234 +0,0 @@
-use std::collections::HashSet;
-use std::time::Duration;
-
-use serde_json::{json, Value};
-
-use crate::core::errors::Result;
-use crate::core::filesystem::{normalize_prefix_path, FileSystem};
-use crate::core::multibackend_wrapper::{MultiWriteWrappedFS, SyncWorkEntry};
-use crate::multibackend::meta::{current_required_ctx, parent_dir};
-
-impl MultiWriteWrappedFS {
- /// Collect effective sync work entries under a path using the current request context.
- async fn collect_sync_work(&self, path: &str) -> Result<Vec<SyncWorkEntry>> {
- let ctx = current_required_ctx()?;
- let inner = &self.inner;
- let normalized = normalize_prefix_path(path);
- let path_info = <Self as FileSystem>::stat(self, &normalized).await?;
- let mut dirs = Vec::new();
- let mut seen_dirs = HashSet::new();
-
- let add_dir = |dirs: &mut Vec<String>, seen_dirs: &mut HashSet<String>, dir: String| {
- if seen_dirs.insert(dir.clone()) {
- dirs.push(dir);
- }
- };
-
- if path_info.is_dir {
- add_dir(&mut dirs, &mut seen_dirs, normalized.clone());
- for dir in inner.pending_dirs_snapshot().await {
- if dir == normalized
- || dir
- .strip_prefix(&normalized)
- .is_some_and(|suffix| suffix.starts_with('/'))
- {
- add_dir(&mut dirs, &mut seen_dirs, dir);
- }
- }
- if dirs.len() == 1 {
- for entry in inner
- .primary()
- .backend
- .tree_directory(&normalized, true, None, None, None, None, None, false)
- .await?
- {
- if entry.info.is_dir {
- add_dir(
- &mut dirs,
- &mut seen_dirs,
- normalize_prefix_path(&entry.path),
- );
- }
- }
- }
- } else {
- add_dir(
- &mut dirs,
- &mut seen_dirs,
- normalize_prefix_path(&parent_dir(&normalized)),
- );
- }
-
- let mut work = Vec::new();
- for dir in dirs {
- work.extend(
- inner
- .sync_work_in_dir(&dir, &ctx)
- .await?
- .into_iter()
- .filter(|work| path_info.is_dir || work.file_path == normalized),
- );
- }
-
- Ok(work)
- }
-
- /// Query effective multi-write sync status under a file or directory path.
- pub async fn system_sync_status(&self, path: &str) -> Result<Value> {
- let work = self.collect_sync_work(path).await?;
- let mut entries = Vec::new();
- let mut pending_target_count = 0usize;
-
- for work in work {
- let mut targets = Vec::new();
- let primary_committed = work.entry.is_primary_committed();
- let mut all_synced = primary_committed;
-
- for backend_name in work.targets {
- let acked_seq = if primary_committed {
- work.entry.acked_seq(&backend_name)
- } else {
- 0
- };
- let in_sync = primary_committed && work.entry.is_in_sync(&backend_name);
- if primary_committed && !in_sync {
- pending_target_count += 1;
- all_synced = false;
- }
- let state = work.entry.backend_state(&backend_name);
- targets.push(json!({
- "name": backend_name,
- "acked_seq": acked_seq,
- "retry_failures": state.map(|state| state.retry_failures).unwrap_or(0),
- "quarantined": state.map(|state| state.quarantined).unwrap_or(false),
- "primary_committed": primary_committed,
- "in_sync": in_sync,
- }));
- }
-
- entries.push(json!({
- "path": work.file_path,
- "latest_seq": work.entry.latest_seq,
- "primary_committed": primary_committed,
- "op": serde_json::to_value(&work.entry.op)?,
- "all_synced": all_synced,
- "targets": targets,
- }));
- }
-
- entries.sort_by(|a, b| {
- let ap = a.get("path").and_then(Value::as_str).unwrap_or_default();
- let bp = b.get("path").and_then(Value::as_str).unwrap_or_default();
- ap.cmp(bp)
- });
-
- Ok(json!({
- "path": normalize_prefix_path(path),
- "entry_count": entries.len(),
- "pending_target_count": pending_target_count,
- "capabilities": {
- "multi_instance_safe": false
- },
- "read_route_metrics": self.inner.read_route_metrics(),
- "entries": entries,
- }))
- }
-
- /// Manually retry lagging multi-write targets under a file or directory path.
- pub async fn system_sync_retry(&self, path: &str) -> Result<Value> {
- let ctx = current_required_ctx()?;
- let work = self.collect_sync_work(path).await?;
- let mut results = Vec::new();
- let mut retried = 0usize;
- let mut failed = 0usize;
- let mut skipped = 0usize;
-
- for work in work {
- if !work.entry.is_primary_committed() {
- skipped += work.targets.len();
- for backend_name in work.targets {
- results.push(json!({
- "path": work.file_path,
- "target": backend_name,
- "status": "awaiting_primary_commit",
- "latest_seq": work.entry.latest_seq,
- "primary_committed": false,
- "acked_seq": 0,
- }));
- }
- continue;
- }
-
- for backend_name in work.targets {
- let acked_seq = work.entry.acked_seq(&backend_name);
- let was_quarantined = work.entry.is_quarantined(&backend_name);
- if work.entry.is_in_sync(&backend_name) {
- skipped += 1;
- results.push(json!({
- "path": work.file_path,
- "target": backend_name,
- "status": "skipped",
- "latest_seq": work.entry.latest_seq,
- "acked_seq": acked_seq,
- }));
- continue;
- }
-
- let mut last_error = None;
- let mut success = false;
- for _attempt in 0..self.inner.max_retry_per_round {
- match self
- .inner
- .replay_operation(&work.file_path, &backend_name, &ctx)
- .await
- {
- Ok(()) => {
- success = true;
- break;
- }
- Err(err) => {
- last_error = Some(err.to_string());
- tokio::time::sleep(Duration::from_millis(
- self.inner.retry_backoff_base_ms,
- ))
- .await;
- }
- }
- }
-
- if success {
- retried += 1;
- results.push(json!({
- "path": work.file_path,
- "target": backend_name,
- "status": "retried",
- "latest_seq": work.entry.latest_seq,
- "acked_seq": work.entry.latest_seq,
- }));
- } else {
- self.inner
- .record_backup_retry_failure(&work.file_path, &backend_name, &ctx)
- .await?;
- failed += 1;
- results.push(json!({
- "path": work.file_path,
- "target": backend_name,
- "status": "failed",
- "latest_seq": work.entry.latest_seq,
- "acked_seq": acked_seq,
- "was_quarantined": was_quarantined,
- "error": last_error.unwrap_or_else(|| "unknown replay error".to_string()),
- }));
- }
- }
- }
-
- Ok(json!({
- "path": normalize_prefix_path(path),
- "retried": retried,
- "failed": failed,
- "skipped": skipped,
- "results": results,
- }))
- }
-}
diff --git a/crates/ragfs/src/multibackend/catch_up.rs b/crates/ragfs/src/multibackend/catch_up.rs
new file mode 100644
index 000000000..0ed190b74
--- /dev/null
+++ b/crates/ragfs/src/multibackend/catch_up.rs
@@ -0,0 +1,1058 @@
+//! Backup catch-up workers and current-state reconciliation primitives.
+
+use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::future::Future;
+use std::path::Path;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use std::time::Duration;
+
+use async_trait::async_trait;
+use rand::Rng;
+use serde_json::Value;
+use tokio::sync::watch;
+use tracing::{debug, warn};
+
+use crate::core::context::{FsContext, FsContextInner, FS_CTX};
+use crate::core::errors::{Error, Result};
+use crate::core::filesystem::FileSystem;
+use crate::core::internal_names::is_multiwrite_internal_path;
+use crate::core::types::{FileInfo, ListSortBy, SortOrder, WriteFlag};
+use crate::lock::OwnedPathLockLease;
+use crate::multibackend::constants::{MULTIWRITE_MOUNT_PREFIX, SYSTEM_DIR};
+use crate::multibackend::meta::{MetadataStore, MultiWriteWorker};
+use crate::multibackend::model::{
+ FileState, PartitionState, PartitionsManifest, ScopeKey, SegmentEventType, SegmentRecord,
+};
+use crate::multibackend::provider::MultiWriteProvider;
+
+#[cfg(test)]
+#[path = "tests/test_catch_up.rs"]
+mod tests;
+
+/// One configured backup handle owned by its catch-up worker.
+pub struct CatchUpTarget {
+ /// Stable backend id stored in partition progress.
+ pub backend_id: String,
+ /// Encrypted business filesystem handle.
+ pub backend: Arc<dyn FileSystem>,
+}
+
+/// Parsed checkpoint state accepted by CatchUpWorker from the future reader.
+#[derive(Clone)]
+pub struct CheckpointSnapshot {
+ /// Last sequence represented by this complete checkpoint.
+ pub checkpoint_to_seq: u64,
+ /// Materialized file states already decoded by the checkpoint reader.
+ pub file_states: Vec<FileState>,
+ /// Directory prefixes requiring current-state reconciliation.
+ pub directory_prefixes: Vec<String>,
+}
+
+/// Typed outcome of reading the latest checkpoint publication.
+pub enum CheckpointReadResult {
+ /// No published checkpoint fits the requested sequence window.
+ NoCheckpoint,
+ /// One complete validated checkpoint snapshot.
+ Snapshot(CheckpointSnapshot),
+ /// Immutable files changed during a reader/GC race; reread latest.
+ RetryLatestCheckpoint,
+}
+
+/// Consumer boundary implemented by the future checkpoint reader.
+#[async_trait]
+pub trait CheckpointConsumer: Send + Sync {
+ /// Return the latest usable parsed checkpoint inside the fixed tail.
+ async fn read_latest(
+ &self,
+ scope: &ScopeKey,
+ synced_seq: u64,
+ tail_seq: u64,
+ ) -> Result<CheckpointReadResult>;
+}
+
+struct EmptyCheckpointConsumer;
+
+#[async_trait]
+impl CheckpointConsumer for EmptyCheckpointConsumer {
+ /// Return no checkpoint until T-011 supplies a reader.
+ async fn read_latest(
+ &self,
+ _scope: &ScopeKey,
+ _synced_seq: u64,
+ _tail_seq: u64,
+ ) -> Result<CheckpointReadResult> {
+ Ok(CheckpointReadResult::NoCheckpoint)
+ }
+}
+
+/// One backup worker that consumes V2 metadata without blocking primary writes.
+pub struct CatchUpWorker {
+ backend_id: String,
+ primary: Arc<dyn FileSystem>,
+ backup: Arc<dyn FileSystem>,
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+ checkpoint_consumer: Arc<dyn CheckpointConsumer>,
+ cancellation: Option<watch::Receiver<bool>>,
+}
+
+/// One Exact PathLock lease protecting a backup catch-up worker.
+pub struct CatchUpLease {
+ store: Arc<MetadataStore>,
+ lease: OwnedPathLockLease,
+ valid: AtomicBool,
+}
+
+enum CatchUpFailure {
+ Cancelled,
+ Failed(Error),
+}
+
+impl From<Error> for CatchUpFailure {
+ /// Preserve ordinary failures while propagating explicit cancellation separately.
+ fn from(error: Error) -> Self {
+ Self::Failed(error)
+ }
+}
+
+type CatchUpResult<T> = std::result::Result<T, CatchUpFailure>;
+
+impl CatchUpLease {
+ /// Acquire the mount-level Exact lease for one logical backup id.
+ async fn acquire(store: Arc<MetadataStore>, backend_id: &str) -> Result<Self> {
+ let path =
+ format!("{MULTIWRITE_MOUNT_PREFIX}/{SYSTEM_DIR}/.multiwrite.catch-up.{backend_id}");
+ let manager = store.pathlock_manager();
+ let lease = manager
+ .acquire_exact(
+ &path,
+ manager.default_lock_timeout().max(Duration::from_secs(5)),
+ None,
+ )
+ .await?;
+ Ok(Self {
+ store,
+ lease,
+ valid: AtomicBool::new(true),
+ })
+ }
+
+ /// Refresh the lease and return false for every non-refreshed result.
+ async fn refresh(&self) -> Result<bool> {
+ let result = self.store.pathlock_manager().refresh(&self.lease).await;
+ let refreshed = matches!(result.as_deref(), Ok("refreshed"));
+ if !refreshed {
+ self.valid.store(false, Ordering::SeqCst);
+ }
+ result.map(|_| refreshed).map_err(Error::from)
+ }
+
+ /// Return whether every explicit refresh has retained the lease.
+ fn is_valid(&self) -> bool {
+ self.valid.load(Ordering::SeqCst)
+ }
+
+ /// Release this worker's Exact lease.
+ async fn release(&self) -> Result<()> {
+ self.store
+ .pathlock_manager()
+ .release(&self.lease)
+ .await
+ .map_err(Error::from)
+ }
+}
+
+impl CatchUpWorker {
+ /// Create one worker for an encrypted primary and backup handle.
+ pub fn new(
+ backend_id: String,
+ primary: Arc<dyn FileSystem>,
+ backup: Arc<dyn FileSystem>,
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+ ) -> Self {
+ Self {
+ backend_id,
+ primary,
+ backup,
+ store,
+ provider,
+ checkpoint_consumer: Arc::new(EmptyCheckpointConsumer),
+ cancellation: None,
+ }
+ }
+
+ /// Attach the parsed-checkpoint consumer supplied by T-011.
+ pub fn with_checkpoint_consumer(
+ mut self,
+ checkpoint_consumer: Arc<dyn CheckpointConsumer>,
+ ) -> Self {
+ self.checkpoint_consumer = checkpoint_consumer;
+ self
+ }
+
+ /// Attach a runtime cancellation receiver for interruptible catch-up rounds.
+ pub(crate) fn with_cancellation(mut self, cancellation: watch::Receiver<bool>) -> Self {
+ self.cancellation = Some(cancellation);
+ self
+ }
+
+ /// Acquire the backup lease, execute one bounded round, and release it.
+ pub async fn run_once(&self) -> Result<()> {
+ let outcome: CatchUpResult<()> = async {
+ let lease =
+ Arc::new(CatchUpLease::acquire(self.store.clone(), &self.backend_id).await?);
+ debug!(backend = %self.backend_id, "multi-write catch-up lease acquired");
+ let operation = self.run_round(&lease).await;
+ let release = lease.release().await;
+ match (operation, release) {
+ (Ok(()), Ok(())) => Ok(()),
+ (Err(CatchUpFailure::Cancelled), Ok(())) => Err(CatchUpFailure::Cancelled),
+ (Err(CatchUpFailure::Failed(error)), _) => Err(CatchUpFailure::Failed(error)),
+ (_, Err(error)) => Err(CatchUpFailure::Failed(error)),
+ }
+ }
+ .await;
+ match outcome {
+ Ok(()) => Ok(()),
+ Err(CatchUpFailure::Cancelled) => Err(Error::internal("catch-up was cancelled")),
+ Err(CatchUpFailure::Failed(error)) => {
+ self.store.record_worker_error(MultiWriteWorker::CatchUp);
+ Err(error)
+ }
+ }
+ }
+
+ /// Wait until runtime cancellation or remain pending for standalone workers.
+ async fn cancelled(&self) {
+ let Some(mut cancellation) = self.cancellation.clone() else {
+ std::future::pending::<()>().await;
+ return;
+ };
+ while !*cancellation.borrow() {
+ if cancellation.changed().await.is_err() {
+ return;
+ }
+ }
+ }
+
+ /// Reconcile registry membership and one fixed partition window per scope.
+ async fn run_round(&self, lease: &Arc<CatchUpLease>) -> CatchUpResult<()> {
+ let guarded = LeaseGuardedFileSystem {
+ inner: self.backup.clone(),
+ lease: lease.clone(),
+ };
+ let mut accounts = self.read_accounts().await?;
+ self.remove_extra_accounts(&guarded, &accounts).await?;
+ if !accounts.iter().any(|account| account == SYSTEM_DIR) {
+ accounts.insert(0, SYSTEM_DIR.to_string());
+ }
+
+ for account in accounts {
+ let path = self.store.paths().account_manifest(&account)?.0;
+ let manifest: PartitionsManifest = match self.store.read_json(&path).await {
+ Ok(manifest) => manifest,
+ Err(Error::NotFound(_)) => continue,
+ Err(error) => {
+ self.store.record_worker_error(MultiWriteWorker::CatchUp);
+ warn!(backend = %self.backend_id, account = %account, error = %error,
+ "multi-write catch-up account metadata unavailable");
+ continue;
+ }
+ };
+ if let Err(error) = manifest.validate() {
+ self.store.record_worker_error(MultiWriteWorker::CatchUp);
+ warn!(backend = %self.backend_id, account = %account, error = %error,
+ "multi-write catch-up account metadata invalid");
+ continue;
+ }
+ for (&partition_id, partition) in &manifest.partitions {
+ if partition.state != PartitionState::Stable {
+ continue;
+ }
+ let scope = ScopeKey {
+ account_id: account.clone(),
+ partition_id,
+ epoch: manifest.epoch,
+ };
+ if let Err(failure) = self.reconcile_scope(&guarded, lease, &scope).await {
+ let error = match failure {
+ CatchUpFailure::Cancelled => return Err(CatchUpFailure::Cancelled),
+ CatchUpFailure::Failed(Error::NotFound(_)) => continue,
+ CatchUpFailure::Failed(error) => error,
+ };
+ if !lease.is_valid() {
+ return Err(error.into());
+ }
+ self.store.record_worker_error(MultiWriteWorker::CatchUp);
+ warn!(
+ backend = %self.backend_id,
+ account = %account,
+ partition = partition_id,
+ error = %error,
+ "multi-write catch-up partition failed"
+ );
+ }
+ }
+ }
+ Ok(())
+ }
+
+ /// Read sorted account ids from the encrypted mount-level registry.
+ async fn read_accounts(&self) -> Result<Vec<String>> {
+ let bytes = self
+ .with_context(
+ SYSTEM_DIR,
+ self.primary.read("/_system/accounts.json", 0, 0),
+ )
+ .await?;
+ let value: Value = serde_json::from_slice(&bytes)?;
+ let accounts = value
+ .get("accounts")
+ .and_then(Value::as_object)
+ .ok_or_else(|| Error::Serialization("accounts.json lacks an accounts object".into()))?;
+ let mut ids = accounts.keys().cloned().collect::<Vec<_>>();
+ ids.sort();
+ Ok(ids)
+ }
+
+ /// Remove backup account roots that are absent from the primary registry.
+ async fn remove_extra_accounts(
+ &self,
+ backup: &LeaseGuardedFileSystem,
+ accounts: &[String],
+ ) -> Result<()> {
+ let expected = accounts.iter().map(String::as_str).collect::<HashSet<_>>();
+ let entries = self
+ .with_context(SYSTEM_DIR, backup.read_internal_dir("/"))
+ .await?;
+ for entry in entries {
+ if entry.is_dir && entry.name != SYSTEM_DIR && !expected.contains(entry.name.as_str()) {
+ let path = format!("/{}", entry.name);
+ self.with_context(&entry.name, backup.remove_all(&path))
+ .await?;
+ }
+ }
+ Ok(())
+ }
+
+ /// Consume one fixed `(synced_seq, tail]` window and advance after convergence.
+ async fn reconcile_scope(
+ &self,
+ backup: &LeaseGuardedFileSystem,
+ lease: &CatchUpLease,
+ scope: &ScopeKey,
+ ) -> CatchUpResult<()> {
+ let manifest = self.provider.read_manifest(scope).await?;
+ let state = manifest
+ .backend_states
+ .get(&self.backend_id)
+ .ok_or_else(|| {
+ Error::invalid_operation(format!("unknown backend id: {}", self.backend_id))
+ })?;
+ let tail = manifest.next_seq - 1;
+ debug!(
+ backend = %self.backend_id,
+ account = %scope.account_id,
+ partition = scope.partition_id,
+ synced_seq = state.synced_seq,
+ tail_seq = tail,
+ lag = tail - state.synced_seq,
+ "multi-write catch-up partition snapshot"
+ );
+ if state.synced_seq == tail {
+ return Ok(());
+ }
+ let records = self
+ .provider
+ .read_committed_head(scope, state.synced_seq + 1, tail + 1)
+ .await;
+ let records = match records {
+ Ok(records) if continuous_records(&records, state.synced_seq + 1, tail + 1) => {
+ Some(records)
+ }
+ _ => self
+ .provider
+ .read_committed_range(scope, state.synced_seq + 1, tail + 1)
+ .await
+ .ok()
+ .filter(|records| continuous_records(records, state.synced_seq + 1, tail + 1)),
+ };
+ let Some(records) = records else {
+ return self
+ .reconcile_checkpoint(backup, lease, scope, state.synced_seq, tail)
+ .await;
+ };
+ let progress = self
+ .reconcile_records(backup, lease, scope, state.synced_seq, tail, records)
+ .await?;
+ self.advance_progress(lease, scope, state.synced_seq, progress)
+ .await?;
+ Ok(())
+ }
+
+ /// Reconcile ordinary paths first, then delayed exact and prefix directory paths.
+ async fn reconcile_records(
+ &self,
+ backup: &LeaseGuardedFileSystem,
+ lease: &CatchUpLease,
+ scope: &ScopeKey,
+ synced_seq: u64,
+ tail: u64,
+ records: Vec<SegmentRecord>,
+ ) -> CatchUpResult<u64> {
+ let (mut ordinary, mut delayed_exact, mut prefixes, mut scope_deletes) = (
+ BTreeMap::new(),
+ BTreeMap::new(),
+ BTreeMap::new(),
+ BTreeMap::new(),
+ );
+ for record in records {
+ match record.event_type {
+ SegmentEventType::Write | SegmentEventType::Remove => {
+ let path = self
+ .store
+ .paths()
+ .backend_path(&scope.account_id, &record.path)?;
+ insert_earliest(&mut ordinary, path, record.seq);
+ }
+ SegmentEventType::RemoveTree => {
+ let path = self
+ .store
+ .paths()
+ .backend_path(&scope.account_id, &record.path)?;
+ insert_earliest(&mut prefixes, path, record.seq);
+ }
+ SegmentEventType::MoveTree => {
+ let source = self
+ .store
+ .paths()
+ .backend_path(&scope.account_id, &record.path)?;
+ let destination = self.store.paths().backend_path(
+ &scope.account_id,
+ record.destination_path.as_deref().ok_or_else(|| {
+ Error::Serialization("move marker lacks destination".to_string())
+ })?,
+ )?;
+ let (exact, dirty) = self
+ .with_context(
+ &scope.account_id,
+ classify_move_paths(backup, &source, &destination),
+ )
+ .await?;
+ for path in exact {
+ insert_earliest(&mut delayed_exact, path, record.seq);
+ }
+ for path in dirty {
+ insert_earliest(&mut prefixes, path, record.seq);
+ }
+ }
+ SegmentEventType::DeleteScope => {
+ if scope.account_id != SYSTEM_DIR {
+ return Err(Error::invalid_operation(
+ "DeleteScope is valid only in the system scope",
+ )
+ .into());
+ }
+ let (account_id, path) =
+ delete_scope_target(self.store.as_ref(), &record.path)?;
+ insert_earliest_scope_delete(&mut scope_deletes, path, account_id, record.seq);
+ }
+ }
+ }
+ let mut failed_from = None;
+ for (path, (account_id, seq)) in scope_deletes {
+ match self
+ .remove_scope_with_retry(backup, lease, &account_id, &path)
+ .await
+ {
+ Ok(()) => {
+ if let Err(error) = self
+ .reset_account_progress_after_scope_delete(lease, &account_id)
+ .await
+ {
+ if !lease.is_valid() {
+ return Err(error.into());
+ }
+ warn!(
+ backend = %self.backend_id,
+ account = %account_id,
+ error = %error,
+ "multi-write catch-up scope-delete progress reset failed"
+ );
+ failed_from = Some(failed_from.map_or(seq, |failed: u64| failed.min(seq)));
+ }
+ }
+ Err(CatchUpFailure::Cancelled) => return Err(CatchUpFailure::Cancelled),
+ Err(CatchUpFailure::Failed(_)) if !lease.is_valid() => {
+ return Err(Error::internal("catch-up lease was not refreshed").into());
+ }
+ Err(CatchUpFailure::Failed(_)) => {
+ failed_from = Some(failed_from.map_or(seq, |failed: u64| failed.min(seq)));
+ }
+ }
+ }
+ for (paths, prefix) in [(ordinary, false), (delayed_exact, false), (prefixes, true)] {
+ for (path, seq) in paths {
+ match self
+ .reconcile_with_retry(backup, lease, &scope.account_id, &path, prefix)
+ .await
+ {
+ Ok(()) => {}
+ Err(CatchUpFailure::Cancelled) => return Err(CatchUpFailure::Cancelled),
+ Err(CatchUpFailure::Failed(_)) if !lease.is_valid() => {
+ return Err(Error::internal("catch-up lease was not refreshed").into());
+ }
+ Err(CatchUpFailure::Failed(_)) => {
+ failed_from = Some(failed_from.map_or(seq, |failed: u64| failed.min(seq)));
+ }
+ }
+ }
+ }
+ Ok(failed_from.map_or(tail, |seq| seq.saturating_sub(1).max(synced_seq)))
+ }
+
+ /// Remove one backup scope root with the same bounded retry policy as path sync.
+ async fn remove_scope_with_retry(
+ &self,
+ backup: &LeaseGuardedFileSystem,
+ lease: &CatchUpLease,
+ account_id: &str,
+ path: &str,
+ ) -> CatchUpResult<()> {
+ let mut last_error = None;
+ for attempt in 0..=3 {
+ match self
+ .with_context(account_id, remove_if_present(backup, path))
+ .await
+ {
+ Ok(()) => return Ok(()),
+ Err(error) => last_error = Some(error),
+ }
+ if !lease.is_valid() || attempt == 3 {
+ break;
+ }
+ let jitter = rand::thread_rng().gen_range(0..=50);
+ let delay_ms = (1u64 << attempt) * 1_000 + jitter;
+ warn!(
+ backend = %self.backend_id,
+ account = %account_id,
+ path = %path,
+ retry = attempt + 1,
+ delay_ms,
+ error = %last_error.as_ref().unwrap(),
+ "multi-write catch-up scope-delete retry"
+ );
+ tokio::select! {
+ _ = tokio::time::sleep(Duration::from_millis(delay_ms)) => {}
+ _ = self.cancelled() => return Err(CatchUpFailure::Cancelled),
+ }
+ }
+ Err(CatchUpFailure::Failed(last_error.unwrap()))
+ }
+
+ /// Reset this backup's progress for a recreated account after scope delete.
+ async fn reset_account_progress_after_scope_delete(
+ &self,
+ lease: &CatchUpLease,
+ account_id: &str,
+ ) -> Result<()> {
+ let path = self.store.paths().account_manifest(account_id)?.0;
+ let manifest: PartitionsManifest = match self.store.read_json(&path).await {
+ Ok(manifest) => manifest,
+ Err(Error::NotFound(_)) => return Ok(()),
+ Err(error) => return Err(error),
+ };
+ manifest.validate()?;
+ if !lease.refresh().await? {
+ return Err(Error::internal("catch-up lease was not refreshed"));
+ }
+ for (&partition_id, _) in &manifest.partitions {
+ let scope = ScopeKey {
+ account_id: account_id.to_string(),
+ partition_id,
+ epoch: manifest.epoch,
+ };
+ self.provider
+ .advance_backend_state(&scope, &self.backend_id, 0)
+ .await?;
+ }
+ Ok(())
+ }
+
+ /// Refresh the lease immediately before a monotonic progress update.
+ async fn advance_progress(
+ &self,
+ lease: &CatchUpLease,
+ scope: &ScopeKey,
+ previous: u64,
+ progress: u64,
+ ) -> Result<()> {
+ if progress == previous {
+ return Ok(());
+ }
+ if !lease.refresh().await? {
+ return Err(Error::internal("catch-up lease was not refreshed"));
+ }
+ self.provider
+ .advance_backend_state(scope, &self.backend_id, progress)
+ .await
+ }
+
+ /// Reconcile one path with fixed exponential delays and bounded jitter.
+ async fn reconcile_with_retry(
+ &self,
+ backup: &LeaseGuardedFileSystem,
+ lease: &CatchUpLease,
+ account_id: &str,
+ path: &str,
+ prefix: bool,
+ ) -> CatchUpResult<()> {
+ let mut last_error = None;
+ for attempt in 0..=3 {
+ let operation = if prefix {
+ self.with_context(
+ account_id,
+ reconcile_prefix(self.primary.as_ref(), backup, path),
+ )
+ .await
+ } else {
+ self.with_context(
+ account_id,
+ reconcile_file(self.primary.as_ref(), backup, path),
+ )
+ .await
+ };
+ match operation {
+ Ok(()) => return Ok(()),
+ Err(error) => last_error = Some(error),
+ }
+ if !lease.is_valid() || attempt == 3 {
+ break;
+ }
+ let jitter = rand::thread_rng().gen_range(0..=50);
+ let delay_ms = (1u64 << attempt) * 1_000 + jitter;
+ warn!(
+ backend = %self.backend_id,
+ account = %account_id,
+ path = %path,
+ retry = attempt + 1,
+ delay_ms,
+ error = %last_error.as_ref().unwrap(),
+ "multi-write catch-up path retry"
+ );
+ tokio::select! {
+ _ = tokio::time::sleep(Duration::from_millis(delay_ms)) => {}
+ _ = self.cancelled() => return Err(CatchUpFailure::Cancelled),
+ }
+ }
+ Err(CatchUpFailure::Failed(last_error.unwrap()))
+ }
+
+ /// Consume one already parsed checkpoint and advance only after full convergence.
+ async fn reconcile_checkpoint(
+ &self,
+ backup: &LeaseGuardedFileSystem,
+ lease: &CatchUpLease,
+ scope: &ScopeKey,
+ synced_seq: u64,
+ tail: u64,
+ ) -> CatchUpResult<()> {
+ let snapshot = match self
+ .checkpoint_consumer
+ .read_latest(scope, synced_seq, tail)
+ .await?
+ {
+ CheckpointReadResult::Snapshot(snapshot) => snapshot,
+ CheckpointReadResult::NoCheckpoint => {
+ return Err(Error::invalid_operation("segment window is not continuous").into());
+ }
+ CheckpointReadResult::RetryLatestCheckpoint => {
+ return Err(Error::invalid_operation("retry latest checkpoint").into());
+ }
+ };
+ if snapshot.checkpoint_to_seq <= synced_seq || snapshot.checkpoint_to_seq > tail {
+ return Err(Error::Serialization(
+ "checkpoint sequence is outside the fixed catch-up window".to_string(),
+ )
+ .into());
+ }
+ let mut paths = BTreeSet::new();
+ for state in snapshot.file_states {
+ state.validate(scope)?;
+ if state.latest_seq.seq > snapshot.checkpoint_to_seq {
+ return Err(Error::Serialization(
+ "checkpoint file state exceeds checkpoint sequence".to_string(),
+ )
+ .into());
+ }
+ paths.insert(
+ self.store
+ .paths()
+ .backend_path(&scope.account_id, &state.path)?,
+ );
+ }
+ for path in paths {
+ self.reconcile_with_retry(backup, lease, &scope.account_id, &path, false)
+ .await?;
+ }
+ let mut prefixes = BTreeSet::new();
+ for prefix in snapshot.directory_prefixes {
+ prefixes.insert(
+ self.store
+ .paths()
+ .backend_path(&scope.account_id, &prefix)?,
+ );
+ }
+ for prefix in prefixes {
+ self.reconcile_with_retry(backup, lease, &scope.account_id, &prefix, true)
+ .await?;
+ }
+ self.advance_progress(lease, scope, synced_seq, snapshot.checkpoint_to_seq)
+ .await?;
+ if snapshot.checkpoint_to_seq == tail {
+ return Ok(());
+ }
+ let records = self
+ .provider
+ .read_committed_range(scope, snapshot.checkpoint_to_seq + 1, tail + 1)
+ .await?;
+ if !continuous_records(&records, snapshot.checkpoint_to_seq + 1, tail + 1) {
+ return Err(Error::invalid_operation(
+ "post-checkpoint segment window is not continuous",
+ )
+ .into());
+ }
+ let progress = self
+ .reconcile_records(
+ backup,
+ lease,
+ scope,
+ snapshot.checkpoint_to_seq,
+ tail,
+ records,
+ )
+ .await?;
+ self.advance_progress(lease, scope, snapshot.checkpoint_to_seq, progress)
+ .await?;
+ Ok(())
+ }
+
+ /// Run one filesystem future with cache bypass and automatic PathLock disabled.
+ async fn with_context<T>(
+ &self,
+ account_id: &str,
+ future: impl Future<Output = Result<T>>,
+ ) -> Result<T> {
+ let context: FsContext = Arc::new(
+ FsContextInner::new(account_id)
+ .with_bypass_cache(true)
+ .with_auto_pathlock_disabled(),
+ );
+ FS_CTX.scope(context, future).await
+ }
+}
+
+/// Insert one path while retaining its earliest dependent sequence.
+fn insert_earliest(paths: &mut BTreeMap<String, u64>, path: String, seq: u64) {
+ paths
+ .entry(path)
+ .and_modify(|current| *current = (*current).min(seq))
+ .or_insert(seq);
+}
+
+/// Insert one scope delete while retaining its earliest dependent sequence.
+fn insert_earliest_scope_delete(
+ paths: &mut BTreeMap<String, (String, u64)>,
+ path: String,
+ account_id: String,
+ seq: u64,
+) {
+ paths
+ .entry(path)
+ .and_modify(|(_, current)| *current = (*current).min(seq))
+ .or_insert((account_id, seq));
+}
+
+/// Convert a logical `/local/{scope}` deletion target into backup path and context.
+fn delete_scope_target(store: &MetadataStore, logical_path: &str) -> Result<(String, String)> {
+ let path = store.paths().raw_backend_path(logical_path)?;
+ let account_id = path
+ .trim_start_matches('/')
+ .split('/')
+ .next()
+ .filter(|value| !value.is_empty())
+ .ok_or_else(|| Error::invalid_path(format!("invalid DeleteScope path: {logical_path}")))?;
+ Ok((account_id.to_string(), path))
+}
+
+struct LeaseGuardedFileSystem {
+ inner: Arc<dyn FileSystem>,
+ lease: Arc<CatchUpLease>,
+}
+
+impl LeaseGuardedFileSystem {
+ /// Reject the next backup mutation unless the lease refreshes literally.
+ async fn before_write(&self) -> Result<()> {
+ if self.lease.refresh().await? {
+ Ok(())
+ } else {
+ Err(Error::internal("catch-up lease was not refreshed"))
+ }
+ }
+}
+
+#[async_trait]
+impl FileSystem for LeaseGuardedFileSystem {
+ /// Refresh the catch-up lease before creating a backup file.
+ async fn create(&self, path: &str) -> Result<()> {
+ self.before_write().await?;
+ self.inner.create(path).await
+ }
+
+ /// Refresh the catch-up lease before creating a backup directory.
+ async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {
+ self.before_write().await?;
+ self.inner.mkdir(path, mode).await
+ }
+
+ /// Refresh the catch-up lease before removing a backup file.
+ async fn remove(&self, path: &str) -> Result<()> {
+ self.before_write().await?;
+ self.inner.remove(path).await
+ }
+
+ /// Refresh the catch-up lease before removing a backup subtree.
+ async fn remove_all(&self, path: &str) -> Result<()> {
+ self.before_write().await?;
+ self.inner.remove_all(path).await
+ }
+
+ /// Read bytes from the guarded backup without mutating it.
+ async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {
+ self.inner.read(path, offset, size).await
+ }
+
+ /// Refresh the catch-up lease before writing backup bytes.
+ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {
+ self.before_write().await?;
+ self.inner.write(path, data, offset, flags).await
+ }
+
+ /// List guarded backup entries without mutating them.
+ async fn read_dir(
+ &self,
+ path: &str,
+ offset: Option<usize>,
+ limit: Option<usize>,
+ sort_by: Option<ListSortBy>,
+ sort_order: Option<SortOrder>,
+ ) -> Result<Vec<FileInfo>> {
+ self.inner
+ .read_dir(path, offset, limit, sort_by, sort_order)
+ .await
+ }
+
+ /// Read guarded backup metadata without mutating it.
+ async fn stat(&self, path: &str) -> Result<FileInfo> {
+ self.inner.stat(path).await
+ }
+
+ /// Refresh the catch-up lease before renaming a backup path.
+ async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {
+ self.before_write().await?;
+ self.inner.rename(old_path, new_path).await
+ }
+
+ /// Refresh the catch-up lease before changing a backup mode.
+ async fn chmod(&self, path: &str, mode: u32) -> Result<()> {
+ self.before_write().await?;
+ self.inner.chmod(path, mode).await
+ }
+}
+
+/// Return whether records exactly cover one requested half-open sequence range.
+fn continuous_records(records: &[SegmentRecord], from: u64, to: u64) -> bool {
+ records.len() as u64 == to - from
+ && records
+ .iter()
+ .enumerate()
+ .all(|(index, record)| record.seq == from + index as u64)
+}
+
+/// Classify move source and destination by the backup's current source type.
+pub async fn classify_move_paths(
+ backup: &dyn FileSystem,
+ source: &str,
+ destination: &str,
+) -> Result<(HashSet<String>, HashSet<String>)> {
+ let paths = HashSet::from([source.to_string(), destination.to_string()]);
+ match backup.stat(source).await {
+ Ok(info) if !info.is_dir => Ok((paths, HashSet::new())),
+ Ok(_) | Err(Error::NotFound(_)) => Ok((HashSet::new(), paths)),
+ Err(error) => Err(error),
+ }
+}
+
+/// Reconcile one file or directory prefix from primary state into a backup.
+pub async fn reconcile_prefix(
+ primary: &dyn FileSystem,
+ backup: &dyn FileSystem,
+ prefix: &str,
+) -> Result<()> {
+ if is_catch_up_internal_path(prefix) {
+ return Ok(());
+ }
+ let root = match primary.stat(prefix).await {
+ Ok(root) => root,
+ Err(Error::NotFound(_)) => {
+ return remove_stale_paths(backup, prefix, &HashSet::new()).await;
+ }
+ Err(error) => return Err(error),
+ };
+ if !root.is_dir {
+ return copy_file_state(primary, backup, prefix).await;
+ }
+
+ let entries = primary
+ .tree_directory(prefix, true, None, None, None, None, None, false)
+ .await?;
+ ensure_directory(backup, prefix, root.mode).await?;
+ let mut current = HashSet::from([prefix.to_string()]);
+ for entry in entries
+ .iter()
+ .filter(|entry| !is_catch_up_internal_path(&entry.path))
+ {
+ current.insert(entry.path.clone());
+ if entry.info.is_dir {
+ ensure_directory(backup, &entry.path, entry.info.mode).await?;
+ } else {
+ copy_file_state(primary, backup, &entry.path).await?;
+ }
+ }
+
+ remove_stale_paths(backup, prefix, &current).await
+}
+
+/// Reconcile one exact path from the primary's current state.
+pub async fn reconcile_file(
+ primary: &dyn FileSystem,
+ backup: &dyn FileSystem,
+ path: &str,
+) -> Result<()> {
+ if is_catch_up_internal_path(path) {
+ return Ok(());
+ }
+ match primary.stat(path).await {
+ Ok(info) if info.is_dir => reconcile_prefix(primary, backup, path).await,
+ Ok(_) => copy_file_state(primary, backup, path).await,
+ Err(Error::NotFound(_)) => remove_if_present(backup, path).await,
+ Err(error) => Err(error),
+ }
+}
+
+/// Remove stale business paths while preserving internal files and their ancestors.
+async fn remove_stale_paths(
+ backup: &dyn FileSystem,
+ prefix: &str,
+ current: &HashSet<String>,
+) -> Result<()> {
+ match backup.stat(prefix).await {
+ Ok(info) if !info.is_dir => {
+ return if current.contains(prefix) || is_multiwrite_internal_path(prefix) {
+ Ok(())
+ } else {
+ remove_if_present(backup, prefix).await
+ };
+ }
+ Ok(_) => {}
+ Err(Error::NotFound(_)) => return Ok(()),
+ Err(error) => return Err(error),
+ }
+ let entries = match backup
+ .tree_directory(prefix, true, None, None, None, None, None, false)
+ .await
+ {
+ Ok(entries) => entries,
+ Err(Error::NotFound(_)) => return Ok(()),
+ Err(error) => return Err(error),
+ };
+ let protected = entries
+ .iter()
+ .filter(|entry| is_catch_up_internal_path(&entry.path))
+ .flat_map(|entry| Path::new(&entry.path).ancestors().skip(1))
+ .filter_map(Path::to_str)
+ .map(str::to_string)
+ .collect::<HashSet<_>>();
+ let mut stale = std::iter::once(prefix.to_string())
+ .chain(entries.into_iter().map(|entry| entry.path))
+ .filter(|path| {
+ !is_catch_up_internal_path(path) && !current.contains(path) && !protected.contains(path)
+ })
+ .collect::<Vec<_>>();
+ stale.sort_by_key(|path| std::cmp::Reverse(path.matches('/').count()));
+ for path in stale {
+ remove_if_present(backup, &path).await?;
+ }
+ Ok(())
+}
+
+/// Recognize internal paths in either logical or mount-relative path space.
+fn is_catch_up_internal_path(path: &str) -> bool {
+ is_multiwrite_internal_path(path)
+ || is_multiwrite_internal_path(&format!(
+ "{MULTIWRITE_MOUNT_PREFIX}/{}",
+ path.trim_start_matches('/')
+ ))
+}
+
+/// Ensure one backup path is a directory with the requested mode.
+async fn ensure_directory(backup: &dyn FileSystem, path: &str, mode: u32) -> Result<()> {
+ match backup.stat(path).await {
+ Ok(info) if info.is_dir => {}
+ Ok(_) => {
+ backup.remove(path).await?;
+ backup.mkdir(path, mode).await?;
+ }
+ Err(Error::NotFound(_)) => {
+ backup.ensure_parent_dirs(path, 0o755).await?;
+ backup.mkdir(path, mode).await?;
+ }
+ Err(error) => return Err(error),
+ }
+ Ok(())
+}
+
+/// Replace one backup file with the primary bytes.
+async fn copy_file_state(
+ primary: &dyn FileSystem,
+ backup: &dyn FileSystem,
+ path: &str,
+) -> Result<()> {
+ if backup.stat(path).await.is_ok_and(|current| current.is_dir) {
+ backup.remove_all(path).await?;
+ }
+ let bytes = match primary.read(path, 0, 0).await {
+ Ok(bytes) => bytes,
+ Err(Error::NotFound(_)) => return remove_if_present(backup, path).await,
+ Err(error) => return Err(error),
+ };
+ backup.ensure_parent_dirs(path, 0o755).await?;
+ backup.write(path, &bytes, 0, WriteFlag::Create).await?;
+ Ok(())
+}
+
+/// Remove one backup path while accepting an already converged absence.
+async fn remove_if_present(backup: &dyn FileSystem, path: &str) -> Result<()> {
+ let result = match backup.stat(path).await {
+ Ok(info) if info.is_dir => backup.remove_all(path).await,
+ Ok(_) => backup.remove(path).await,
+ Err(Error::NotFound(_)) => return Ok(()),
+ Err(error) => return Err(error),
+ };
+ match result {
+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),
+ Err(error) => Err(error),
+ }
+}
diff --git a/crates/ragfs/src/multibackend/checkpoint.rs b/crates/ragfs/src/multibackend/checkpoint.rs
new file mode 100644
index 000000000..85f0987b3
--- /dev/null
+++ b/crates/ragfs/src/multibackend/checkpoint.rs
@@ -0,0 +1,1751 @@
+//! Checkpoint construction, validation, publication, and reading.
+
+use std::collections::{BTreeMap, BTreeSet};
+use std::sync::Arc;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use tokio::sync::watch;
+use tracing::warn;
+
+use crate::core::context::{FsContextInner, FS_CTX};
+use crate::core::errors::{Error, Result};
+use crate::core::filesystem::FileSystem;
+use crate::core::internal_names::is_multiwrite_internal_path;
+use crate::multibackend::catch_up::{CheckpointConsumer, CheckpointReadResult, CheckpointSnapshot};
+use crate::multibackend::codec::{decode_checkpoint_chunk, encode_checkpoint_chunk, sha256_hex};
+use crate::multibackend::constants::{
+ CHECKPOINT_CHUNK_FILE_EXTENSION, CHECKPOINT_CHUNK_FILE_PREFIX, CHECKPOINT_DIR_PREFIX,
+ CHECKPOINT_DIR_TIME_FORMAT, MANIFEST_FILE, MAX_CHECKPOINT_NODES, MULTIWRITE_MOUNT_PREFIX,
+};
+use crate::multibackend::gc::MetadataGc;
+use crate::multibackend::meta::{MetadataStore, MultiWriteWorker};
+use crate::multibackend::model::{
+ is_checkpoint_directory_name, CheckpointManifest, CheckpointNode, CheckpointsManifest,
+ ChunkDescriptor, DirectoryOperation, FileState, LatestCheckpoint, PartitionsManifest,
+ ProtocolStatus, ScopeKey, ScopedSeq, SegmentEventType, SegmentRecord,
+};
+use crate::multibackend::provider::MultiWriteProvider;
+use crate::multibackend::router::AccountRouter;
+
+/// Periodically builds checkpoints for every Stable account partition.
+pub struct CheckpointWorker {
+ store: Arc<MetadataStore>,
+ builder: CheckpointBuilder,
+ gc: MetadataGc,
+}
+impl CheckpointWorker {
+ /// Create a worker over the production primary, store, and provider.
+ pub fn new(
+ primary: Arc<dyn FileSystem>,
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+ ) -> Self {
+ let builder = CheckpointBuilder::new(primary, store.clone(), provider.clone());
+ Self {
+ gc: MetadataGc::new(store.clone(), provider),
+ builder,
+ store,
+ }
+ }
+
+ /// Run checkpoint rounds at the configured interval until cancellation.
+ pub async fn run(&self, interval: Duration, mut cancellation: watch::Receiver<bool>) {
+ let mut ticker = tokio::time::interval(interval);
+ ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+ loop {
+ tokio::select! {
+ _ = ticker.tick() => if let Err(error) = self.run_once(interval).await {
+ warn!(error = %error, "multi-write checkpoint discovery failed");
+ },
+ changed = cancellation.changed() => {
+ if changed.is_err() || *cancellation.borrow() {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ /// Build one checkpoint round while isolating account and scope failures.
+ pub async fn run_once(&self, interval: Duration) -> Result<()> {
+ let result = self.run_once_inner(interval).await;
+ if result.is_err() {
+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);
+ }
+ result
+ }
+
+ /// Execute one checkpoint round and return discovery failures.
+ async fn run_once_inner(&self, interval: Duration) -> Result<()> {
+ if self.store.protocol_status().await? != ProtocolStatus::Stable {
+ return Ok(());
+ }
+ let now = now_ns()?;
+ let interval_ns = u64::try_from(interval.as_nanos()).unwrap_or(u64::MAX);
+ for account in self.store.initialized_accounts().await? {
+ let path = self.store.paths().account_manifest(&account)?.0;
+ let manifest: PartitionsManifest = match self.store.read_json(&path).await {
+ Ok(manifest) => manifest,
+ Err(Error::NotFound(_)) => continue,
+ Err(error) => {
+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);
+ warn!(account = %account, error = %error,
+ "multi-write checkpoint account metadata unavailable");
+ continue;
+ }
+ };
+ if let Err(error) = manifest.validate() {
+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);
+ warn!(account = %account, error = %error,
+ "multi-write checkpoint account metadata invalid");
+ continue;
+ }
+ for (&partition_id, partition) in &manifest.partitions {
+ if partition.state != crate::multibackend::model::PartitionState::Stable {
+ continue;
+ }
+ let scope = ScopeKey {
+ account_id: account.clone(),
+ partition_id,
+ epoch: manifest.epoch,
+ };
+ let pointer_path = self
+ .store
+ .paths()
+ .checkpoints_manifest(&account, partition_id)?
+ .0;
+ let checkpoints: CheckpointsManifest =
+ match self.store.read_json(&pointer_path).await {
+ Ok(checkpoints) => checkpoints,
+ Err(Error::NotFound(_)) => continue,
+ Err(error) => {
+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);
+ warn!(account = %account, partition = partition_id, error = %error,
+ "multi-write checkpoint pointer unavailable");
+ continue;
+ }
+ };
+ if let Err(error) = checkpoints.validate() {
+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);
+ warn!(account = %account, partition = partition_id, error = %error,
+ "multi-write checkpoint pointer invalid");
+ continue;
+ }
+ if checkpoints
+ .latest_checkpoint
+ .as_ref()
+ .is_some_and(|latest| now < latest.updated_at_ns.saturating_add(interval_ns))
+ {
+ continue;
+ }
+ match self.builder.build_scope(&scope).await {
+ Ok(CheckpointBuildResult::Published(_)) => {
+ if let Err(error) = self.gc.collect_scope(&scope).await {
+ warn!(account = %account, partition = partition_id, error = %error,
+ "multi-write metadata garbage collection failed");
+ }
+ }
+ Ok(_) => {}
+ Err(Error::NotFound(_)) => {}
+ Err(error) => {
+ self.store.record_worker_error(MultiWriteWorker::Checkpoint);
+ warn!(account = %account, partition = partition_id, error = %error,
+ "multi-write checkpoint build failed");
+ }
+ }
+ }
+ }
+ Ok(())
+ }
+}
+
+/// Result of one bounded checkpoint build attempt.
+pub enum CheckpointBuildResult {
+ /// No new sealed sequence exists for this scope.
+ NoSealedChanges,
+ /// A validated checkpoint became the latest publication.
+ Published(LatestCheckpoint),
+ /// Another builder published first and this candidate was discarded.
+ Retry,
+}
+
+#[derive(Default)]
+struct SegmentDelta {
+ states: BTreeMap<String, FileState>,
+ exact_paths: BTreeSet<String>,
+ prefixes: BTreeSet<String>,
+}
+
+impl SegmentDelta {
+ /// Return whether this segment delta cannot change checkpoint contents.
+ fn is_empty(&self) -> bool {
+ self.states.is_empty() && self.exact_paths.is_empty() && self.prefixes.is_empty()
+ }
+}
+
+enum ChunkMergeTarget<'a> {
+ All,
+ Range(&'a str, Option<&'a str>),
+ RootOnly(&'a str),
+}
+
+/// Builds immutable checkpoints from sealed records and current primary state.
+pub struct CheckpointBuilder {
+ primary: Arc<dyn FileSystem>,
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+ router: AccountRouter,
+}
+impl CheckpointBuilder {
+ /// Create a builder over the encrypted primary and raw metadata store.
+ pub fn new(
+ primary: Arc<dyn FileSystem>,
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+ ) -> Self {
+ Self {
+ primary,
+ router: AccountRouter::new(store.clone()),
+ store,
+ provider,
+ }
+ }
+
+ /// Build and compare-before-publish one Stable partition checkpoint.
+ pub async fn build_scope(&self, scope: &ScopeKey) -> Result<CheckpointBuildResult> {
+ scope.validate()?;
+ let pointer_path = self
+ .store
+ .paths()
+ .checkpoints_manifest(&scope.account_id, scope.partition_id)?
+ .0;
+ let lease = self
+ .store
+ .pathlock_manager()
+ .acquire_exact(
+ &pointer_path,
+ self.store
+ .pathlock_manager()
+ .default_lock_timeout()
+ .max(Duration::from_secs(5)),
+ None,
+ )
+ .await?;
+ let read_pointer = async {
+ let start: CheckpointsManifest = self.store.read_json(&pointer_path).await?;
+ start.validate()?;
+ Ok(start)
+ }
+ .await;
+ let release = self
+ .store
+ .pathlock_manager()
+ .release(&lease)
+ .await
+ .map_err(Error::from);
+ let start = finish_with_release(read_pointer, release)?;
+ let segments = self.provider.read_manifest(scope).await?;
+ segments.validate()?;
+ let Some(tail) = segments
+ .segments
+ .iter()
+ .filter_map(|segment| segment.segment_to_seq)
+ .last()
+ else {
+ return Ok(CheckpointBuildResult::NoSealedChanges);
+ };
+ let old_to = start
+ .latest_checkpoint
+ .as_ref()
+ .map_or(0, |latest| latest.checkpoint_to_seq);
+ if tail <= old_to {
+ return Ok(CheckpointBuildResult::NoSealedChanges);
+ }
+ let min_synced = segments
+ .backend_states
+ .values()
+ .map(|state| state.synced_seq)
+ .min();
+ let created_at_ns = now_ns()?;
+ let candidate = checkpoint_candidate_name(created_at_ns)?;
+ let directory = self
+ .store
+ .paths()
+ .checkpoints_dir(&scope.account_id, scope.partition_id)?
+ .0;
+ let candidate_path = format!("{directory}/{}", candidate.trim_end_matches('/'));
+ let candidate_lease = self
+ .store
+ .pathlock_manager()
+ .acquire_exact(
+ &candidate_path,
+ self.store
+ .pathlock_manager()
+ .default_lock_timeout()
+ .max(Duration::from_secs(5)),
+ None,
+ )
+ .await?;
+ let operation = async {
+ let mut manifest = self
+ .prepare_candidate_checkpoint(scope, &start, &candidate, created_at_ns, old_to)
+ .await?;
+ let mut chunk_index = checkpoint_chunk_count(&manifest.root);
+ for descriptor in segments
+ .segments
+ .iter()
+ .filter(|descriptor| descriptor.segment_to_seq.is_some_and(|to| to > old_to))
+ {
+ let records = self.provider.read_sealed_segment(scope, descriptor).await?;
+ let records = records
+ .into_iter()
+ .filter(|record| record.seq > old_to && record.seq <= tail)
+ .collect::<Vec<_>>();
+ if records.is_empty() {
+ continue;
+ }
+ let delta = self.segment_delta(scope, records).await?;
+ self.apply_checkpoint_delta(
+ scope,
+ &candidate,
+ &mut manifest,
+ &delta,
+ &mut chunk_index,
+ )
+ .await?;
+ }
+ self.prune_checkpoint_tombstones(
+ scope,
+ &candidate,
+ &mut manifest,
+ min_synced,
+ &mut chunk_index,
+ )
+ .await?;
+ manifest.version = 1;
+ manifest.partition_id = scope.partition_id;
+ manifest.epoch = scope.epoch;
+ manifest.checkpoint_from_seq = old_to + 1;
+ manifest.checkpoint_to_seq = tail;
+ manifest.created_at_ns = created_at_ns;
+ manifest.checksum = tree_checksum(&manifest.root)?;
+ manifest.validate_for_scope(scope)?;
+ let manifest_bytes = serde_json::to_vec(&manifest)?;
+ self.store
+ .write_bytes(
+ &format!("{directory}/{candidate}{MANIFEST_FILE}"),
+ &manifest_bytes,
+ )
+ .await?;
+ verify_checkpoint(self.store.as_ref(), scope, &manifest, &candidate, tail).await?;
+ let latest = LatestCheckpoint {
+ path: candidate,
+ checkpoint_to_seq: tail,
+ updated_at_ns: created_at_ns,
+ };
+ let lease = self
+ .store
+ .pathlock_manager()
+ .acquire_exact(
+ &pointer_path,
+ self.store
+ .pathlock_manager()
+ .default_lock_timeout()
+ .max(Duration::from_secs(5)),
+ None,
+ )
+ .await?;
+ let publish = async {
+ let current: CheckpointsManifest = self.store.read_json(&pointer_path).await?;
+ if current.latest_checkpoint != start.latest_checkpoint {
+ return Ok(CheckpointBuildResult::Retry);
+ }
+ self.store
+ .publish_json(
+ &pointer_path,
+ &CheckpointsManifest {
+ version: 1,
+ latest_checkpoint: Some(latest.clone()),
+ },
+ CheckpointsManifest::validate,
+ )
+ .await?;
+ Ok(CheckpointBuildResult::Published(latest))
+ }
+ .await;
+ let release = self
+ .store
+ .pathlock_manager()
+ .release(&lease)
+ .await
+ .map_err(Error::from);
+ finish_with_release(publish, release)
+ }
+ .await;
+ let release = self
+ .store
+ .pathlock_manager()
+ .release(&candidate_lease)
+ .await
+ .map_err(Error::from);
+ finish_with_release(operation, release)
+ }
+
+ /// Prepare a candidate checkpoint directory from the latest checkpoint or an empty base.
+ async fn prepare_candidate_checkpoint(
+ &self,
+ scope: &ScopeKey,
+ start: &CheckpointsManifest,
+ candidate: &str,
+ created_at_ns: u64,
+ old_to: u64,
+ ) -> Result<CheckpointManifest> {
+ if let Some(latest) = &start.latest_checkpoint {
+ let manifest = read_manifest(self.store.as_ref(), scope, &latest.path).await?;
+ if manifest.checkpoint_to_seq != latest.checkpoint_to_seq {
+ return Err(Error::Serialization(
+ "checkpoint pointer sequence mismatch".into(),
+ ));
+ }
+ let manifest = copy_checkpoint_files(
+ self.store.as_ref(),
+ scope,
+ &latest.path,
+ candidate,
+ &manifest,
+ )
+ .await?;
+ return Ok(manifest);
+ }
+
+ let mut chunk_index = 0;
+ let (root, chunks) = encode_states_with_index(scope, Vec::new(), &mut chunk_index)?;
+ let checksum = tree_checksum(&root)?;
+ let manifest = CheckpointManifest {
+ version: 1,
+ partition_id: scope.partition_id,
+ epoch: scope.epoch,
+ checkpoint_from_seq: 1,
+ checkpoint_to_seq: old_to,
+ created_at_ns,
+ root,
+ checksum,
+ };
+ manifest.validate_for_scope(scope)?;
+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;
+ let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?;
+ self.store
+ .write_bytes(
+ &format!("{directory}/{MANIFEST_FILE}"),
+ &serde_json::to_vec(&manifest)?,
+ )
+ .await?;
+ Ok(manifest)
+ }
+
+ /// Convert one sealed segment into checkpoint path replacements and prefix invalidations.
+ async fn segment_delta(
+ &self,
+ scope: &ScopeKey,
+ records: Vec<SegmentRecord>,
+ ) -> Result<SegmentDelta> {
+ let mut delta = SegmentDelta::default();
+ for record in records {
+ match record.event_type {
+ SegmentEventType::Write | SegmentEventType::Remove => {
+ let backend_path = self
+ .store
+ .paths()
+ .backend_path(&scope.account_id, &record.path)?;
+ let deleted = match FS_CTX
+ .scope(
+ Arc::new(
+ FsContextInner::new(&scope.account_id)
+ .with_bypass_cache(true)
+ .with_auto_pathlock_disabled(),
+ ),
+ self.primary.stat(&backend_path),
+ )
+ .await
+ {
+ Ok(_) => None,
+ Err(Error::NotFound(_)) => Some(true),
+ Err(error) => return Err(error),
+ };
+ delta.exact_paths.insert(record.path.clone());
+ delta.states.insert(
+ record.path.clone(),
+ FileState {
+ path: record.path,
+ latest_seq: ScopedSeq {
+ scope: scope.clone(),
+ seq: record.seq,
+ },
+ deleted,
+ },
+ );
+ }
+ SegmentEventType::RemoveTree | SegmentEventType::MoveTree => {
+ let mut prefixes = vec![record.path.clone()];
+ if let Some(destination) = record.destination_path {
+ prefixes.push(destination);
+ }
+ for prefix in prefixes {
+ delta.prefixes.insert(prefix.clone());
+ delta
+ .states
+ .retain(|path, _| !is_path_within(path, &prefix));
+ for path in self.scan_prefix(scope, &prefix).await? {
+ delta.states.insert(
+ path.clone(),
+ FileState {
+ path,
+ latest_seq: ScopedSeq {
+ scope: scope.clone(),
+ seq: record.seq,
+ },
+ deleted: None,
+ },
+ );
+ }
+ }
+ }
+ SegmentEventType::DeleteScope => {}
+ }
+ }
+ Ok(delta)
+ }
+
+ /// Apply one segment delta to the candidate manifest by rewriting only needed chunks.
+ async fn apply_checkpoint_delta(
+ &self,
+ scope: &ScopeKey,
+ candidate: &str,
+ manifest: &mut CheckpointManifest,
+ delta: &SegmentDelta,
+ chunk_index: &mut usize,
+ ) -> Result<()> {
+ if delta.is_empty() {
+ return Ok(());
+ }
+ if manifest.root.chunks.is_empty() {
+ self.rewrite_full_checkpoint_chunk(scope, candidate, manifest, delta, chunk_index)
+ .await?;
+ } else {
+ self.rewrite_split_checkpoint_chunks(scope, candidate, manifest, delta, chunk_index)
+ .await?;
+ }
+ manifest.root.validate()?;
+ Ok(())
+ }
+
+ /// Rewrite a checkpoint whose complete state still lives in the root chunk.
+ async fn rewrite_full_checkpoint_chunk(
+ &self,
+ scope: &ScopeKey,
+ candidate: &str,
+ manifest: &mut CheckpointManifest,
+ delta: &SegmentDelta,
+ chunk_index: &mut usize,
+ ) -> Result<()> {
+ let old_root = manifest.root.clone();
+ let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?;
+ let states =
+ read_checkpoint_chunk_states(self.store.as_ref(), scope, &directory, &old_root, true)
+ .await?;
+ let mut remaining = delta.states.clone();
+ let states = merge_checkpoint_states(states, delta, &mut remaining, ChunkMergeTarget::All);
+ let (root, chunks) = encode_states_with_index(scope, states, chunk_index)?;
+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;
+ manifest.root = root;
+ remove_replaced_checkpoint_files(
+ self.store.as_ref(),
+ scope,
+ candidate,
+ checkpoint_chunk_files(&old_root),
+ &manifest.root,
+ )
+ .await;
+ Ok(())
+ }
+
+ /// Rewrite affected child chunks and insert new states not covered by old ranges.
+ async fn rewrite_split_checkpoint_chunks(
+ &self,
+ scope: &ScopeKey,
+ candidate: &str,
+ manifest: &mut CheckpointManifest,
+ delta: &SegmentDelta,
+ chunk_index: &mut usize,
+ ) -> Result<()> {
+ let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?;
+ let account_root = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id);
+ let mut remaining = delta.states.clone();
+
+ if checkpoint_state_is_dirty(&account_root, delta) || remaining.contains_key(&account_root)
+ {
+ let old_root = manifest.root.clone();
+ let children = std::mem::take(&mut manifest.root.chunks);
+ let states = read_checkpoint_chunk_states(
+ self.store.as_ref(),
+ scope,
+ &directory,
+ &old_root,
+ true,
+ )
+ .await?;
+ let states = merge_checkpoint_states(
+ states,
+ delta,
+ &mut remaining,
+ ChunkMergeTarget::RootOnly(&account_root),
+ );
+ let (mut root, chunks) = encode_states_with_index(scope, states, chunk_index)?;
+ root.chunks = children;
+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;
+ manifest.root = root;
+ remove_replaced_checkpoint_files(
+ self.store.as_ref(),
+ scope,
+ candidate,
+ vec![old_root.file],
+ &manifest.root,
+ )
+ .await;
+ }
+
+ let mut next_children = Vec::new();
+ let children = std::mem::take(&mut manifest.root.chunks);
+ for descriptor in children {
+ if checkpoint_descriptor_is_dirty(&descriptor, delta)? {
+ let old_files = checkpoint_chunk_files(&descriptor);
+ let states = read_checkpoint_chunk_states(
+ self.store.as_ref(),
+ scope,
+ &directory,
+ &descriptor,
+ false,
+ )
+ .await?;
+ let (start, end) = checkpoint_descriptor_range(&descriptor)?;
+ let states = merge_checkpoint_states(
+ states,
+ delta,
+ &mut remaining,
+ ChunkMergeTarget::Range(start, end),
+ );
+ let (mut descriptors, chunks) =
+ encode_child_states_with_index(scope, states, start, end, chunk_index)?;
+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;
+ let replacement_root = ChunkDescriptor {
+ file: String::new(),
+ root_path: account_root.clone(),
+ start_path: None,
+ end_path: None,
+ file_state_count: 0,
+ checksum: String::new(),
+ chunks: descriptors.clone(),
+ };
+ remove_replaced_checkpoint_files(
+ self.store.as_ref(),
+ scope,
+ candidate,
+ old_files,
+ &replacement_root,
+ )
+ .await;
+ next_children.append(&mut descriptors);
+ } else {
+ next_children.push(descriptor);
+ }
+ }
+
+ if !remaining.is_empty() {
+ return Err(Error::Serialization(
+ "checkpoint delta paths were not assigned to a chunk".into(),
+ ));
+ }
+ sort_checkpoint_children(&mut next_children, &account_root)?;
+ manifest.root.chunks = next_children;
+ Ok(())
+ }
+
+ /// Remove checkpoint tombstones that no current backend still needs.
+ async fn prune_checkpoint_tombstones(
+ &self,
+ scope: &ScopeKey,
+ candidate: &str,
+ manifest: &mut CheckpointManifest,
+ min_synced: Option<u64>,
+ chunk_index: &mut usize,
+ ) -> Result<()> {
+ if manifest.root.chunks.is_empty() {
+ let old_root = manifest.root.clone();
+ let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?;
+ let states = read_checkpoint_chunk_states(
+ self.store.as_ref(),
+ scope,
+ &directory,
+ &old_root,
+ true,
+ )
+ .await?;
+ let retained = retain_checkpoint_states(states.clone(), min_synced);
+ if retained != states {
+ let (root, chunks) = encode_states_with_index(scope, retained, chunk_index)?;
+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;
+ manifest.root = root;
+ remove_replaced_checkpoint_files(
+ self.store.as_ref(),
+ scope,
+ candidate,
+ checkpoint_chunk_files(&old_root),
+ &manifest.root,
+ )
+ .await;
+ }
+ return Ok(());
+ }
+
+ let directory = checkpoint_directory(self.store.as_ref(), scope, candidate)?;
+ let account_root = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id);
+ let old_root = manifest.root.clone();
+ let children = std::mem::take(&mut manifest.root.chunks);
+ let states =
+ read_checkpoint_chunk_states(self.store.as_ref(), scope, &directory, &old_root, true)
+ .await?;
+ let retained = retain_checkpoint_states(states.clone(), min_synced);
+ if retained != states {
+ let (mut root, chunks) = encode_states_with_index(scope, retained, chunk_index)?;
+ root.chunks = children;
+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;
+ manifest.root = root;
+ remove_replaced_checkpoint_files(
+ self.store.as_ref(),
+ scope,
+ candidate,
+ vec![old_root.file],
+ &manifest.root,
+ )
+ .await;
+ } else {
+ manifest.root.chunks = children;
+ }
+
+ let mut next_children = Vec::new();
+ for descriptor in std::mem::take(&mut manifest.root.chunks) {
+ let old_files = checkpoint_chunk_files(&descriptor);
+ let states = read_checkpoint_chunk_states(
+ self.store.as_ref(),
+ scope,
+ &directory,
+ &descriptor,
+ false,
+ )
+ .await?;
+ let retained = retain_checkpoint_states(states.clone(), min_synced);
+ if retained == states {
+ next_children.push(descriptor);
+ continue;
+ }
+ let (start, end) = checkpoint_descriptor_range(&descriptor)?;
+ let (mut descriptors, chunks) =
+ encode_child_states_with_index(scope, retained, start, end, chunk_index)?;
+ write_checkpoint_chunks(self.store.as_ref(), scope, candidate, &chunks).await?;
+ let replacement_root = ChunkDescriptor {
+ file: String::new(),
+ root_path: account_root.clone(),
+ start_path: None,
+ end_path: None,
+ file_state_count: 0,
+ checksum: String::new(),
+ chunks: descriptors.clone(),
+ };
+ remove_replaced_checkpoint_files(
+ self.store.as_ref(),
+ scope,
+ candidate,
+ old_files,
+ &replacement_root,
+ )
+ .await;
+ next_children.append(&mut descriptors);
+ }
+ sort_checkpoint_children(&mut next_children, &account_root)?;
+ manifest.root.chunks = next_children;
+ Ok(())
+ }
+
+ /// Scan one current primary prefix and retain paths owned by this scope.
+ async fn scan_prefix(&self, scope: &ScopeKey, logical_prefix: &str) -> Result<Vec<String>> {
+ let backend_prefix = self
+ .store
+ .paths()
+ .backend_path(&scope.account_id, logical_prefix)?;
+ let context = Arc::new(
+ FsContextInner::new(&scope.account_id)
+ .with_bypass_cache(true)
+ .with_auto_pathlock_disabled(),
+ );
+ let paths = FS_CTX
+ .scope(context, async {
+ let root = match self.primary.stat(&backend_prefix).await {
+ Ok(root) => root,
+ Err(Error::NotFound(_)) => return Ok(Vec::new()),
+ Err(error) => return Err(error),
+ };
+ let mut paths = vec![backend_prefix.clone()];
+ if root.is_dir {
+ paths.extend(
+ self.primary
+ .tree_directory(
+ &backend_prefix,
+ true,
+ None,
+ None,
+ None,
+ None,
+ None,
+ false,
+ )
+ .await?
+ .into_iter()
+ .map(|entry| entry.path),
+ );
+ }
+ Ok(paths)
+ })
+ .await?;
+ let mut selected = Vec::new();
+ for path in paths {
+ let logical = format!("{MULTIWRITE_MOUNT_PREFIX}{path}");
+ if !is_multiwrite_internal_path(&logical)
+ && self.router.route(&scope.account_id, &logical).await?.scope == *scope
+ {
+ selected.push(logical);
+ }
+ }
+ selected.sort();
+ selected.dedup();
+ Ok(selected)
+ }
+}
+
+/// Reads complete validated checkpoint snapshots under the publication lease.
+pub struct CheckpointReader {
+ store: Arc<MetadataStore>,
+}
+impl CheckpointReader {
+ /// Create a reader over the raw metadata store.
+ pub fn new(store: Arc<MetadataStore>) -> Self {
+ Self { store }
+ }
+}
+
+#[async_trait]
+impl CheckpointConsumer for CheckpointReader {
+ /// Read the latest checkpoint or return a typed retry for immutable races.
+ async fn read_latest(
+ &self,
+ scope: &ScopeKey,
+ synced_seq: u64,
+ tail_seq: u64,
+ ) -> Result<CheckpointReadResult> {
+ let pointer_path = self
+ .store
+ .paths()
+ .checkpoints_manifest(&scope.account_id, scope.partition_id)?
+ .0;
+ let lease = self
+ .store
+ .pathlock_manager()
+ .acquire_exact(
+ &pointer_path,
+ self.store
+ .pathlock_manager()
+ .default_lock_timeout()
+ .max(Duration::from_secs(5)),
+ None,
+ )
+ .await?;
+ let pointer = async {
+ let pointer: CheckpointsManifest = self.store.read_json(&pointer_path).await?;
+ pointer.validate()?;
+ let Some(latest) = pointer.latest_checkpoint else {
+ return Ok(None);
+ };
+ if latest.checkpoint_to_seq <= synced_seq || latest.checkpoint_to_seq > tail_seq {
+ return Ok(None);
+ }
+ Ok(Some(latest))
+ }
+ .await;
+ let release = self
+ .store
+ .pathlock_manager()
+ .release(&lease)
+ .await
+ .map_err(Error::from);
+ let latest = finish_with_release(pointer, release)?;
+ let Some(latest) = latest else {
+ return Ok(CheckpointReadResult::NoCheckpoint);
+ };
+ let states = match read_published(self.store.as_ref(), scope, &latest).await {
+ Ok(states) => states,
+ Err(Error::NotFound(_)) | Err(Error::Serialization(_)) => {
+ return Ok(CheckpointReadResult::RetryLatestCheckpoint)
+ }
+ Err(error) => return Err(error),
+ };
+ let account_manifest_path = self.store.paths().account_manifest(&scope.account_id)?.0;
+ let account_manifest: PartitionsManifest =
+ self.store.read_json(&account_manifest_path).await?;
+ account_manifest.validate()?;
+ let mut directory_prefixes = BTreeSet::new();
+ for event in account_manifest.directory_events {
+ if event.positions.iter().any(|position| {
+ position.partition_id == scope.partition_id
+ && position.epoch == scope.epoch
+ && position.seq <= latest.checkpoint_to_seq
+ }) {
+ directory_prefixes.insert(event.source_path);
+ if event.operation == DirectoryOperation::MoveTree {
+ directory_prefixes.insert(event.destination_path.ok_or_else(|| {
+ Error::Serialization("move directory event lacks destination".into())
+ })?);
+ }
+ }
+ }
+ Ok(CheckpointReadResult::Snapshot(CheckpointSnapshot {
+ checkpoint_to_seq: latest.checkpoint_to_seq,
+ file_states: states,
+ directory_prefixes: directory_prefixes.into_iter().collect(),
+ }))
+ }
+}
+
+/// Encode a deterministic checkpoint tree while continuing the caller's chunk index.
+fn encode_states_with_index(
+ scope: &ScopeKey,
+ mut states: Vec<FileState>,
+ index: &mut usize,
+) -> Result<(ChunkDescriptor, Vec<(String, Vec<u8>)>)> {
+ let (node, root_path) = checkpoint_node_from_states(scope, &mut states)?;
+ encode_checkpoint_tree(node, &root_path, index)
+}
+
+/// Encode states as one or more non-root checkpoint descriptors.
+fn encode_child_states_with_index(
+ scope: &ScopeKey,
+ mut states: Vec<FileState>,
+ start_path: &str,
+ end_path: Option<&str>,
+ index: &mut usize,
+) -> Result<(Vec<ChunkDescriptor>, Vec<(String, Vec<u8>)>)> {
+ if states.is_empty() {
+ return Ok((Vec::new(), Vec::new()));
+ }
+ let (mut node, root_path) = checkpoint_node_from_states(scope, &mut states)?;
+ let mut files = Vec::new();
+ let mut descriptors = Vec::new();
+ let children = std::mem::take(&mut node.children);
+ if node.latest_seq.is_some() {
+ descriptors.push(encode_node(node, &root_path, false, index, &mut files)?);
+ }
+ partition_children(children, &root_path, index, &mut files, &mut descriptors)?;
+ assign_checkpoint_child_ranges(&mut descriptors, start_path, end_path)?;
+ Ok((descriptors, files))
+}
+
+/// Build the account-relative checkpoint radix tree from sorted file states.
+fn checkpoint_node_from_states(
+ scope: &ScopeKey,
+ states: &mut Vec<FileState>,
+) -> Result<(CheckpointNode, String)> {
+ for state in states.iter() {
+ state.validate(scope)?;
+ }
+ states.sort_by(|left, right| left.path.cmp(&right.path));
+ if states.windows(2).any(|pair| pair[0].path == pair[1].path) {
+ return Err(Error::Serialization(
+ "checkpoint contains duplicate paths".into(),
+ ));
+ }
+ let root_path = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id);
+ let mut node = CheckpointNode {
+ path_fragment: String::new(),
+ latest_seq: None,
+ deleted: None,
+ children: Vec::new(),
+ };
+ for state in states.iter() {
+ if !is_canonical_account_path(&state.path, &root_path) {
+ return Err(Error::Serialization(
+ "checkpoint path crosses account root".into(),
+ ));
+ }
+ let relative = state
+ .path
+ .strip_prefix(&root_path)
+ .ok_or_else(|| Error::Serialization("checkpoint path crosses account root".into()))?;
+ insert_state(&mut node, relative, state)?;
+ }
+ Ok((node, root_path))
+}
+
+/// Encode one preconstructed radix tree through the production chunk splitter.
+fn encode_checkpoint_tree(
+ mut node: CheckpointNode,
+ root_path: &str,
+ index: &mut usize,
+) -> Result<(ChunkDescriptor, Vec<(String, Vec<u8>)>)> {
+ node.validate()?;
+ let mut files = Vec::new();
+ let root = if checkpoint_node_count(&node) <= MAX_CHECKPOINT_NODES {
+ encode_node(node, root_path, true, index, &mut files)?
+ } else {
+ let children = std::mem::take(&mut node.children);
+ let mut descriptor = encode_node(node, root_path, true, index, &mut files)?;
+ partition_children(
+ children,
+ root_path,
+ index,
+ &mut files,
+ &mut descriptor.chunks,
+ )?;
+ assign_checkpoint_child_ranges(&mut descriptor.chunks, root_path, None)?;
+ descriptor
+ };
+ Ok((root, files))
+}
+
+/// Insert one file state into the account-relative component radix tree.
+fn insert_state(root: &mut CheckpointNode, relative: &str, state: &FileState) -> Result<()> {
+ let mut node = root;
+ let mut remaining = relative;
+ while !remaining.is_empty() {
+ let Some(index) = node
+ .children
+ .iter()
+ .position(|child| common_prefix_len(&child.path_fragment, remaining) > 0)
+ else {
+ node.children.push(CheckpointNode {
+ path_fragment: remaining.to_string(),
+ latest_seq: Some(state.latest_seq.seq),
+ deleted: state.deleted,
+ children: Vec::new(),
+ });
+ node.children
+ .sort_by(|left, right| left.path_fragment.cmp(&right.path_fragment));
+ return Ok(());
+ };
+ let common = common_prefix_len(&node.children[index].path_fragment, remaining);
+ if common == node.children[index].path_fragment.len() {
+ remaining = &remaining[common..];
+ node = &mut node.children[index];
+ } else {
+ let mut old = node.children.remove(index);
+ let suffix = old.path_fragment.split_off(common);
+ let prefix = std::mem::replace(&mut old.path_fragment, suffix);
+ let mut parent = CheckpointNode {
+ path_fragment: prefix,
+ latest_seq: None,
+ deleted: None,
+ children: vec![old],
+ };
+ remaining = &remaining[common..];
+ if remaining.is_empty() {
+ parent.latest_seq = Some(state.latest_seq.seq);
+ parent.deleted = state.deleted;
+ } else {
+ parent.children.push(CheckpointNode {
+ path_fragment: remaining.to_string(),
+ latest_seq: Some(state.latest_seq.seq),
+ deleted: state.deleted,
+ children: Vec::new(),
+ });
+ parent
+ .children
+ .sort_by(|left, right| left.path_fragment.cmp(&right.path_fragment));
+ }
+ node.children.insert(index, parent);
+ return Ok(());
+ }
+ }
+ if node.latest_seq.is_some() {
+ return Err(Error::Serialization(
+ "checkpoint contains duplicate paths".into(),
+ ));
+ }
+ node.latest_seq = Some(state.latest_seq.seq);
+ node.deleted = state.deleted;
+ Ok(())
+}
+
+/// Return the longest UTF-8-safe common prefix length.
+fn common_prefix_len(left: &str, right: &str) -> usize {
+ left.char_indices()
+ .zip(right.char_indices())
+ .take_while(|((_, left), (_, right))| left == right)
+ .last()
+ .map_or(0, |((index, ch), _)| index + ch.len_utf8())
+}
+
+/// Split ordered direct child ranges into bounded immutable chunks.
+fn partition_children(
+ children: Vec<CheckpointNode>,
+ parent_path: &str,
+ index: &mut usize,
+ files: &mut Vec<(String, Vec<u8>)>,
+ descriptors: &mut Vec<ChunkDescriptor>,
+) -> Result<()> {
+ let mut group = Vec::new();
+ let mut group_nodes = 1;
+ for mut child in children {
+ let child_nodes = checkpoint_node_count(&child);
+ if child_nodes + 1 > MAX_CHECKPOINT_NODES {
+ flush_child_group(&mut group, parent_path, index, files, descriptors)?;
+ let fragment = child.path_fragment.clone();
+ let mut grandchildren = std::mem::take(&mut child.children);
+ if child.latest_seq.is_some() {
+ descriptors.push(encode_node(child, parent_path, false, index, files)?);
+ }
+ for grandchild in &mut grandchildren {
+ grandchild.path_fragment.insert_str(0, &fragment);
+ }
+ partition_children(grandchildren, parent_path, index, files, descriptors)?;
+ } else {
+ if group_nodes + child_nodes > MAX_CHECKPOINT_NODES {
+ flush_child_group(&mut group, parent_path, index, files, descriptors)?;
+ group_nodes = 1;
+ }
+ group_nodes += child_nodes;
+ group.push(child);
+ }
+ }
+ flush_child_group(&mut group, parent_path, index, files, descriptors)
+}
+
+/// Encode one non-empty direct child range as a synthetic-root chunk.
+fn flush_child_group(
+ children: &mut Vec<CheckpointNode>,
+ parent_path: &str,
+ index: &mut usize,
+ files: &mut Vec<(String, Vec<u8>)>,
+ descriptors: &mut Vec<ChunkDescriptor>,
+) -> Result<()> {
+ if children.is_empty() {
+ return Ok(());
+ }
+ let node = CheckpointNode {
+ path_fragment: String::new(),
+ latest_seq: None,
+ deleted: None,
+ children: std::mem::take(children),
+ };
+ descriptors.push(encode_node(node, parent_path, false, index, files)?);
+ Ok(())
+}
+
+/// Encode one bounded radix subtree and derive its exact path range.
+fn encode_node(
+ node: CheckpointNode,
+ root_path: &str,
+ root: bool,
+ index: &mut usize,
+ files: &mut Vec<(String, Vec<u8>)>,
+) -> Result<ChunkDescriptor> {
+ let (count, first_path) = checkpoint_node_range(&node, root_path);
+ let start_path = if root {
+ None
+ } else {
+ Some(
+ first_path
+ .or_else(|| checkpoint_node_start_path(&node, root_path))
+ .ok_or_else(|| {
+ Error::Serialization("checkpoint child chunk lacks start path".into())
+ })?,
+ )
+ };
+ let bytes = encode_checkpoint_chunk(&node)?;
+ let checksum = sha256_hex(&bytes);
+ let file = checkpoint_chunk_file_name(*index, &checksum);
+ *index += 1;
+ files.push((file.clone(), bytes));
+ Ok(ChunkDescriptor {
+ file,
+ root_path: root_path.to_string(),
+ start_path,
+ end_path: None,
+ file_state_count: count as u32,
+ checksum,
+ chunks: Vec::new(),
+ })
+}
+
+/// Count nodes in one radix subtree.
+fn checkpoint_node_count(root: &CheckpointNode) -> usize {
+ let mut nodes = vec![root];
+ let mut count = 0;
+ while let Some(node) = nodes.pop() {
+ count += 1;
+ nodes.extend(&node.children);
+ }
+ count
+}
+
+/// Return the state count and first ordered path encoded in one chunk.
+fn checkpoint_node_range(root: &CheckpointNode, root_path: &str) -> (usize, Option<String>) {
+ let mut paths = Vec::new();
+ collect_node_paths(root, root_path, &mut paths);
+ (paths.len(), paths.first().cloned())
+}
+
+/// Return the lowest owned path prefix for one encoded checkpoint subtree.
+fn checkpoint_node_start_path(node: &CheckpointNode, root_path: &str) -> Option<String> {
+ let path = format!("{root_path}{}", node.path_fragment);
+ if node.latest_seq.is_some() || !node.path_fragment.is_empty() {
+ return Some(path);
+ }
+ node.children
+ .first()
+ .map(|child| format!("{path}{}", child.path_fragment))
+}
+
+/// Assign contiguous left-closed and right-open ownership ranges to siblings.
+fn assign_checkpoint_child_ranges(
+ children: &mut Vec<ChunkDescriptor>,
+ start_path: &str,
+ end_path: Option<&str>,
+) -> Result<()> {
+ if children.is_empty() {
+ return Ok(());
+ }
+ children.sort_by(|left, right| left.start_path.cmp(&right.start_path));
+ let starts = children
+ .iter()
+ .map(|child| {
+ child.start_path.clone().ok_or_else(|| {
+ Error::Serialization("checkpoint child chunk lacks start path".into())
+ })
+ })
+ .collect::<Result<Vec<_>>>()?;
+ for index in 0..children.len() {
+ children[index].start_path = Some(if index == 0 {
+ start_path.to_string()
+ } else {
+ starts[index].clone()
+ });
+ children[index].end_path = if index + 1 < starts.len() {
+ Some(starts[index + 1].clone())
+ } else {
+ end_path.map(str::to_string)
+ };
+ }
+ Ok(())
+}
+
+/// Collect state paths from one radix subtree in canonical order.
+fn collect_node_paths(node: &CheckpointNode, parent: &str, paths: &mut Vec<String>) {
+ let path = format!("{parent}{}", node.path_fragment);
+ if node.latest_seq.is_some() {
+ paths.push(path.clone());
+ }
+ for child in &node.children {
+ collect_node_paths(child, &path, paths);
+ }
+}
+
+/// Read and validate one published checkpoint reference.
+async fn read_published(
+ store: &MetadataStore,
+ scope: &ScopeKey,
+ latest: &LatestCheckpoint,
+) -> Result<Vec<FileState>> {
+ let manifest = read_manifest(store, scope, &latest.path).await?;
+ if manifest.checkpoint_to_seq != latest.checkpoint_to_seq {
+ return Err(Error::Serialization(
+ "checkpoint pointer sequence mismatch".into(),
+ ));
+ }
+ read_chunks(store, scope, &manifest, &latest.path).await
+}
+
+/// Read and validate one immutable checkpoint manifest.
+async fn read_manifest(
+ store: &MetadataStore,
+ scope: &ScopeKey,
+ candidate: &str,
+) -> Result<CheckpointManifest> {
+ let directory = checkpoint_directory(store, scope, candidate)?;
+ let manifest: CheckpointManifest = store
+ .read_json(&format!("{directory}/{MANIFEST_FILE}"))
+ .await?;
+ manifest.validate_for_scope(scope)?;
+ if tree_checksum(&manifest.root)? != manifest.checksum {
+ return Err(Error::Serialization(
+ "checkpoint tree checksum mismatch".into(),
+ ));
+ }
+ Ok(manifest)
+}
+
+/// Build the second-resolution checkpoint directory name used by new publications.
+fn checkpoint_candidate_name(created_at_ns: u64) -> Result<String> {
+ let seconds = i64::try_from(created_at_ns / 1_000_000_000)
+ .map_err(|_| Error::internal("checkpoint timestamp overflow"))?;
+ let datetime = DateTime::<Utc>::from_timestamp(seconds, 0)
+ .ok_or_else(|| Error::internal("checkpoint timestamp is out of range"))?;
+ Ok(format!(
+ "{CHECKPOINT_DIR_PREFIX}{}/",
+ datetime.format(CHECKPOINT_DIR_TIME_FORMAT)
+ ))
+}
+
+/// Build a checkpoint chunk filename without embedding any sequence number.
+fn checkpoint_chunk_file_name(index: usize, checksum: &str) -> String {
+ format!("{CHECKPOINT_CHUNK_FILE_PREFIX}{index:06}-{checksum}{CHECKPOINT_CHUNK_FILE_EXTENSION}")
+}
+
+/// Rename descriptor file references while recording old-to-new chunk copies.
+fn rename_checkpoint_chunk_files(
+ descriptor: &mut ChunkDescriptor,
+ index: &mut usize,
+ copies: &mut Vec<(String, String, String)>,
+) {
+ let source = descriptor.file.clone();
+ let candidate = checkpoint_chunk_file_name(*index, &descriptor.checksum);
+ *index += 1;
+ descriptor.file = candidate.clone();
+ copies.push((source, candidate, descriptor.checksum.clone()));
+ for child in &mut descriptor.chunks {
+ rename_checkpoint_chunk_files(child, index, copies);
+ }
+}
+
+/// Copy a published checkpoint manifest and referenced chunks into a candidate directory.
+async fn copy_checkpoint_files(
+ store: &MetadataStore,
+ scope: &ScopeKey,
+ source: &str,
+ candidate: &str,
+ manifest: &CheckpointManifest,
+) -> Result<CheckpointManifest> {
+ let source_directory = checkpoint_directory(store, scope, source)?;
+ let candidate_directory = checkpoint_directory(store, scope, candidate)?;
+ let mut candidate_manifest = manifest.clone();
+ let mut copies = Vec::new();
+ let mut index = 0;
+ rename_checkpoint_chunk_files(&mut candidate_manifest.root, &mut index, &mut copies);
+ candidate_manifest.checksum = tree_checksum(&candidate_manifest.root)?;
+ candidate_manifest.validate_for_scope(scope)?;
+ for (source_file, candidate_file, checksum) in copies {
+ let bytes = store
+ .read_bytes(&format!("{source_directory}/{source_file}"))
+ .await?;
+ if sha256_hex(&bytes) != checksum {
+ return Err(Error::Serialization(
+ "checkpoint chunk checksum mismatch".into(),
+ ));
+ }
+ store
+ .write_bytes(&format!("{candidate_directory}/{candidate_file}"), &bytes)
+ .await?;
+ }
+ store
+ .write_bytes(
+ &format!("{candidate_directory}/{MANIFEST_FILE}"),
+ &serde_json::to_vec(&candidate_manifest)?,
+ )
+ .await?;
+ Ok(candidate_manifest)
+}
+
+/// Write checkpoint chunk bytes into one candidate directory.
+async fn write_checkpoint_chunks(
+ store: &MetadataStore,
+ scope: &ScopeKey,
+ candidate: &str,
+ chunks: &[(String, Vec<u8>)],
+) -> Result<()> {
+ let directory = checkpoint_directory(store, scope, candidate)?;
+ for (file, bytes) in chunks {
+ store
+ .write_bytes(&format!("{directory}/{file}"), bytes)
+ .await?;
+ }
+ Ok(())
+}
+
+/// Read and validate one checkpoint chunk without reading its child descriptors.
+async fn read_checkpoint_chunk_states(
+ store: &MetadataStore,
+ scope: &ScopeKey,
+ directory: &str,
+ descriptor: &ChunkDescriptor,
+ root: bool,
+) -> Result<Vec<FileState>> {
+ let account_root = format!("{MULTIWRITE_MOUNT_PREFIX}/{}", scope.account_id);
+ if descriptor.root_path != account_root {
+ return Err(Error::Serialization(
+ "checkpoint descriptor root mismatch".into(),
+ ));
+ }
+ let bytes = store
+ .read_bytes(&format!("{directory}/{}", descriptor.file))
+ .await?;
+ if sha256_hex(&bytes) != descriptor.checksum {
+ return Err(Error::Serialization(
+ "checkpoint chunk checksum mismatch".into(),
+ ));
+ }
+ let node = decode_checkpoint_chunk(&bytes)?;
+ if encode_checkpoint_chunk(&node)? != bytes {
+ return Err(Error::Serialization(
+ "checkpoint chunk is not canonical".into(),
+ ));
+ }
+ let mut states = Vec::new();
+ flatten_node(scope, &node, &descriptor.root_path, true, &mut states)?;
+ if states.len() != descriptor.file_state_count as usize {
+ return Err(Error::Serialization(
+ "checkpoint chunk state count mismatch".into(),
+ ));
+ }
+ if states.windows(2).any(|pair| pair[0].path >= pair[1].path)
+ || states
+ .iter()
+ .any(|state| !is_canonical_account_path(&state.path, &account_root))
+ {
+ return Err(Error::Serialization(
+ "checkpoint chunk paths are invalid or unordered".into(),
+ ));
+ }
+ if !root {
+ let (start, end) = checkpoint_descriptor_range(descriptor)?;
+ if states
+ .iter()
+ .any(|state| !checkpoint_range_contains(start, end, &state.path))
+ {
+ return Err(Error::Serialization(
+ "checkpoint chunk range mismatch".into(),
+ ));
+ }
+ }
+ Ok(states)
+}
+
+/// Merge old chunk states with replacement states that belong to one target range.
+fn merge_checkpoint_states(
+ states: Vec<FileState>,
+ delta: &SegmentDelta,
+ remaining: &mut BTreeMap<String, FileState>,
+ target: ChunkMergeTarget<'_>,
+) -> Vec<FileState> {
+ let mut merged = states
+ .into_iter()
+ .filter(|state| !checkpoint_state_is_dirty(&state.path, delta))
+ .map(|state| (state.path.clone(), state))
+ .collect::<BTreeMap<_, _>>();
+ let selected = remaining
+ .keys()
+ .filter(|path| checkpoint_target_contains(&target, path))
+ .cloned()
+ .collect::<Vec<_>>();
+ for path in selected {
+ if let Some(state) = remaining.remove(&path) {
+ merged.insert(path, state);
+ }
+ }
+ merged.into_values().collect()
+}
+
+/// Return whether one merge target owns a logical checkpoint path.
+fn checkpoint_target_contains(target: &ChunkMergeTarget<'_>, path: &str) -> bool {
+ match target {
+ ChunkMergeTarget::All => true,
+ ChunkMergeTarget::Range(start, end) => checkpoint_range_contains(start, *end, path),
+ ChunkMergeTarget::RootOnly(root) => path == *root,
+ }
+}
+
+/// Return whether a checkpoint state path is replaced or invalidated by a delta.
+fn checkpoint_state_is_dirty(path: &str, delta: &SegmentDelta) -> bool {
+ delta.exact_paths.contains(path)
+ || delta
+ .prefixes
+ .iter()
+ .any(|prefix| is_path_within(path, prefix))
+}
+
+/// Return whether a non-root chunk range intersects one segment delta.
+fn checkpoint_descriptor_is_dirty(
+ descriptor: &ChunkDescriptor,
+ delta: &SegmentDelta,
+) -> Result<bool> {
+ let (start, end) = checkpoint_descriptor_range(descriptor)?;
+ Ok(delta
+ .exact_paths
+ .iter()
+ .any(|path| checkpoint_range_contains(start, end, path))
+ || delta
+ .states
+ .keys()
+ .any(|path| checkpoint_range_contains(start, end, path))
+ || delta
+ .prefixes
+ .iter()
+ .any(|prefix| checkpoint_range_overlaps_prefix(start, end, prefix)))
+}
+
+/// Return the validated left-closed and right-open range for a non-root descriptor.
+fn checkpoint_descriptor_range(descriptor: &ChunkDescriptor) -> Result<(&str, Option<&str>)> {
+ let start = descriptor
+ .start_path
+ .as_deref()
+ .ok_or_else(|| Error::Serialization("checkpoint child chunk lacks start path".into()))?;
+ let end = descriptor.end_path.as_deref();
+ if let Some(end) = end {
+ if start >= end {
+ return Err(Error::Serialization(
+ "checkpoint chunk range is reversed".into(),
+ ));
+ }
+ }
+ Ok((start, end))
+}
+
+/// Return whether a left-closed range contains one logical path.
+fn checkpoint_range_contains(start: &str, end: Option<&str>, path: &str) -> bool {
+ start <= path && end.is_none_or(|end| path < end)
+}
+
+/// Return whether one ownership range can contain paths under a dirty prefix.
+fn checkpoint_range_overlaps_prefix(start: &str, end: Option<&str>, prefix: &str) -> bool {
+ checkpoint_range_contains(start, end, prefix) || is_path_within(start, prefix)
+}
+
+/// Sort child descriptors and normalize contiguous ownership ranges.
+fn sort_checkpoint_children(children: &mut Vec<ChunkDescriptor>, root_path: &str) -> Result<()> {
+ assign_checkpoint_child_ranges(children, root_path, None)
+}
+
+/// Retain live states and tombstones still needed by at least one backend.
+fn retain_checkpoint_states(states: Vec<FileState>, min_synced: Option<u64>) -> Vec<FileState> {
+ states
+ .into_iter()
+ .filter(|state| {
+ state.deleted.is_none()
+ || min_synced.is_some_and(|synced| state.latest_seq.seq >= synced)
+ })
+ .collect()
+}
+
+/// Return every chunk descriptor reachable from a root descriptor.
+fn checkpoint_chunk_descriptors(root: &ChunkDescriptor) -> Vec<&ChunkDescriptor> {
+ let mut descriptors = Vec::new();
+ let mut stack = vec![root];
+ while let Some(descriptor) = stack.pop() {
+ descriptors.push(descriptor);
+ stack.extend(descriptor.chunks.iter());
+ }
+ descriptors
+}
+
+/// Return every non-empty chunk filename reachable from one descriptor tree.
+fn checkpoint_chunk_files(root: &ChunkDescriptor) -> Vec<String> {
+ checkpoint_chunk_descriptors(root)
+ .into_iter()
+ .filter_map(|descriptor| (!descriptor.file.is_empty()).then(|| descriptor.file.clone()))
+ .collect()
+}
+
+/// Count chunk files already referenced by one descriptor tree.
+fn checkpoint_chunk_count(root: &ChunkDescriptor) -> usize {
+ checkpoint_chunk_files(root).len()
+}
+
+/// Best-effort delete candidate chunk files no longer referenced after a rewrite.
+async fn remove_replaced_checkpoint_files(
+ store: &MetadataStore,
+ scope: &ScopeKey,
+ candidate: &str,
+ old_files: Vec<String>,
+ new_root: &ChunkDescriptor,
+) {
+ let Ok(directory) = checkpoint_directory(store, scope, candidate) else {
+ return;
+ };
+ let retained = checkpoint_chunk_files(new_root)
+ .into_iter()
+ .collect::<BTreeSet<_>>();
+ for file in old_files {
+ if retained.contains(&file) {
+ continue;
+ }
+ if let Err(error) = store.remove_file(&format!("{directory}/{file}")).await {
+ warn!(file = %file, error = %error, "checkpoint chunk cleanup failed");
+ }
+ }
+}
+
+/// Stream-validate a candidate checkpoint without materializing all file states.
+async fn verify_checkpoint(
+ store: &MetadataStore,
+ scope: &ScopeKey,
+ manifest: &CheckpointManifest,
+ candidate: &str,
+ tail: u64,
+) -> Result<()> {
+ if tree_checksum(&manifest.root)? != manifest.checksum {
+ return Err(Error::Serialization(
+ "checkpoint tree checksum mismatch".into(),
+ ));
+ }
+ let directory = checkpoint_directory(store, scope, candidate)?;
+ let mut descriptors = vec![(&manifest.root, true)];
+ let mut previous_path: Option<String> = None;
+ while let Some((descriptor, root)) = descriptors.pop() {
+ let states =
+ read_checkpoint_chunk_states(store, scope, &directory, descriptor, root).await?;
+ for state in states {
+ if state.latest_seq.seq > tail
+ || previous_path
+ .as_ref()
+ .is_some_and(|previous| previous >= &state.path)
+ {
+ return Err(Error::Serialization(
+ "checkpoint candidate verification failed".into(),
+ ));
+ }
+ previous_path = Some(state.path);
+ }
+ descriptors.extend(descriptor.chunks.iter().rev().map(|child| (child, false)));
+ }
+ Ok(())
+}
+
+/// Read every referenced chunk and reconstruct sorted file states.
+async fn read_chunks(
+ store: &MetadataStore,
+ scope: &ScopeKey,
+ manifest: &CheckpointManifest,
+ candidate: &str,
+) -> Result<Vec<FileState>> {
+ let directory = checkpoint_directory(store, scope, candidate)?;
+ let mut descriptors = vec![(&manifest.root, true)];
+ let mut states = Vec::new();
+ while let Some((descriptor, root)) = descriptors.pop() {
+ states.extend(
+ read_checkpoint_chunk_states(store, scope, &directory, descriptor, root).await?,
+ );
+ descriptors.extend(descriptor.chunks.iter().rev().map(|child| (child, false)));
+ }
+ if states.windows(2).any(|pair| pair[0].path >= pair[1].path) {
+ return Err(Error::Serialization(
+ "checkpoint paths are not strictly ordered".into(),
+ ));
+ }
+ Ok(states)
+}
+
+/// Flatten one decoded checkpoint node into validated file states.
+fn flatten_node(
+ scope: &ScopeKey,
+ node: &CheckpointNode,
+ parent: &str,
+ root: bool,
+ states: &mut Vec<FileState>,
+) -> Result<()> {
+ if (!root || !node.path_fragment.is_empty()) && !valid_path_fragment(&node.path_fragment) {
+ return Err(Error::Serialization(
+ "checkpoint path fragment is not relative".into(),
+ ));
+ }
+ let path = format!("{parent}{}", node.path_fragment);
+ if let Some(seq) = node.latest_seq {
+ let state = FileState {
+ path: path.clone(),
+ latest_seq: ScopedSeq {
+ scope: scope.clone(),
+ seq,
+ },
+ deleted: node.deleted,
+ };
+ state.validate(scope)?;
+ states.push(state);
+ }
+ for child in &node.children {
+ flatten_node(scope, child, &path, false, states)?;
+ }
+ Ok(())
+}
+
+/// Resolve one isolated checkpoint directory from its relative latest pointer.
+fn checkpoint_directory(
+ store: &MetadataStore,
+ scope: &ScopeKey,
+ candidate: &str,
+) -> Result<String> {
+ let name = candidate.strip_suffix('/').ok_or_else(|| {
+ Error::Serialization("checkpoint pointer must reference a directory".into())
+ })?;
+ if !is_checkpoint_directory_name(name) {
+ return Err(Error::Serialization(
+ "checkpoint pointer directory is invalid".into(),
+ ));
+ }
+ let root = store
+ .paths()
+ .checkpoints_dir(&scope.account_id, scope.partition_id)?
+ .0;
+ Ok(format!("{root}/{name}"))
+}
+
+/// Return whether one radix edge is a non-empty relative fragment.
+fn valid_path_fragment(fragment: &str) -> bool {
+ !fragment.is_empty() && !fragment.contains('\0')
+}
+
+/// Return whether a path is canonical and belongs to one account root.
+fn is_canonical_account_path(path: &str, root: &str) -> bool {
+ is_path_within(path, root)
+ && !path.ends_with('/')
+ && path
+ .split('/')
+ .skip(1)
+ .all(|part| !part.is_empty() && part != "." && part != "..")
+}
+
+/// Hash the canonical serialized descriptor tree.
+fn tree_checksum(root: &ChunkDescriptor) -> Result<String> {
+ Ok(sha256_hex(&serde_json::to_vec(root)?))
+}
+
+/// Return whether a path equals a prefix or is one of its descendants.
+fn is_path_within(path: &str, prefix: &str) -> bool {
+ path == prefix
+ || path
+ .strip_prefix(prefix)
+ .is_some_and(|suffix| suffix.starts_with('/'))
+}
+
+/// Return the current Unix timestamp in nanoseconds.
+fn now_ns() -> Result<u64> {
+ let nanos = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .map_err(|error| Error::internal(error.to_string()))?
+ .as_nanos();
+ u64::try_from(nanos).map_err(|_| Error::internal("checkpoint timestamp overflow"))
+}
+
+/// Preserve an operation result while requiring lease release to succeed.
+fn finish_with_release<T>(operation: Result<T>, release: Result<()>) -> Result<T> {
+ match (operation, release) {
+ (Ok(value), Ok(())) => Ok(value),
+ (Err(error), _) => Err(error),
+ (Ok(_), Err(error)) => Err(error),
+ }
+}
diff --git a/crates/ragfs/src/multibackend/codec.rs b/crates/ragfs/src/multibackend/codec.rs
new file mode 100644
index 000000000..bd0e527a5
--- /dev/null
+++ b/crates/ragfs/src/multibackend/codec.rs
@@ -0,0 +1,359 @@
+//! Stable binary codecs for V2 segment and checkpoint files.
+
+use sha2::{Digest, Sha256};
+use uuid::Uuid;
+
+use crate::core::errors::{Error, Result};
+use crate::multibackend::constants::{MAX_CHECKPOINT_NODES, MAX_SEGMENT_RECORDS};
+use crate::multibackend::model::{CheckpointNode, DecodedSegment, SegmentEventType, SegmentRecord};
+
+const SEGMENT_MAGIC: &[u8; 4] = b"OVSG";
+const CHECKPOINT_MAGIC: &[u8; 4] = b"OVCP";
+const FORMAT_VERSION: u16 = 1;
+const MAX_STRING_LEN: usize = 1024 * 1024;
+const MIN_RECORD_LEN: usize = 14;
+const MAX_RECORD_LEN: usize = 2 * 1024 * 1024;
+const MAX_CHECKPOINT_DEPTH: usize = 256;
+
+/// Encode segment records in the stable OVSG binary format.
+pub fn encode_segment(records: &[SegmentRecord]) -> Result<Vec<u8>> {
+ if records.len() > MAX_SEGMENT_RECORDS {
+ return Err(serialization("segment has too many records"));
+ }
+
+ let mut bytes = Vec::new();
+ bytes.extend_from_slice(SEGMENT_MAGIC);
+ bytes.extend_from_slice(&FORMAT_VERSION.to_le_bytes());
+ for record in records {
+ record.validate()?;
+ let frame = encode_record(record)?;
+ if frame.len() > MAX_RECORD_LEN {
+ return Err(serialization("segment record is too large"));
+ }
+ bytes.extend_from_slice(&(frame.len() as u32).to_le_bytes());
+ bytes.extend_from_slice(&frame);
+ }
+ Ok(bytes)
+}
+
+/// Decode complete segment records and recover a truncated final frame.
+pub fn decode_segment(bytes: &[u8]) -> Result<DecodedSegment> {
+ let mut reader = Reader::new(bytes);
+ read_header(&mut reader, SEGMENT_MAGIC)?;
+ let mut records = Vec::new();
+ let mut valid_bytes = reader.position();
+ let mut truncated_tail = false;
+
+ while reader.remaining() > 0 {
+ if records.len() == MAX_SEGMENT_RECORDS {
+ return Err(serialization("segment has too many records"));
+ }
+ let frame_start = reader.position();
+ if reader.remaining() < 4 {
+ truncated_tail = true;
+ break;
+ }
+ let frame_len = reader.read_u32()? as usize;
+ if frame_len < MIN_RECORD_LEN {
+ return Err(serialization("segment record is too small"));
+ }
+ if frame_len > MAX_RECORD_LEN {
+ return Err(serialization("segment record is too large"));
+ }
+ if frame_len > reader.remaining() {
+ truncated_tail = true;
+ break;
+ }
+ let mut frame = Reader::new(reader.read_exact(frame_len)?);
+ let record = decode_record(&mut frame)?;
+ if frame.remaining() != 0 {
+ return Err(serialization("segment record has trailing bytes"));
+ }
+ records.push(record);
+ valid_bytes = reader.position();
+ debug_assert!(valid_bytes > frame_start);
+ }
+
+ Ok(DecodedSegment {
+ records,
+ valid_bytes: valid_bytes as u64,
+ truncated_tail,
+ })
+}
+
+/// Encode one checkpoint tree in the stable OVCP binary format.
+pub fn encode_checkpoint_chunk(root: &CheckpointNode) -> Result<Vec<u8>> {
+ let node_count = validate_checkpoint_shape(root)?;
+ root.validate()?;
+
+ let mut bytes = Vec::new();
+ bytes.extend_from_slice(CHECKPOINT_MAGIC);
+ bytes.extend_from_slice(&FORMAT_VERSION.to_le_bytes());
+ bytes.extend_from_slice(&(node_count as u32).to_le_bytes());
+ encode_checkpoint_node(root, &mut bytes)?;
+ Ok(bytes)
+}
+
+/// Decode one complete OVCP checkpoint tree.
+pub fn decode_checkpoint_chunk(bytes: &[u8]) -> Result<CheckpointNode> {
+ let mut reader = Reader::new(bytes);
+ read_header(&mut reader, CHECKPOINT_MAGIC)?;
+ let node_count = reader.read_u32()? as usize;
+ if node_count == 0 || node_count > MAX_CHECKPOINT_NODES {
+ return Err(serialization("invalid checkpoint node count"));
+ }
+
+ let mut remaining_nodes = node_count;
+ let root = decode_checkpoint_node(&mut reader, &mut remaining_nodes, 1)?;
+ if remaining_nodes != 0 {
+ return Err(serialization(
+ "checkpoint node count does not match payload",
+ ));
+ }
+ if reader.remaining() != 0 {
+ return Err(serialization("checkpoint has trailing bytes"));
+ }
+ root.validate()?;
+ Ok(root)
+}
+
+/// Return a lowercase hexadecimal SHA-256 digest.
+pub fn sha256_hex(bytes: &[u8]) -> String {
+ format!("{:x}", Sha256::digest(bytes))
+}
+
+/// Encode one validated segment record payload.
+fn encode_record(record: &SegmentRecord) -> Result<Vec<u8>> {
+ let mut bytes = Vec::new();
+ bytes.extend_from_slice(&record.seq.to_le_bytes());
+ bytes.push(record.event_type as u8);
+ let flags =
+ u8::from(record.op_id.is_some()) | (u8::from(record.destination_path.is_some()) << 1);
+ bytes.push(flags);
+ write_string(&mut bytes, &record.path)?;
+ if let Some(op_id) = record.op_id {
+ bytes.extend_from_slice(op_id.as_bytes());
+ }
+ if let Some(destination) = &record.destination_path {
+ write_string(&mut bytes, destination)?;
+ }
+ Ok(bytes)
+}
+
+/// Decode and validate one complete segment record payload.
+fn decode_record(reader: &mut Reader<'_>) -> Result<SegmentRecord> {
+ let seq = reader.read_u64()?;
+ let event_type = match reader.read_u8()? {
+ 1 => SegmentEventType::Write,
+ 2 => SegmentEventType::Remove,
+ 3 => SegmentEventType::RemoveTree,
+ 4 => SegmentEventType::MoveTree,
+ 5 => SegmentEventType::DeleteScope,
+ _ => return Err(serialization("unknown segment event type")),
+ };
+ let flags = reader.read_u8()?;
+ if flags & !0b11 != 0 {
+ return Err(serialization("unknown segment record flags"));
+ }
+ let path = reader.read_string()?;
+ let op_id = if flags & 1 != 0 {
+ let raw: [u8; 16] = reader
+ .read_exact(16)?
+ .try_into()
+ .map_err(|_| serialization("invalid operation id"))?;
+ Some(Uuid::from_bytes(raw))
+ } else {
+ None
+ };
+ let destination_path = if flags & 2 != 0 {
+ Some(reader.read_string()?)
+ } else {
+ None
+ };
+ let record = SegmentRecord {
+ seq,
+ event_type,
+ path,
+ op_id,
+ destination_path,
+ };
+ record.validate()?;
+ Ok(record)
+}
+
+/// Validate checkpoint size and depth without recursive traversal.
+fn validate_checkpoint_shape(root: &CheckpointNode) -> Result<usize> {
+ let mut stack = vec![(root, 1_usize)];
+ let mut node_count = 0_usize;
+ while let Some((node, depth)) = stack.pop() {
+ if depth > MAX_CHECKPOINT_DEPTH {
+ return Err(serialization("checkpoint exceeds maximum depth"));
+ }
+ node_count += 1;
+ if node_count > MAX_CHECKPOINT_NODES {
+ return Err(serialization("checkpoint has too many nodes"));
+ }
+ if node.path_fragment.len() > MAX_STRING_LEN {
+ return Err(serialization("checkpoint path fragment is too large"));
+ }
+ stack.extend(node.children.iter().map(|child| (child, depth + 1)));
+ }
+ Ok(node_count)
+}
+
+/// Encode one checkpoint node after iterative shape validation.
+fn encode_checkpoint_node(node: &CheckpointNode, bytes: &mut Vec<u8>) -> Result<()> {
+ write_string(bytes, &node.path_fragment)?;
+ let flags = u8::from(node.latest_seq.is_some()) | (u8::from(node.deleted.is_some()) << 1);
+ bytes.push(flags);
+ if let Some(seq) = node.latest_seq {
+ bytes.extend_from_slice(&seq.to_le_bytes());
+ }
+ bytes.extend_from_slice(&(node.children.len() as u32).to_le_bytes());
+ for child in &node.children {
+ encode_checkpoint_node(child, bytes)?;
+ }
+ Ok(())
+}
+
+/// Decode one node while enforcing global node and depth budgets before allocation.
+fn decode_checkpoint_node(
+ reader: &mut Reader<'_>,
+ remaining_nodes: &mut usize,
+ depth: usize,
+) -> Result<CheckpointNode> {
+ if depth > MAX_CHECKPOINT_DEPTH {
+ return Err(serialization("checkpoint exceeds maximum depth"));
+ }
+ if *remaining_nodes == 0 {
+ return Err(serialization("checkpoint contains too many nodes"));
+ }
+ *remaining_nodes -= 1;
+
+ let path_fragment = reader.read_string()?;
+ let flags = reader.read_u8()?;
+ if flags & !0b11 != 0 {
+ return Err(serialization("unknown checkpoint node flags"));
+ }
+ let latest_seq = if flags & 1 != 0 {
+ Some(reader.read_u64()?)
+ } else {
+ None
+ };
+ let deleted = if flags & 2 != 0 { Some(true) } else { None };
+ let child_count = reader.read_u32()? as usize;
+ if child_count > *remaining_nodes {
+ return Err(serialization(
+ "checkpoint child count exceeds remaining node budget",
+ ));
+ }
+
+ let mut children = Vec::new();
+ for _ in 0..child_count {
+ children.push(decode_checkpoint_node(reader, remaining_nodes, depth + 1)?);
+ }
+ Ok(CheckpointNode {
+ path_fragment,
+ latest_seq,
+ deleted,
+ children,
+ })
+}
+
+/// Append one bounded length-prefixed UTF-8 string.
+fn write_string(bytes: &mut Vec<u8>, value: &str) -> Result<()> {
+ if value.len() > MAX_STRING_LEN {
+ return Err(serialization("string is too large"));
+ }
+ bytes.extend_from_slice(&(value.len() as u32).to_le_bytes());
+ bytes.extend_from_slice(value.as_bytes());
+ Ok(())
+}
+
+/// Read and validate a format magic and version.
+fn read_header(reader: &mut Reader<'_>, magic: &[u8; 4]) -> Result<()> {
+ if reader.read_exact(4)? != magic {
+ return Err(serialization("invalid binary format magic"));
+ }
+ if reader.read_u16()? != FORMAT_VERSION {
+ return Err(serialization("unsupported binary format version"));
+ }
+ Ok(())
+}
+
+/// Build a serialization error with a concise message.
+fn serialization(message: impl Into<String>) -> Error {
+ Error::Serialization(message.into())
+}
+
+struct Reader<'a> {
+ bytes: &'a [u8],
+ position: usize,
+}
+
+impl<'a> Reader<'a> {
+ /// Create a reader positioned at the start of a byte slice.
+ fn new(bytes: &'a [u8]) -> Self {
+ Self { bytes, position: 0 }
+ }
+
+ /// Return the current byte offset.
+ fn position(&self) -> usize {
+ self.position
+ }
+
+ /// Return the number of unread bytes.
+ fn remaining(&self) -> usize {
+ self.bytes.len() - self.position
+ }
+
+ /// Read an exact byte range or reject truncated input.
+ fn read_exact(&mut self, length: usize) -> Result<&'a [u8]> {
+ let end = self
+ .position
+ .checked_add(length)
+ .ok_or_else(|| serialization("binary length overflow"))?;
+ let value = self
+ .bytes
+ .get(self.position..end)
+ .ok_or_else(|| serialization("truncated binary data"))?;
+ self.position = end;
+ Ok(value)
+ }
+
+ /// Read one unsigned byte.
+ fn read_u8(&mut self) -> Result<u8> {
+ Ok(self.read_exact(1)?[0])
+ }
+
+ /// Read one little-endian u16.
+ fn read_u16(&mut self) -> Result<u16> {
+ Ok(u16::from_le_bytes(self.read_exact(2)?.try_into().unwrap()))
+ }
+
+ /// Read one little-endian u32.
+ fn read_u32(&mut self) -> Result<u32> {
+ Ok(u32::from_le_bytes(self.read_exact(4)?.try_into().unwrap()))
+ }
+
+ /// Read one little-endian u64.
+ fn read_u64(&mut self) -> Result<u64> {
+ Ok(u64::from_le_bytes(self.read_exact(8)?.try_into().unwrap()))
+ }
+
+ /// Read one bounded length-prefixed UTF-8 string.
+ fn read_string(&mut self) -> Result<String> {
+ let length = self.read_u32()? as usize;
+ if length > MAX_STRING_LEN {
+ return Err(serialization("string is too large"));
+ }
+ let raw = self.read_exact(length)?;
+ let value =
+ std::str::from_utf8(raw).map_err(|_| serialization("string is not valid UTF-8"))?;
+ Ok(value.to_owned())
+ }
+}
+
+#[cfg(test)]
+#[path = "tests/test_codec.rs"]
+mod tests;
diff --git a/crates/ragfs/src/multibackend/config.rs b/crates/ragfs/src/multibackend/config.rs
index be305f73c..f2d0f581c 100644
--- a/crates/ragfs/src/multibackend/config.rs
+++ b/crates/ragfs/src/multibackend/config.rs
@@ -1,12 +1,11 @@
//! Multi-backend config validation and normalization helpers.
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
use serde_json::Value;
use crate::core::errors::{Error, Result};
-use crate::core::multibackend_wrapper::{BackendEntry, SyncMode};
-use crate::core::types::{BackendsConfig, ConfigValue, PluginConfig, RedirectPolicy};
+use crate::core::types::{BackendsConfig, ConfigValue, PluginConfig};
/// Convert one nested backup `params` object into plugin config values.
pub fn item_params_to_config_values(value: &Value) -> Result<HashMap<String, ConfigValue>> {
@@ -66,62 +65,32 @@ pub fn validate_primary_encryption_flags(
Ok(())
}
-/// Validate redirect targets against the assembled backup entries.
-pub fn validate_redirect_targets(
- redirects: &[RedirectPolicy],
- backup_entries: &[BackendEntry],
-) -> Result<()> {
- for policy in redirects {
- let targets = match policy {
- RedirectPolicy::FileOverSizePolicy { target, .. } => target.as_ref(),
- RedirectPolicy::FileExtensionPolicy { target, .. } => target.as_ref(),
- };
- let target_names = targets
- .ok_or_else(|| Error::config("redirect policy target must not be empty".to_string()))?;
- if target_names.is_empty() {
+/// Validate V2 backup settings.
+pub fn validate_backups_config(bc: &BackendsConfig) -> Result<()> {
+ if !matches!(bc.provider.as_str(), "filesystem" | "cache") {
+ return Err(Error::config(
+ "backups.provider must be 'filesystem' or 'cache'".to_string(),
+ ));
+ }
+
+ let mut names = HashSet::new();
+ for item in &bc.items {
+ let name = item.name.trim();
+ if name.is_empty() {
+ return Err(Error::config("backup name must not be empty".to_string()));
+ }
+ if name == "primary" {
return Err(Error::config(
- "redirect policy target must not be empty".to_string(),
+ "backup backend name 'primary' is reserved".to_string(),
));
}
- for name in target_names {
- if !backup_entries.iter().any(|be| &be.name == name) {
- return Err(Error::config(format!(
- "redirect target '{}' not found in backup entries",
- name
- )));
- }
+ if !names.insert(name) {
+ return Err(Error::config(format!(
+ "duplicate backup name '{}'",
+ item.name
+ )));
}
}
- Ok(())
-}
-/// Validate that exclude policies do not silently carry redirect targets.
-pub fn validate_backup_excludes(bc: &BackendsConfig) -> Result<()> {
- for item in &bc.items {
- for policy in item.excludes.as_deref().unwrap_or(&[]) {
- match policy {
- RedirectPolicy::FileOverSizePolicy { target, .. }
- | RedirectPolicy::FileExtensionPolicy { target, .. } => {
- if target.is_some() {
- return Err(Error::config(format!(
- "exclude policy for backup '{}' must not contain target",
- item.name
- )));
- }
- }
- }
- }
- }
Ok(())
}
-
-/// Convert config sync settings into the runtime sync mode.
-pub fn sync_mode_from_config(bc: &BackendsConfig) -> SyncMode {
- match bc.sync_type.as_str() {
- "sync" => SyncMode::Sync {
- ack_count: bc.write_ack_count.unwrap_or(usize::MAX),
- timeout_ms: bc.write_ack_timeout_ms.unwrap_or(0),
- },
- _ => SyncMode::Async,
- }
-}
diff --git a/crates/ragfs/src/multibackend/constants.rs b/crates/ragfs/src/multibackend/constants.rs
new file mode 100644
index 000000000..160f71154
--- /dev/null
+++ b/crates/ragfs/src/multibackend/constants.rs
@@ -0,0 +1,38 @@
+//! Shared constants for V2 multi-write metadata.
+
+/// Canonical logical mount prefix used by V2 multi-write metadata.
+pub(crate) const MULTIWRITE_MOUNT_PREFIX: &str = "/local";
+/// Internal metadata directory name.
+pub(crate) const SYSTEM_DIR: &str = "_system";
+/// Mount-level protocol manifest file name.
+pub(crate) const MULTIWRITE_PROTOCOL_FILE: &str = ".multiwrite.json";
+/// Account partition directory name.
+pub(crate) const PARTITIONS_DIR: &str = "partitions";
+/// Standard manifest file name.
+pub(crate) const MANIFEST_FILE: &str = "manifest.json";
+/// Segment blob directory name.
+pub(crate) const SEGMENTS_DIR: &str = "segments";
+/// Checkpoint directory name.
+pub(crate) const CHECKPOINTS_DIR: &str = "checkpoints";
+
+/// Mutable head segment file prefix.
+pub(crate) const HEAD_SEGMENT_FILE_PREFIX: &str = "head-";
+/// Immutable sealed segment file prefix.
+pub(crate) const SEALED_SEGMENT_FILE_PREFIX: &str = "segment-";
+/// Segment blob file extension.
+pub(crate) const SEGMENT_FILE_EXTENSION: &str = ".ovsg";
+/// Checkpoint candidate directory prefix.
+pub(crate) const CHECKPOINT_DIR_PREFIX: &str = "cp-";
+/// Checkpoint candidate timestamp format.
+pub(crate) const CHECKPOINT_DIR_TIME_FORMAT: &str = "%Y%m%d%H%M%S";
+/// Checkpoint chunk file prefix.
+pub(crate) const CHECKPOINT_CHUNK_FILE_PREFIX: &str = "chunk-";
+/// Checkpoint chunk file extension.
+pub(crate) const CHECKPOINT_CHUNK_FILE_EXTENSION: &str = ".ovcp";
+
+/// Total number of fixed routing buckets.
+pub const VBUCKETS: u32 = 65_536;
+/// Maximum records allowed in one segment.
+pub(crate) const MAX_SEGMENT_RECORDS: usize = 8_192;
+/// Maximum file states allowed in one checkpoint chunk.
+pub(crate) const MAX_CHECKPOINT_NODES: usize = 500_000;
\ No newline at end of file
diff --git a/crates/ragfs/src/multibackend/factory.rs b/crates/ragfs/src/multibackend/factory.rs
index 0f657d850..da9d8d4bc 100644
--- a/crates/ragfs/src/multibackend/factory.rs
+++ b/crates/ragfs/src/multibackend/factory.rs
@@ -1,6 +1,6 @@
//! Multi-backend runtime assembly from validated mount configuration.
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
@@ -13,10 +13,12 @@ use crate::core::multibackend_wrapper::{BackendEntry, MultiWriteWrappedFS};
use crate::core::plugin::ServicePlugin;
use crate::core::types::{BackendRole, BackendsConfig, ConfigValue, PluginConfig};
use crate::multibackend::config::{
- item_params_to_config_values, sync_mode_from_config, validate_backup_excludes,
- validate_primary_encryption_flags, validate_redirect_targets,
+ item_params_to_config_values, validate_backups_config, validate_primary_encryption_flags,
};
-use crate::multibackend::meta::RelativePathFsContextResolver;
+use crate::multibackend::meta::{MetadataStore, RelativePathFsContextResolver};
+#[cfg(feature = "cache")]
+use crate::multibackend::provider::CacheProvider;
+use crate::multibackend::provider::{FilesystemProvider, MultiWriteProvider};
use crate::multibackend::types::MultiBackendBuildContext;
use crate::shape::validate::ensure_backend_shape;
@@ -51,14 +53,15 @@ pub async fn init_backend_plugin(
Ok(Arc::from(fs))
}
-/// Build the multi-backend wrapper from primary and backup configuration.
-pub async fn build_multi_write_fs(
+/// Build the multi-backend wrapper without starting background work.
+pub(crate) async fn build_inactive_multi_write_fs(
registry: &Arc<RwLock<HashMap<String, Arc<dyn ServicePlugin>>>>,
config: &PluginConfig,
bc: &BackendsConfig,
build_ctx: MultiBackendBuildContext,
) -> Result<MultiWriteWrappedFS> {
let global_encryption_enabled = build_ctx.global_encryption_enabled();
+ validate_backups_config(bc)?;
validate_primary_encryption_flags(config, global_encryption_enabled)?;
let primary_raw = init_backend_plugin(registry, &config.name, &config.params).await?;
@@ -92,22 +95,37 @@ pub async fn build_multi_write_fs(
primary_raw.clone()
};
- let mut backup_entries: Vec<BackendEntry> = Vec::new();
- let mut seen_names: HashSet<String> = HashSet::new();
-
- for item in &bc.items {
- if item.name == "primary" {
- return Err(Error::config(
- "backup backend name 'primary' is reserved".to_string(),
- ));
- }
- if !seen_names.insert(item.name.clone()) {
- return Err(Error::config(format!(
- "duplicate backup name '{}'",
- item.name
- )));
+ let metadata_store = Arc::new(MetadataStore::new(
+ primary_backend.clone(),
+ build_ctx.pathlock_manager.clone(),
+ &build_ctx.backend_prefix,
+ )?);
+ let metadata_provider: Arc<dyn MultiWriteProvider> = match bc.provider.as_str() {
+ "filesystem" => Arc::new(FilesystemProvider::new(metadata_store.clone())),
+ "cache" => {
+ #[cfg(feature = "cache")]
+ {
+ let runtime = build_ctx.cache_runtime.clone().ok_or_else(|| {
+ Error::config("backups.provider = 'cache' requires a top-level CacheRuntime")
+ })?;
+ Arc::new(CacheProvider::new(
+ metadata_store.clone(),
+ runtime,
+ bc.namespace.clone(),
+ )?)
+ }
+ #[cfg(not(feature = "cache"))]
+ {
+ return Err(Error::config(
+ "backups.provider = 'cache' requires the ragfs cache feature",
+ ));
+ }
}
+ _ => unreachable!("backup provider validated before construction"),
+ };
+ let mut backup_entries: Vec<BackendEntry> = Vec::new();
+ for item in &bc.items {
let backup_params = item_params_to_config_values(&item.params)?;
let backup_raw = init_backend_plugin(registry, &item.backend, &backup_params).await?;
let backup_encrypted = global_encryption_enabled
@@ -160,26 +178,17 @@ pub async fn build_multi_write_fs(
role: BackendRole::Backup,
backend: backup_backend,
raw_backend: None,
- operations: item.operations.clone().unwrap_or_default(),
- excludes: item.excludes.clone().unwrap_or_default(),
});
}
- validate_redirect_targets(&config.primary_redirects, &backup_entries)?;
- validate_backup_excludes(bc)?;
-
MultiWriteWrappedFS::builder(primary_backend)
.with_primary_raw_backend(primary_raw)
+ .with_metadata_store(metadata_store)
+ .with_metadata_provider(metadata_provider)
.with_backups(backup_entries)
- .with_redirects(config.primary_redirects.clone())
- .sync_mode(sync_mode_from_config(bc))
- .write_concurrency(bc.write_concurrency)
- .retry_interval(Duration::from_millis(
- bc.retry_interval_ms.unwrap_or(30_000),
- ))
- .retry_backoff_base_ms(bc.retry_backoff_base_ms.unwrap_or(1_000))
- .max_retry_per_round(bc.retry_max_retries_per_round.unwrap_or(3))
- .quarantine_after_failures(bc.retry_quarantine_after_failures.unwrap_or(9))
+ .initial_partitions(bc.initial_partitions)
+ .checkpoint_interval(Duration::from_secs(bc.checkpoint_interval_secs))
.ctx_resolver(Arc::new(RelativePathFsContextResolver))
- .build()
+ .build_inactive()
+ .await
}
diff --git a/crates/ragfs/src/multibackend/gc.rs b/crates/ragfs/src/multibackend/gc.rs
new file mode 100644
index 000000000..64507ca22
--- /dev/null
+++ b/crates/ragfs/src/multibackend/gc.rs
@@ -0,0 +1,250 @@
+//! Garbage collection for versioned multi-write metadata.
+
+use std::collections::BTreeMap;
+use std::sync::Arc;
+use std::time::Duration;
+
+use crate::core::errors::{Error, Result};
+use crate::multibackend::constants::SEGMENT_FILE_EXTENSION;
+use crate::multibackend::meta::{MetadataStore, MultiWriteWorker};
+use crate::multibackend::model::{is_checkpoint_directory_name, CheckpointsManifest, DirectoryEvent, LatestCheckpoint,PartitionState, PartitionsManifest, ProtocolStatus, ScopeKey, SegmentManifest,};
+use crate::multibackend::provider::MultiWriteProvider;
+
+struct ScopeSnapshot { latest: Option<LatestCheckpoint>, segments: SegmentManifest}
+
+/// Collects obsolete segment and checkpoint metadata after publication.
+pub struct MetadataGc {
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+}
+
+impl MetadataGc {
+ /// Create a collector over the shared metadata store and segment provider.
+ pub fn new(store: Arc<MetadataStore>, provider: Arc<dyn MultiWriteProvider>) -> Self {
+ Self { store, provider }
+ }
+
+ /// Collect one scope while treating concurrent account deletion as complete.
+ pub async fn collect_scope(&self, scope: &ScopeKey) -> Result<()> {
+ match self.collect_scope_inner(scope).await {
+ Err(Error::NotFound(_)) => Ok(()),
+ Err(error) => {
+ self.store.record_worker_error(MultiWriteWorker::Gc);
+ Err(error)
+ }
+ result => result,
+ }
+ }
+
+ /// Apply account safety checks before collecting one Stable scope.
+ async fn collect_scope_inner(&self, scope: &ScopeKey) -> Result<()> {
+ if self.store.protocol_status().await? != ProtocolStatus::Stable {
+ return Ok(());
+ }
+ scope.validate()?;
+ let account = self.store.update_partitions_manifest(&scope.account_id,
+ |manifest| Ok(manifest.clone())).await?;
+ if account.partitions.values().any(|entry| entry.state != PartitionState::Stable)
+ || account.epoch != scope.epoch
+ || !account.partitions.contains_key(&scope.partition_id)
+ {
+ return Ok(());
+ }
+ let stable = stable_partitions(&account);
+ let positions_complete =
+ directory_positions_complete(&account.directory_events, &stable, account.epoch);
+ let retained = if positions_complete {
+ self.collect_completed_events(&scope.account_id, &account, &stable).await?
+ } else {
+ Some(account.directory_events.clone())
+ };
+ let Some(retained) = retained else {
+ return Ok(());
+ };
+ let allow_segments =
+ positions_complete && directory_positions_complete(&retained, &stable, account.epoch);
+ let latest = self.read_latest(scope).await?;
+ if allow_segments {
+ self.collect_segments(scope, latest.as_ref(), &retained).await?;
+ }
+ self.collect_checkpoints(scope).await
+ }
+
+ /// Remove completed events only if their account identity and content remain unchanged.
+ async fn collect_completed_events(&self, account_id: &str,
+ account: &PartitionsManifest, stable: &[u32]) -> Result<Option<Vec<DirectoryEvent>>> {
+ let mut snapshots = BTreeMap::new();
+ for &partition_id in stable {
+ let scope = ScopeKey { account_id: account_id.to_string(),
+ partition_id, epoch: account.epoch };
+ snapshots.insert(partition_id, self.read_scope_snapshot(&scope).await?);
+ }
+ let completed = account.directory_events.iter()
+ .filter(|event| event_is_complete(event, &snapshots))
+ .cloned().collect::<Vec<_>>();
+ let account_id = account_id.to_string();
+ let expected_epoch = account.epoch;
+ let expected_stable = stable.to_vec();
+ self.store.update_partitions_manifest(&account_id, move |current| {
+ if current.epoch != expected_epoch
+ || stable_partitions(current) != expected_stable
+ {
+ return Ok(None);
+ }
+ current.directory_events.retain(|event|
+ !completed.iter().any(|candidate| candidate == event));
+ Ok(Some(current.directory_events.clone()))
+ }).await
+ }
+
+ /// Read a checkpoint pointer briefly and pair it with segment progress.
+ async fn read_scope_snapshot(&self, scope: &ScopeKey) -> Result<ScopeSnapshot> {
+ let latest = self.read_latest(scope).await?;
+ let segments = self.provider.read_manifest(scope).await?;
+ segments.validate()?;
+ Ok(ScopeSnapshot { latest, segments })
+ }
+
+ /// Read the latest checkpoint while holding only its pointer Exact lock.
+ async fn read_latest(&self, scope: &ScopeKey) -> Result<Option<LatestCheckpoint>> {
+ let path = self.store.paths().checkpoints_manifest(
+ &scope.account_id, scope.partition_id)?.0;
+ let lease = self.store.pathlock_manager().acquire_exact(
+ &path,
+ self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)),
+ None,
+ ).await?;
+ let operation = async {
+ let pointer: CheckpointsManifest = self.store.read_json(&path).await?;
+ pointer.validate()?;
+ Ok(pointer.latest_checkpoint)
+ }.await;
+ let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from);
+ finish_with_release(operation, release)
+ }
+
+ /// Prune one eligible sealed prefix and remove every unreferenced segment blob.
+ async fn collect_segments(&self, scope: &ScopeKey, latest: Option<&LatestCheckpoint>,
+ retained: &[DirectoryEvent]) -> Result<()> {
+ let manifest = self.provider.read_manifest(scope).await?;
+ manifest.validate()?;
+ let removable = manifest.segments.iter().take_while(|descriptor| {
+ descriptor.segment_to_seq.is_some_and(|to| {
+ latest.is_some_and(|checkpoint| checkpoint.checkpoint_to_seq >= to)
+ && manifest.backend_states.values().all(|state| state.synced_seq >= to)
+ && !retained.iter().any(|event| event.positions.iter().any(|position| {
+ position.partition_id == scope.partition_id
+ && position.epoch == scope.epoch
+ && (descriptor.segment_from_seq..=to).contains(&position.seq)
+ }))
+ })
+ }).map(|descriptor| descriptor.path.clone()).collect::<Vec<_>>();
+ if !removable.is_empty() {
+ for descriptor in self.provider.prune_sealed(scope, &removable).await? {
+ self.remove_segment(scope, &descriptor.path).await?;
+ }
+ }
+ let directory = self.store.paths().segments_dir(
+ &scope.account_id, scope.partition_id)?.0;
+ for entry in self.store.list_directory(&directory).await? {
+ if !entry.is_dir && entry.name.ends_with(SEGMENT_FILE_EXTENSION) {
+ self.remove_segment(scope, &entry.name).await?;
+ }
+ }
+ Ok(())
+ }
+
+ /// Delete one segment blob while holding its Exact path lock.
+ async fn remove_segment(&self, scope: &ScopeKey, name: &str) -> Result<()> {
+ let directory = self.store.paths().segments_dir(
+ &scope.account_id, scope.partition_id)?.0;
+ let path = format!("{directory}/{name}");
+ let lease = self.store.pathlock_manager().acquire_exact(
+ &path,
+ self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)),
+ None,
+ ).await?;
+ let operation = async {
+ let manifest = self.provider.read_manifest(scope).await?;
+ if manifest.segments.iter().any(|descriptor| descriptor.path == name) {
+ return Ok(());
+ }
+ self.store.remove_file(&path).await
+ }.await;
+ let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from);
+ finish_with_release(operation, release)
+ }
+
+ /// Delete non-latest checkpoint candidates in manifest, chunks, directory order.
+ async fn collect_checkpoints(&self, scope: &ScopeKey) -> Result<()> {
+ let directory = self.store.paths().checkpoints_dir(
+ &scope.account_id, scope.partition_id)?.0;
+ for entry in self.store.list_directory(&directory).await? {
+ if !entry.is_dir || !is_checkpoint_directory_name(&entry.name) {
+ continue;
+ }
+ let candidate = format!("{directory}/{}", entry.name);
+ let lease = self.store.pathlock_manager().acquire_exact(
+ &candidate,
+ self.store.pathlock_manager().default_lock_timeout().max(Duration::from_secs(5)),
+ None,
+ ).await?;
+ let operation = async {
+ if self.read_latest(scope).await?.as_ref().is_some_and(
+ |latest| latest.path.trim_end_matches('/') == entry.name)
+ {
+ return Ok(());
+ }
+ self.store.remove_all(&candidate).await
+ }.await;
+ let release = self.store.pathlock_manager().release(&lease).await.map_err(Error::from);
+ finish_with_release(operation, release)?;
+ }
+ Ok(())
+ }
+}
+
+/// Return Stable partition identifiers in deterministic order.
+fn stable_partitions(manifest: &PartitionsManifest) -> Vec<u32> {
+ manifest.partitions.iter().filter_map(|(&partition_id, entry)| {
+ (entry.state == PartitionState::Stable).then_some(partition_id)
+ }).collect()
+}
+
+/// Return whether every event has one current-epoch position per Stable partition.
+fn directory_positions_complete(
+ events: &[DirectoryEvent],
+ stable: &[u32],
+ epoch: u64,
+) -> bool {
+ events.iter().all(|event| stable.iter().all(|partition_id| {
+ event.positions.iter().any(|position| {
+ position.partition_id == *partition_id && position.epoch == epoch
+ })
+ }))
+}
+
+/// Return whether every marker is absorbed and applied by every current backend.
+fn event_is_complete(
+ event: &DirectoryEvent,
+ snapshots: &BTreeMap<u32, ScopeSnapshot>,
+) -> bool {
+ event.positions.iter().all(|position| snapshots.get(&position.partition_id)
+ .is_some_and(|snapshot| {
+ snapshot.latest.as_ref().is_some_and(
+ |latest| latest.checkpoint_to_seq >= position.seq)
+ && snapshot.segments.backend_states.values().all(
+ |state| state.synced_seq >= position.seq)
+ }))
+}
+
+/// Preserve an operation failure while requiring lock release to succeed.
+fn finish_with_release<T>(operation: Result<T>, release: Result<()>) -> Result<T> {
+ match (operation, release) {
+ (Ok(value), Ok(())) => Ok(value),
+ (Ok(_), Err(error)) | (Err(error), Ok(())) => Err(error),
+ (Err(error), Err(release_error)) => Err(Error::internal(format!(
+ "{error}; lock release failed: {release_error}"
+ ))),
+ }
+}
diff --git a/crates/ragfs/src/multibackend/meta.rs b/crates/ragfs/src/multibackend/meta.rs
index 777a3cdcf..6c347df44 100644
--- a/crates/ragfs/src/multibackend/meta.rs
+++ b/crates/ragfs/src/multibackend/meta.rs
@@ -1,26 +1,26 @@
//! Multi-write metadata management.
-//!
-//! Provides `MetaStateStore` for serialized read-modify-write of `.redirect.json` and
-//! `.sync_log.json` through `primary_backend`, and `FsContextResolver` for recovering
-//! `FsContext` from paths in background tasks.
-use std::collections::HashMap;
+use std::collections::BTreeMap;
+use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
-use std::time::Duration;
-use tokio::sync::Mutex;
-use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use serde::{de::DeserializeOwned, Serialize};
use crate::core::context::{FsContext, FsContextInner, FS_CTX};
use crate::core::errors::{Error, Result};
use crate::core::filesystem::FileSystem;
-use crate::core::types::{RedirectMeta, SyncLogMeta, WriteFlag};
+use crate::core::types::{FileInfo, WriteFlag};
+use crate::lock::PathLockManager;
+use crate::multibackend::constants::{MULTIWRITE_MOUNT_PREFIX, VBUCKETS};
+use crate::multibackend::model::{
+ BackendState, CheckpointsManifest, PartitionEntry, PartitionManifest, PartitionState,
+ PartitionsManifest, ProtocolState, ProtocolStatus, ScopeKey, SegmentManifest,
+};
+use crate::multibackend::router::{build_initial_routes, MultiWritePaths};
/// Trait for resolving `FsContext` from a filesystem path.
///
-/// Used by background tasks (retry_loop, backfill, system_sync_retry) that lack a
-/// foreground request context. Implementations extract `account_id` from the path
-/// (e.g. `/local/{account_id}/...`).
+/// Implementations extract `account_id` from the path (e.g. `/local/{account_id}/...`).
pub trait FsContextResolver: Send + Sync {
/// Recover `FsContext` from a normalized path.
/// Returns an error if the path cannot be resolved to a valid context.
@@ -34,7 +34,10 @@ impl FsContextResolver for DefaultFsContextResolver {
fn resolve(&self, path: &str) -> Result<FsContext> {
let parts: Vec<&str> = path.trim_start_matches('/').split('/').collect();
// Path format: /local/{account_id}/...
- if parts.len() >= 2 && parts[0] == "local" && !parts[1].is_empty() {
+ if parts.len() >= 2
+ && parts[0] == MULTIWRITE_MOUNT_PREFIX.trim_start_matches('/')
+ && !parts[1].is_empty()
+ {
Ok(Arc::new(FsContextInner::new(parts[1].to_string())))
} else {
Err(Error::internal(format!(
@@ -61,429 +64,461 @@ impl FsContextResolver for RelativePathFsContextResolver {
}
}
-/// Internal file name for redirect metadata.
-pub(crate) const REDIRECT_FILE: &str = ".redirect.json";
-/// Internal file name for sync-log metadata.
-pub(crate) const SYNC_LOG_FILE: &str = ".sync_log.json";
-/// Hidden multi-write internal file names (redirect / sync-log metadata).
-pub(crate) const MULTIWRITE_INTERNAL_NAMES: &[&str] = &[SYNC_LOG_FILE, REDIRECT_FILE];
-const GLOBAL_STATE_FILE: &str = "/_system/.multiwrite.global.json";
-const GLOBAL_STATE_VERSION: u32 = 1;
-
-type TrackedLock = Mutex<()>;
-
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-struct GlobalMultiWriteState {
- version: u32,
- next_seq: u64,
+/// Closed worker labels used by the multi-write error counter.
+#[derive(Clone, Copy)]
+pub(crate) enum MultiWriteWorker {
+ Flush,
+ Checkpoint,
+ CatchUp,
+ Gc,
}
-impl Default for GlobalMultiWriteState {
- /// Create the default persisted global state for multi-write sequencing.
- fn default() -> Self {
- Self {
- version: GLOBAL_STATE_VERSION,
- next_seq: 1,
- }
- }
+/// Primary filesystem store for V2 multi-write metadata.
+pub struct MetadataStore {
+ primary: Arc<dyn FileSystem>,
+ pathlock_manager: Arc<PathLockManager>,
+ paths: MultiWritePaths,
+ worker_errors: [AtomicU64; 4],
}
-/// Unified metadata store for `.redirect.json` and `.sync_log.json`.
-///
-/// All reads and writes go through `primary_backend`, inheriting its encryption
-/// configuration. Directory-level locks ensure serialized access to both metadata
-/// files within the same directory.
-pub struct MetaStateStore {
- /// Primary backend (may be encrypted)
- primary_backend: Arc<dyn FileSystem>,
- /// Per-directory locks for serialized read-modify-write
- dir_locks: Mutex<HashMap<String, Arc<TrackedLock>>>,
- /// Dedicated global state lock for next_seq persistence.
- global_state_lock: Mutex<()>,
- /// Context resolver for background tasks
- ctx_resolver: Arc<dyn FsContextResolver>,
-}
-
-impl MetaStateStore {
- /// Create a new MetaStateStore.
+impl MetadataStore {
+ /// Create a V2 metadata store for one logical mount prefix.
pub fn new(
- primary_backend: Arc<dyn FileSystem>,
- ctx_resolver: Arc<dyn FsContextResolver>,
- ) -> Self {
- Self::with_cache_config(primary_backend, ctx_resolver, 0, Duration::ZERO)
+ primary: Arc<dyn FileSystem>,
+ pathlock_manager: Arc<PathLockManager>,
+ mount_prefix: &str,
+ ) -> Result<Self> {
+ Ok(Self {
+ primary,
+ pathlock_manager,
+ paths: MultiWritePaths::new(mount_prefix)?,
+ worker_errors: std::array::from_fn(|_| AtomicU64::new(0)),
+ })
}
- /// Create a new MetaStateStore; cache parameters are retained as no-op compatibility knobs.
- pub fn with_cache_config(
- primary_backend: Arc<dyn FileSystem>,
- ctx_resolver: Arc<dyn FsContextResolver>,
- _meta_cache_capacity: usize,
- _meta_cache_ttl: Duration,
- ) -> Self {
- Self {
- primary_backend,
- dir_locks: Mutex::new(HashMap::new()),
- global_state_lock: Mutex::new(()),
- ctx_resolver,
- }
+ /// Record one final failure for the selected worker.
+ pub(crate) fn record_worker_error(&self, worker: MultiWriteWorker) {
+ self.worker_errors[worker as usize].fetch_add(1, Ordering::Relaxed);
}
- /// Get or create a per-directory lock.
- async fn get_dir_lock(&self, dir: &str) -> Arc<TrackedLock> {
- let mut locks = self.dir_locks.lock().await;
- if let Some(lock) = locks.get(dir).cloned() {
- return lock;
- }
- let lock = Arc::new(TrackedLock::new(()));
- locks.insert(dir.to_string(), lock.clone());
- lock
+ /// Return the final failure count for the selected worker.
+ pub(crate) fn worker_error_count(&self, worker: MultiWriteWorker) -> u64 {
+ self.worker_errors[worker as usize].load(Ordering::Relaxed)
}
- /// Return the dedicated `_system` context used by global metadata.
- fn system_ctx() -> FsContext {
- Arc::new(FsContextInner::new("_system".to_string()))
+ /// Return the logical-to-backend path mapper owned by this store.
+ pub fn paths(&self) -> &MultiWritePaths {
+ &self.paths
}
- /// Build the effective context for metadata access.
- fn effective_meta_ctx(dir: &str, ctx: &FsContext) -> FsContext {
- if dir == "/" {
- Arc::new(Self::system_ctx().with_auto_pathlock_disabled())
- } else {
- Arc::new(ctx.with_auto_pathlock_disabled())
+ /// List initialized scope directories currently visible on the primary filesystem.
+ pub(crate) async fn initialized_accounts(&self) -> Result<Vec<String>> {
+ let mut accounts = Vec::new();
+ for entry in self.primary.read_internal_dir("/").await? {
+ if !entry.is_dir {
+ continue;
+ }
+ let manifest = self.paths.account_manifest(&entry.name)?.1;
+ match self.primary.stat(&manifest).await {
+ Ok(_) => accounts.push(entry.name),
+ Err(Error::NotFound(_)) => {}
+ Err(error) => return Err(error),
+ }
+ }
+ accounts.sort();
+ Ok(accounts)
+ }
+
+ /// Return whether the primary account root still exists.
+ pub(crate) async fn account_exists(&self, account_id: &str) -> Result<bool> {
+ let root = self
+ .paths
+ .raw_backend_path(&format!("{MULTIWRITE_MOUNT_PREFIX}/{account_id}"))?;
+ match self.primary.stat(&root).await {
+ Ok(_) => Ok(true),
+ Err(Error::NotFound(_)) => Ok(false),
+ Err(error) => Err(error),
}
}
- /// Build the full path for a metadata file in a directory.
- fn meta_path(dir: &str, filename: &str) -> String {
- if dir == "/" {
- format!("/{}", filename)
- } else {
- format!("{}/{}", dir, filename)
- }
+ /// Return the path lock manager used by V2 metadata operations.
+ pub(crate) fn pathlock_manager(&self) -> &PathLockManager {
+ &self.pathlock_manager
}
- /// Read one JSON metadata file, returning default for missing or empty files.
- async fn read_meta<T>(&self, dir: &str, filename: &str, ctx: &FsContext) -> Result<T>
- where
- T: DeserializeOwned + Default,
- {
- let path = Self::meta_path(dir, filename);
- let effective_ctx = Self::effective_meta_ctx(dir, ctx);
- match FS_CTX
- .scope(effective_ctx, async {
- self.primary_backend.read(&path, 0, 0).await
- })
- .await
+ /// Build the filesystem context used for one logical metadata path.
+ fn metadata_context(&self, logical_path: &str) -> Result<FsContext> {
+ let parts: Vec<&str> = logical_path.trim_start_matches('/').split('/').collect();
+ if parts.len() >= 2
+ && parts[0] == MULTIWRITE_MOUNT_PREFIX.trim_start_matches('/')
+ && !parts[1].is_empty()
{
- Ok(data) => {
- if data.is_empty() {
- Ok(T::default())
- } else {
- serde_json::from_slice(&data).map_err(Error::from)
- }
- }
- Err(Error::NotFound(_)) => Ok(T::default()),
- Err(e) => Err(e),
+ return Ok(Arc::new(
+ FsContextInner::new(parts[1].to_string())
+ .with_bypass_cache(true)
+ .with_auto_pathlock_disabled(),
+ ));
}
+ Err(Error::internal(format!(
+ "cannot resolve metadata FsContext from path: {logical_path}"
+ )))
}
- /// Read redirect metadata from a directory (returns default if not found).
- async fn read_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result<RedirectMeta> {
- self.read_meta(dir, REDIRECT_FILE, ctx).await
- }
-
- /// Read sync log metadata from a directory (returns default if not found).
- async fn read_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result<SyncLogMeta> {
- self.read_meta(dir, SYNC_LOG_FILE, ctx).await
- }
-
- /// Read both metadata files from the primary backend.
- async fn read_dir_meta_pair(
- &self,
- dir: &str,
- ctx: &FsContext,
- ) -> Result<(RedirectMeta, SyncLogMeta)> {
- let redirect = self.read_redirect_meta(dir, ctx).await?;
- let sync_log = self.read_sync_log_meta(dir, ctx).await?;
- Ok((redirect, sync_log))
- }
-
- /// Write one JSON metadata file to a directory.
- async fn write_meta<T>(
- &self,
- dir: &str,
- filename: &str,
- meta: &T,
- ctx: &FsContext,
- ) -> Result<()>
- where
- T: Serialize,
- {
- let path = Self::meta_path(dir, filename);
- let data = serde_json::to_vec(meta)?;
- let effective_ctx = Self::effective_meta_ctx(dir, ctx);
+ /// Read one complete binary value from a canonical logical path.
+ pub(crate) async fn read_bytes(&self, logical_path: &str) -> Result<Vec<u8>> {
+ let backend_path = self.paths.raw_backend_path(logical_path)?;
+ let context = self.metadata_context(logical_path)?;
FS_CTX
- .scope(effective_ctx, async {
- self.primary_backend
- .write(&path, &data, 0, WriteFlag::Create)
- .await
- .map(|_| ())
- })
+ .scope(context, self.primary.read(&backend_path, 0, 0))
.await
}
- /// Write redirect metadata to a directory.
- async fn write_redirect_meta(
- &self,
- dir: &str,
- meta: &RedirectMeta,
- ctx: &FsContext,
- ) -> Result<()> {
- self.write_meta(dir, REDIRECT_FILE, meta, ctx).await
+ /// Read and validate the current V2 protocol status.
+ pub(crate) async fn protocol_status(&self) -> Result<ProtocolStatus> {
+ let state: ProtocolState = self.read_json(&self.paths.mount_protocol().0).await?;
+ state.validate()?;
+ Ok(state.status)
}
- /// Write sync log metadata to a directory.
- async fn write_sync_log_meta(
+ /// Replace protocol bytes under an Exact lock when the current value matches.
+ pub(crate) async fn compare_protocol(
&self,
- dir: &str,
- meta: &SyncLogMeta,
- ctx: &FsContext,
- ) -> Result<()> {
- self.write_meta(dir, SYNC_LOG_FILE, meta, ctx).await
+ expected: &[u8],
+ replacement: &[u8],
+ ) -> Result<bool> {
+ let logical_path = self.paths.mount_protocol().0;
+ let lease = self
+ .pathlock_manager
+ .acquire_exact(
+ &logical_path,
+ self.pathlock_manager
+ .default_lock_timeout()
+ .max(std::time::Duration::from_secs(5)),
+ None,
+ )
+ .await?;
+ let operation = async {
+ match self.read_bytes(&logical_path).await {
+ Ok(current) if current == expected => {
+ self.write_bytes(&logical_path, replacement).await?;
+ Ok(true)
+ }
+ Ok(_) | Err(Error::NotFound(_)) => Ok(false),
+ Err(error) => Err(error),
+ }
+ }
+ .await;
+ let release = self
+ .pathlock_manager
+ .release(&lease)
+ .await
+ .map_err(Error::from);
+ merge_operation_and_cleanup(operation, release)
}
- /// Serialized read-modify-write of both `.redirect.json` and `.sync_log.json` in a directory.
- ///
- /// Acquires the directory lock, reads both metadata files, applies `op`, and writes both back.
- /// This prevents concurrent updates from losing entries.
- pub async fn update_dir_meta<F>(&self, dir: &str, ctx: &FsContext, op: F) -> Result<()>
+ /// Read and deserialize one JSON value from a canonical logical path.
+ pub async fn read_json<T>(&self, logical_path: &str) -> Result<T>
where
- F: FnOnce(&mut RedirectMeta, &mut SyncLogMeta) -> Result<()>,
+ T: DeserializeOwned,
{
- let lock = self.get_dir_lock(dir).await;
- let _guard = lock.lock().await;
-
- let (mut redirect_meta, mut sync_log_meta) = self.read_dir_meta_pair(dir, ctx).await?;
- let original_redirect = redirect_meta.clone();
- let original_sync_log = sync_log_meta.clone();
-
- op(&mut redirect_meta, &mut sync_log_meta)?;
+ let bytes = self.read_bytes(logical_path).await?;
+ serde_json::from_slice(&bytes).map_err(Error::from)
+ }
- if redirect_meta != original_redirect {
- self.write_redirect_meta(dir, &redirect_meta, ctx).await?;
- }
- if sync_log_meta != original_sync_log {
- self.write_sync_log_meta(dir, &sync_log_meta, ctx).await?;
- }
- Ok(())
+ /// Write complete binary bytes to a canonical logical path.
+ pub(crate) async fn write_bytes(&self, logical_path: &str, bytes: &[u8]) -> Result<()> {
+ let backend_path = self.paths.raw_backend_path(logical_path)?;
+ let context = self.metadata_context(logical_path)?;
+ FS_CTX
+ .scope(context, async {
+ self.primary
+ .ensure_parent_dirs(&backend_path, 0o755)
+ .await?;
+ self.primary
+ .write(&backend_path, bytes, 0, WriteFlag::Create)
+ .await?;
+ Ok(())
+ })
+ .await
}
- /// Serialized read-modify-write of two directories' metadata (for cross-directory rename).
- ///
- /// Acquires both directory locks in lexicographic order to prevent deadlock,
- /// then reads and updates all four metadata files within the same critical section.
- /// Caller must ensure source_dir != target_dir; use update_dir_meta for same-directory case.
- pub async fn update_dual_dir_meta<F>(
- &self,
- source_dir: &str,
- target_dir: &str,
- ctx: &FsContext,
- op: F,
- ) -> Result<()>
+ /// Validate and write one JSON value at a logical path.
+ pub async fn publish_json<T, F>(&self, logical_path: &str, value: &T, validate: F) -> Result<()>
where
- F: FnOnce(
- &mut RedirectMeta,
- &mut SyncLogMeta,
- &mut RedirectMeta,
- &mut SyncLogMeta,
- ) -> Result<()>,
+ T: Serialize,
+ F: FnOnce(&T) -> Result<()>,
{
- // Acquire locks in lexicographic order to avoid deadlock.
- let (first_dir, second_dir) = if source_dir < target_dir {
- (source_dir, target_dir)
- } else {
- (target_dir, source_dir)
- };
-
- let lock1 = self.get_dir_lock(first_dir).await;
- let lock2 = self.get_dir_lock(second_dir).await;
- let _guard1 = lock1.lock().await;
- let _guard2 = lock2.lock().await;
-
- let (mut src_redirect, mut src_sync_log) = self.read_dir_meta_pair(source_dir, ctx).await?;
- let (mut tgt_redirect, mut tgt_sync_log) = self.read_dir_meta_pair(target_dir, ctx).await?;
- let original_src_redirect = src_redirect.clone();
- let original_src_sync_log = src_sync_log.clone();
- let original_tgt_redirect = tgt_redirect.clone();
- let original_tgt_sync_log = tgt_sync_log.clone();
-
- op(
- &mut src_redirect,
- &mut src_sync_log,
- &mut tgt_redirect,
- &mut tgt_sync_log,
- )?;
-
- if src_redirect != original_src_redirect {
- self.write_redirect_meta(source_dir, &src_redirect, ctx)
- .await?;
- }
- if src_sync_log != original_src_sync_log {
- self.write_sync_log_meta(source_dir, &src_sync_log, ctx)
- .await?;
- }
- if tgt_redirect != original_tgt_redirect {
- self.write_redirect_meta(target_dir, &tgt_redirect, ctx)
- .await?;
+ let bytes = serde_json::to_vec(value)?;
+ validate(value)?;
+ self.write_bytes(logical_path, &bytes).await
+ }
+
+ /// Create one immutable blob or accept an identical existing value.
+ pub async fn write_immutable(&self, logical_path: &str, bytes: &[u8]) -> Result<()> {
+ match self.read_bytes(logical_path).await {
+ Ok(existing) if existing == bytes => Ok(()),
+ Ok(_) => Err(Error::AlreadyExists(format!(
+ "immutable metadata differs: {logical_path}"
+ ))),
+ Err(Error::NotFound(_)) => self.write_bytes(logical_path, bytes).await,
+ Err(error) => Err(error),
}
- if tgt_sync_log != original_tgt_sync_log {
- self.write_sync_log_meta(target_dir, &tgt_sync_log, ctx)
- .await?;
- }
- Ok(())
}
- /// Read redirect metadata for a directory (public, used by read_dir to merge redirect entries).
- pub async fn get_redirect_meta(&self, dir: &str, ctx: &FsContext) -> Result<RedirectMeta> {
- self.read_redirect_meta(dir, ctx).await
+ /// List one raw metadata directory without public hidden-name filtering.
+ pub(crate) async fn list_directory(&self, logical_path: &str) -> Result<Vec<FileInfo>> {
+ let backend_path = self.paths.raw_backend_path(logical_path)?;
+ self.primary.read_internal_dir(&backend_path).await
}
- /// Read sync log metadata for a directory (public, used by retry_loop).
- pub async fn get_sync_log_meta(&self, dir: &str, ctx: &FsContext) -> Result<SyncLogMeta> {
- self.read_sync_log_meta(dir, ctx).await
+ /// Remove one raw metadata file while treating absence as success.
+ pub(crate) async fn remove_file(&self, logical_path: &str) -> Result<()> {
+ let backend_path = self.paths.raw_backend_path(logical_path)?;
+ self.remove_if_present(&backend_path).await
}
- /// Get a reference to the context resolver.
- pub fn ctx_resolver(&self) -> &Arc<dyn FsContextResolver> {
- &self.ctx_resolver
+ /// Remove one raw metadata tree while treating absence as success.
+ pub(crate) async fn remove_all(&self, logical_path: &str) -> Result<()> {
+ let backend_path = self.paths.raw_backend_path(logical_path)?;
+ match self.primary.remove_all(&backend_path).await {
+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),
+ Err(error) => Err(error),
+ }
}
- /// Get a reference to the primary backend.
- pub fn primary_backend(&self) -> &Arc<dyn FileSystem> {
- &self.primary_backend
+ /// Recheck destructive GC safety while the caller holds the account Exact lock.
+ pub(crate) async fn gc_scope_is_safe(&self, scope: &ScopeKey) -> Result<bool> {
+ let path = self.paths.account_manifest(&scope.account_id)?.0;
+ let manifest: PartitionsManifest = self.read_json(&path).await?;
+ manifest.validate()?;
+ scope.validate_against_manifest(&manifest)?;
+ let stable = manifest
+ .partitions
+ .iter()
+ .filter_map(|(&id, entry)| (entry.state == PartitionState::Stable).then_some(id))
+ .collect::<Vec<_>>();
+ Ok(stable.len() == manifest.partitions.len()
+ && manifest.directory_events.iter().all(|event| {
+ stable.iter().all(|id| {
+ event.positions.iter().any(|position| {
+ position.partition_id == *id && position.epoch == manifest.epoch
+ })
+ })
+ }))
+ }
+
+ /// Initialize one account under an Exact manifest lock and return its manifest.
+ pub async fn initialize_account(
+ &self,
+ account_id: &str,
+ partition_count: u32,
+ backup_names: &[String],
+ ) -> Result<PartitionsManifest> {
+ let logical_manifest = self.paths.account_manifest(account_id)?.0;
+ let lease = self
+ .pathlock_manager
+ .acquire_exact(
+ &logical_manifest,
+ self.pathlock_manager
+ .default_lock_timeout()
+ .max(std::time::Duration::from_secs(5)),
+ None,
+ )
+ .await?;
+ let result = self
+ .initialize_account_locked(account_id, partition_count, backup_names)
+ .await;
+ let release = self
+ .pathlock_manager
+ .release(&lease)
+ .await
+ .map_err(Error::from);
+ merge_operation_and_cleanup(result, release)
}
- /// Allocate and persist the next global sequence number.
- pub async fn next_seq(&self) -> Result<u64> {
- let _guard = self.global_state_lock.lock().await;
- let mut state = self.read_global_state().await?;
- let seq = state.next_seq;
- state.next_seq = state.next_seq.saturating_add(1);
- self.write_global_state(&state).await?;
- Ok(seq)
+ /// Update one partitions manifest while holding its Exact lock.
+ pub(crate) async fn update_partitions_manifest<T, F>(
+ &self,
+ account_id: &str,
+ update: F,
+ ) -> Result<T>
+ where
+ F: FnOnce(&mut PartitionsManifest) -> Result<T>,
+ {
+ let logical_manifest = self.paths.account_manifest(account_id)?.0;
+ let lease = self
+ .pathlock_manager
+ .acquire_exact(
+ &logical_manifest,
+ self.pathlock_manager
+ .default_lock_timeout()
+ .max(std::time::Duration::from_secs(5)),
+ None,
+ )
+ .await?;
+ let operation = async {
+ let mut manifest: PartitionsManifest = self.read_json(&logical_manifest).await?;
+ manifest.validate()?;
+ let original = manifest.clone();
+ let output = update(&mut manifest)?;
+ validate_partitions_manifest_update(&original, &manifest)?;
+ manifest.validate()?;
+ if manifest != original {
+ self.publish_json(&logical_manifest, &manifest, PartitionsManifest::validate)
+ .await?;
+ }
+ Ok(output)
+ }
+ .await;
+ let release = self
+ .pathlock_manager
+ .release(&lease)
+ .await
+ .map_err(Error::from);
+ merge_operation_and_cleanup(operation, release)
}
- /// Read the persisted global state file.
- async fn read_global_state(&self) -> Result<GlobalMultiWriteState> {
- let ctx = Self::system_ctx();
- match FS_CTX
- .scope(ctx.clone(), async {
- self.primary_backend.read(GLOBAL_STATE_FILE, 0, 0).await
- })
+ /// Initialize account metadata while the caller holds the manifest lock.
+ async fn initialize_account_locked(
+ &self,
+ account_id: &str,
+ partition_count: u32,
+ backup_names: &[String],
+ ) -> Result<PartitionsManifest> {
+ let logical_manifest = self.paths.account_manifest(account_id)?.0;
+ match self
+ .read_json::<PartitionsManifest>(&logical_manifest)
.await
{
- Ok(data) => {
- if data.is_empty() {
- Ok(GlobalMultiWriteState::default())
- } else {
- let state: GlobalMultiWriteState = serde_json::from_slice(&data)?;
- if state.version != GLOBAL_STATE_VERSION {
- return Err(Error::config(format!(
- "unsupported multi-write global state version {}",
- state.version
- )));
- }
- Ok(state)
- }
+ Ok(manifest) => {
+ manifest.validate()?;
+ return Ok(manifest);
}
- Err(Error::NotFound(_)) => Ok(GlobalMultiWriteState::default()),
- Err(e) => Err(e),
+ Err(Error::NotFound(_)) => {}
+ Err(error) => return Err(error),
}
- }
- /// Persist the global sequence state through the primary backend.
- async fn write_global_state(&self, state: &GlobalMultiWriteState) -> Result<()> {
- let ctx = Self::system_ctx();
- let data = serde_json::to_vec(state)?;
- FS_CTX
- .scope(ctx.clone(), async {
- self.primary_backend
- .ensure_parent_dirs(GLOBAL_STATE_FILE, 0o755)
- .await?;
- self.primary_backend
- .write(GLOBAL_STATE_FILE, &data, 0, WriteFlag::Create)
- .await
- .map(|_| ())
+ let routes = build_initial_routes(partition_count)?;
+ let partitions = (0..partition_count)
+ .map(|partition_id| {
+ (
+ partition_id,
+ PartitionEntry {
+ state: PartitionState::Stable,
+ },
+ )
})
- .await
- }
-}
-
-/// Per-path serialization queue for async write ordering.
-///
-/// Ensures that multiple writes to the same path are executed in FIFO order
-/// on backup backends, preventing out-of-order application.
-pub struct PathSerializer {
- queues: Mutex<HashMap<String, Arc<TrackedLock>>>,
-}
-
-impl PathSerializer {
- /// Create a new PathSerializer.
- pub fn new() -> Self {
- Self::with_limits(0, Duration::ZERO)
- }
-
- /// Create a PathSerializer; limit arguments are ignored for compatibility.
- pub fn with_limits(_capacity: usize, _ttl: Duration) -> Self {
- Self {
- queues: Mutex::new(HashMap::new()),
+ .collect::<BTreeMap<_, _>>();
+ let manifest = PartitionsManifest {
+ version: 1,
+ epoch: 1,
+ vbuckets: VBUCKETS,
+ partitions,
+ routes,
+ directory_events: Vec::new(),
+ };
+ manifest.validate()?;
+
+ let backend_states = backup_names
+ .iter()
+ .map(|name| (name.clone(), BackendState { synced_seq: 0 }))
+ .collect::<BTreeMap<_, _>>();
+ for partition_id in 0..partition_count {
+ let partition = PartitionManifest {
+ version: 1,
+ partition_id,
+ epoch: 1,
+ };
+ self.write_validated_json_immutable(
+ &self.paths.partition_manifest(account_id, partition_id)?.0,
+ &partition,
+ PartitionManifest::validate,
+ )
+ .await?;
+
+ let segments = SegmentManifest {
+ version: 1,
+ next_seq: 1,
+ segments: Vec::new(),
+ backend_states: backend_states.clone(),
+ };
+ self.write_validated_json_immutable(
+ &self.paths.segment_manifest(account_id, partition_id)?.0,
+ &segments,
+ SegmentManifest::validate,
+ )
+ .await?;
+
+ let checkpoints = CheckpointsManifest {
+ version: 1,
+ latest_checkpoint: None,
+ };
+ self.write_validated_json_immutable(
+ &self.paths.checkpoints_manifest(account_id, partition_id)?.0,
+ &checkpoints,
+ CheckpointsManifest::validate,
+ )
+ .await?;
}
+ self.write_validated_json_immutable(
+ &logical_manifest,
+ &manifest,
+ PartitionsManifest::validate,
+ )
+ .await?;
+ Ok(manifest)
}
- /// Run one async operation under the per-path serialization lock.
- pub async fn with_path_lock<F, Fut, T>(&self, path: &str, op: F) -> T
+ /// Validate, serialize, and create one immutable JSON metadata file.
+ async fn write_validated_json_immutable<T, F>(
+ &self,
+ logical_path: &str,
+ value: &T,
+ validate: F,
+ ) -> Result<()>
where
- F: FnOnce() -> Fut,
- Fut: std::future::Future<Output = T>,
+ T: Serialize,
+ F: FnOnce(&T) -> Result<()>,
{
- let mut queues = self.queues.lock().await;
- let lock = queues
- .entry(path.to_string())
- .or_insert_with(|| Arc::new(TrackedLock::new(())))
- .clone();
- drop(queues);
-
- let _guard = lock.lock().await;
- op().await
+ let bytes = serde_json::to_vec(value)?;
+ validate(value)?;
+ self.write_immutable(logical_path, &bytes).await
}
- /// Return the number of tracked queue entries.
- #[cfg(test)]
- pub async fn len(&self) -> usize {
- self.queues.lock().await.len()
- }
-}
-
-impl Default for PathSerializer {
- fn default() -> Self {
- Self::new()
+ /// Remove a temporary file while treating an absent file as clean.
+ async fn remove_if_present(&self, backend_path: &str) -> Result<()> {
+ match self.primary.remove(backend_path).await {
+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),
+ Err(error) => Err(error),
+ }
}
}
-/// Extract the directory path from a file path.
-pub(crate) fn parent_dir(path: &str) -> String {
- match path.rfind('/') {
- Some(0) => "/".to_string(),
- Some(pos) => path[..pos].to_string(),
- None => "/".to_string(),
+/// Reject updates that change the fixed partition routing identity.
+fn validate_partitions_manifest_update(
+ original: &PartitionsManifest,
+ updated: &PartitionsManifest,
+) -> Result<()> {
+ if original.epoch != updated.epoch
+ || original.partitions != updated.partitions
+ || original.routes != updated.routes
+ {
+ return Err(Error::invalid_operation(
+ "partition routing cannot change after account initialization",
+ ));
}
+ Ok(())
}
-/// Extract the file name from a path.
-pub(crate) fn file_name(path: &str) -> &str {
- match path.rfind('/') {
- Some(pos) => &path[pos + 1..],
- None => path,
+/// Preserve an operation error while requiring cleanup to succeed.
+fn merge_operation_and_cleanup<T>(operation: Result<T>, cleanup: Result<()>) -> Result<T> {
+ match (operation, cleanup) {
+ (Ok(value), Ok(())) => Ok(value),
+ (Ok(_), Err(error)) => Err(error),
+ (Err(error), Ok(())) => Err(error),
+ (Err(error), Err(cleanup_error)) => Err(Error::internal(format!(
+ "{error}; cleanup failed: {cleanup_error}"
+ ))),
}
}
@@ -497,21 +532,6 @@ pub fn current_required_ctx() -> Result<FsContext> {
#[cfg(test)]
mod tests {
use super::*;
- use crate::plugins::memfs::MemFileSystem;
- use std::sync::Arc;
-
- #[test]
- fn test_parent_dir() {
- assert_eq!(parent_dir("/a/b/c.txt"), "/a/b");
- assert_eq!(parent_dir("/a"), "/");
- assert_eq!(parent_dir("/"), "/");
- }
-
- #[test]
- fn test_file_name() {
- assert_eq!(file_name("/a/b/c.txt"), "c.txt");
- assert_eq!(file_name("/a"), "a");
- }
#[test]
fn test_default_resolver() {
@@ -527,48 +547,4 @@ mod tests {
let resolver = DefaultFsContextResolver;
assert!(resolver.resolve("/invalid/path").is_err());
}
-
-
- #[tokio::test]
- async fn test_invalid_sync_log_json_returns_error() {
- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let store = MetaStateStore::new(primary.clone(), Arc::new(DefaultFsContextResolver));
- let ctx = Arc::new(FsContextInner::new("acct".to_string()));
-
- FS_CTX
- .scope(ctx.clone(), async {
- primary
- .ensure_parent_dirs("/local/acct/docs/.sync_log.json", 0o755)
- .await?;
- primary
- .write(
- "/local/acct/docs/.sync_log.json",
- b"{not valid json",
- 0,
- WriteFlag::Create,
- )
- .await?;
- Ok::<(), Error>(())
- })
- .await
- .unwrap();
-
- let result = store.get_sync_log_meta("/local/acct/docs", &ctx).await;
- assert!(result.is_err(), "corrupted metadata must fail fast");
- }
-
- #[tokio::test]
- async fn test_next_seq_uses_dedicated_global_lock() {
- let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
- let store = MetaStateStore::new(primary, Arc::new(DefaultFsContextResolver));
-
- assert_eq!(store.next_seq().await.unwrap(), 1);
- assert_eq!(store.next_seq().await.unwrap(), 2);
-
- let dir_locks = store.dir_locks.lock().await;
- assert!(
- !dir_locks.contains_key(GLOBAL_STATE_FILE),
- "global sequence locking should not occupy the directory lock pool"
- );
- }
}
diff --git a/crates/ragfs/src/multibackend/migration/mod.rs b/crates/ragfs/src/multibackend/migration/mod.rs
new file mode 100644
index 000000000..2db469c39
--- /dev/null
+++ b/crates/ragfs/src/multibackend/migration/mod.rs
@@ -0,0 +1,231 @@
+//! V2 protocol initialization and resumable full-data import.
+
+use std::sync::Arc;
+
+use uuid::Uuid;
+
+use crate::core::context::{FsContextInner, FS_CTX};
+use crate::core::errors::{Error, Result};
+use crate::core::filesystem::FileSystem;
+use crate::core::internal_names::is_multiwrite_internal_path;
+use crate::multibackend::constants::{MULTIWRITE_MOUNT_PREFIX, SYSTEM_DIR};
+use crate::multibackend::meta::MetadataStore;
+use crate::multibackend::model::{
+ PendingEvent, PendingEventKind, ProtocolState, ProtocolStatus, ProtocolVersion,
+ SegmentEventType,
+};
+use crate::multibackend::provider::{bootstrap_account, MultiWriteProvider};
+use crate::multibackend::runtime::read_active_accounts;
+
+/// Reads or initializes the persisted V2 multi-write protocol state.
+pub struct ProtocolDetector {
+ store: Arc<MetadataStore>,
+}
+
+impl ProtocolDetector {
+ /// Create a detector backed by the supplied raw-primary metadata store.
+ pub fn new(store: Arc<MetadataStore>) -> Self {
+ Self { store }
+ }
+
+ /// Read the protocol state, creating V2 migrating state when absent.
+ pub async fn detect(&self) -> Result<ProtocolState> {
+ let (protocol_path, _) = self.store.paths().mount_protocol();
+ match self.store.read_bytes(&protocol_path).await {
+ Ok(bytes) => decode_protocol(&bytes),
+ Err(Error::NotFound(_)) => {
+ let state = ProtocolState {
+ protocol_version: ProtocolVersion::V2,
+ status: ProtocolStatus::Migrating,
+ };
+ let bytes = serde_json::to_vec(&state)?;
+ match self.store.write_immutable(&protocol_path, &bytes).await {
+ Ok(()) => Ok(state),
+ Err(Error::AlreadyExists(_)) => {
+ decode_protocol(&self.store.read_bytes(&protocol_path).await?)
+ }
+ Err(error) => Err(error),
+ }
+ }
+ Err(error) => Err(error),
+ }
+ }
+}
+
+/// Imports current primary files into V2 segment metadata.
+pub struct FullDataImporter {
+ primary: Arc<dyn FileSystem>,
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+ initial_partitions: u32,
+ backup_names: Vec<String>,
+}
+
+impl FullDataImporter {
+ /// Create an importer for one primary, metadata store, and persistence provider.
+ pub fn new(
+ primary: Arc<dyn FileSystem>,
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+ initial_partitions: u32,
+ backup_names: Vec<String>,
+ ) -> Self {
+ Self {
+ primary,
+ store,
+ provider,
+ initial_partitions,
+ backup_names,
+ }
+ }
+
+ /// Import every current account file and atomically mark V2 stable.
+ pub async fn run(&self) -> Result<()> {
+ let (protocol_path, _) = self.store.paths().mount_protocol();
+ let migrating_bytes = self.store.read_bytes(&protocol_path).await?;
+ if decode_protocol(&migrating_bytes)?.status == ProtocolStatus::Stable {
+ return Ok(());
+ }
+
+ // ponytail: fixed delay assumes account bootstrap finishes within 10s; use readiness signaling if that changes.
+ tokio::time::sleep(std::time::Duration::from_secs(10)).await;
+ let mut accounts = read_active_accounts(self.primary.as_ref())
+ .await?
+ .into_iter()
+ .collect::<Vec<_>>();
+ accounts.sort();
+ self.import_scope(SYSTEM_DIR).await?;
+ for account in &accounts {
+ if !read_active_accounts(self.primary.as_ref())
+ .await?
+ .contains(account)
+ {
+ continue;
+ }
+ self.import_scope(account).await?;
+ }
+ self.cleanup_v1_metadata().await?;
+
+ let stable_bytes = serde_json::to_vec(&ProtocolState {
+ protocol_version: ProtocolVersion::V2,
+ status: ProtocolStatus::Stable,
+ })?;
+ if self
+ .store
+ .compare_protocol(&migrating_bytes, &stable_bytes)
+ .await?
+ {
+ return Ok(());
+ }
+ if decode_protocol(&self.store.read_bytes(&protocol_path).await?)?.status
+ == ProtocolStatus::Stable
+ {
+ Ok(())
+ } else {
+ Err(Error::invalid_operation(
+ "multi-write protocol changed during full-data import",
+ ))
+ }
+ }
+
+ /// Initialize one V2 scope and import its current file states.
+ async fn import_scope(&self, account: &str) -> Result<()> {
+ let partitions = self
+ .store
+ .initialize_account(account, self.initial_partitions, &self.backup_names)
+ .await?;
+ bootstrap_account(
+ self.provider.as_ref(),
+ self.store.as_ref(),
+ account,
+ &partitions,
+ )
+ .await?;
+ let context = Arc::new(FsContextInner::new(account).with_auto_pathlock_disabled());
+ FS_CTX.scope(context, self.import_account(account)).await
+ }
+
+ /// Scan one account through the business filesystem.
+ async fn import_account(&self, account: &str) -> Result<()> {
+ let root = format!("/{account}");
+ let mut pending = vec![root];
+ while let Some(directory) = pending.pop() {
+ let mut entries = self.primary.read_internal_dir(&directory).await?;
+ entries.sort_by(|left, right| left.name.cmp(&right.name));
+ for entry in entries {
+ let path = child_path(&directory, &entry.name);
+ let logical_path = format!("{MULTIWRITE_MOUNT_PREFIX}{path}");
+ if is_multiwrite_internal_path(&logical_path) {
+ continue;
+ }
+ if entry.is_dir {
+ pending.push(path);
+ continue;
+ }
+ self.provider
+ .flush(
+ None,
+ vec![PendingEvent {
+ operation_id: Uuid::new_v4(),
+ account_id: account.to_string(),
+ kind: PendingEventKind::Data(SegmentEventType::Write),
+ path: logical_path,
+ destination_path: None,
+ route_hint: None,
+ }],
+ )
+ .await?;
+ }
+ }
+ Ok(())
+ }
+
+ /// Remove V1 metadata through the configured primary filesystem.
+ /// Returns an error when scanning or primary removal fails.
+ async fn cleanup_v1_metadata(&self) -> Result<()> {
+ match self.primary.stat("/_system/.multiwrite.global.json").await {
+ Ok(_) => {}
+ Err(Error::NotFound(_)) => return Ok(()),
+ Err(error) => return Err(error),
+ }
+
+ let page = self
+ .primary
+ .glob_directory("/", "**/.sync_log.json", true, None, None, None)
+ .await?;
+ for entry in page.entries {
+ if entry.is_dir {
+ continue;
+ }
+ match self.primary.remove(&entry.path).await {
+ Ok(()) | Err(Error::NotFound(_)) => {}
+ Err(error) => return Err(error),
+ }
+ }
+
+ match self
+ .primary
+ .remove("/_system/.multiwrite.global.json")
+ .await
+ {
+ Ok(()) | Err(Error::NotFound(_)) => Ok(()),
+ Err(error) => Err(error),
+ }
+ }
+}
+
+/// Deserialize and validate exact V2 protocol bytes.
+fn decode_protocol(bytes: &[u8]) -> Result<ProtocolState> {
+ let state: ProtocolState = serde_json::from_slice(bytes)?;
+ state.validate()?;
+ Ok(state)
+}
+
+/// Join one direct child name to a canonical mount-relative directory.
+fn child_path(directory: &str, name: &str) -> String {
+ if directory == "/" {
+ format!("/{name}")
+ } else {
+ format!("{directory}/{name}")
+ }
+}
diff --git a/crates/ragfs/src/multibackend/mod.rs b/crates/ragfs/src/multibackend/mod.rs
index 44e8e4ab2..569a2225a 100644
--- a/crates/ragfs/src/multibackend/mod.rs
+++ b/crates/ragfs/src/multibackend/mod.rs
@@ -1,14 +1,32 @@
-//! Multi-backend configuration, factory, metadata, and admin support.
+//! Multi-backend configuration, factory, and metadata support.
-/// Multi-backend administrative APIs.
-pub mod admin;
+/// Prefix reconciliation primitives for backup catch-up.
+pub mod catch_up;
+/// Checkpoint construction, validation, publication, and reading.
+pub mod checkpoint;
+/// Stable binary codecs for V2 segments and checkpoints.
+pub mod codec;
/// Multi-backend config validation and normalization helpers.
pub mod config;
+/// Shared constants for V2 multi-write metadata.
+pub(crate) mod constants;
/// Multi-backend runtime assembly from validated config.
pub mod factory;
+/// Metadata garbage collection.
+pub mod gc;
/// Metadata state management shared by multi-backend runtime paths.
pub mod meta;
+/// V2 protocol initialization and full-data import.
+pub mod migration;
+/// V2 multi-write persistence and in-memory models.
+pub mod model;
+/// V2 multi-write persistence provider interfaces and implementations.
+pub mod provider;
+/// Logical path mapping and account partition routing.
+pub mod router;
+/// Foreground event queue and metadata flush worker.
+pub mod runtime;
/// Shared build-time types for multi-backend assembly.
pub mod types;
-pub use meta::{FsContextResolver, MetaStateStore};
+pub use meta::FsContextResolver;
diff --git a/crates/ragfs/src/multibackend/model.rs b/crates/ragfs/src/multibackend/model.rs
new file mode 100644
index 000000000..85b782d0f
--- /dev/null
+++ b/crates/ragfs/src/multibackend/model.rs
@@ -0,0 +1,647 @@
+//! V2 multi-write persistence and in-memory data models.
+#![allow(missing_docs)]
+use crate::core::errors::{Error, Result};
+use crate::multibackend::constants::{
+ CHECKPOINT_DIR_PREFIX, HEAD_SEGMENT_FILE_PREFIX, MAX_CHECKPOINT_NODES, MAX_SEGMENT_RECORDS,
+ SEALED_SEGMENT_FILE_PREFIX, SEGMENT_FILE_EXTENSION,
+};
+use crate::multibackend::router::validate_routes;
+use serde::{Deserialize, Serialize};
+use std::collections::{BTreeMap, HashSet};
+use uuid::Uuid;
+macro_rules! ensure { ($condition:expr, $($arg:tt)*) => { if !$condition { return Err(Error::Serialization(format!($($arg)*))); } }; }
+pub use crate::multibackend::constants::VBUCKETS;
+const FORMAT_VERSION: u32 = 1;
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct ProtocolState {
+ pub protocol_version: ProtocolVersion,
+ pub status: ProtocolStatus,
+}
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+pub enum ProtocolVersion {
+ V2,
+}
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "snake_case")]
+pub enum ProtocolStatus {
+ Migrating,
+ Stable,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct PartitionsManifest {
+ pub version: u32,
+ pub epoch: u64,
+ pub vbuckets: u32,
+ pub partitions: BTreeMap<u32, PartitionEntry>,
+ pub routes: Vec<RouteEntry>,
+ pub directory_events: Vec<DirectoryEvent>,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct PartitionEntry {
+ pub state: PartitionState,
+}
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+pub enum PartitionState {
+ Stable,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct RouteEntry {
+ pub start: u32,
+ pub end: u32,
+ pub partition: u32,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct PartitionManifest {
+ pub version: u32,
+ pub partition_id: u32,
+ pub epoch: u64,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct DirectoryEvent {
+ pub op_id: Uuid,
+ pub operation: DirectoryOperation,
+ pub source_path: String,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub destination_path: Option<String>,
+ pub positions: Vec<MarkerPosition>,
+}
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+pub enum DirectoryOperation {
+ RemoveTree,
+ MoveTree,
+ DeleteScope,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct MarkerPosition {
+ pub partition_id: u32,
+ pub epoch: u64,
+ pub seq: u64,
+}
+#[derive(Debug, Clone, Hash, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ScopeKey {
+ pub account_id: String,
+ pub partition_id: u32,
+ pub epoch: u64,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ScopedSeq {
+ pub scope: ScopeKey,
+ pub seq: u64,
+}
+#[derive(Debug, Clone)]
+pub struct PartitionContext {
+ pub scope: ScopeKey,
+ pub segment_dir: String,
+}
+#[derive(Debug, Clone)]
+pub struct PendingEvent {
+ pub operation_id: Uuid,
+ pub account_id: String,
+ pub kind: PendingEventKind,
+ pub path: String,
+ pub destination_path: Option<String>,
+ pub route_hint: Option<PartitionContext>,
+}
+#[derive(Debug, Clone)]
+pub enum PendingEventKind {
+ Data(SegmentEventType),
+ DeleteAccount,
+}
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
+#[repr(u8)]
+pub enum SegmentEventType {
+ Write = 1,
+ Remove = 2,
+ RemoveTree = 3,
+ MoveTree = 4,
+ DeleteScope = 5,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SegmentRecord {
+ pub seq: u64,
+ pub event_type: SegmentEventType,
+ pub path: String,
+ pub op_id: Option<Uuid>,
+ pub destination_path: Option<String>,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SegmentManifest {
+ pub version: u32,
+ pub next_seq: u64,
+ pub segments: Vec<SegmentDescriptor>,
+ pub backend_states: BTreeMap<String, BackendState>,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct SegmentDescriptor {
+ pub path: String,
+ pub segment_from_seq: u64,
+ pub segment_to_seq: Option<u64>,
+ pub record_count: u32,
+ pub byte_len: u64,
+ pub checksum: Option<String>,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct BackendState {
+ pub synced_seq: u64,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FlushResult {
+ pub scope: ScopeKey,
+ pub first_seq: u64,
+ pub last_seq: u64,
+ pub sealed_segments: Vec<SegmentDescriptor>,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DecodedSegment {
+ pub records: Vec<SegmentRecord>,
+ pub valid_bytes: u64,
+ pub truncated_tail: bool,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct FileState {
+ pub path: String,
+ pub latest_seq: ScopedSeq,
+ pub deleted: Option<bool>,
+}
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct CheckpointNode {
+ pub path_fragment: String,
+ pub latest_seq: Option<u64>,
+ pub deleted: Option<bool>,
+ pub children: Vec<CheckpointNode>,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct CheckpointsManifest {
+ pub version: u32,
+ #[serde(deserialize_with = "Option::deserialize")]
+ pub latest_checkpoint: Option<LatestCheckpoint>,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct LatestCheckpoint {
+ pub path: String,
+ pub checkpoint_to_seq: u64,
+ pub updated_at_ns: u64,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct CheckpointManifest {
+ pub version: u32,
+ pub partition_id: u32,
+ pub epoch: u64,
+ pub checkpoint_from_seq: u64,
+ pub checkpoint_to_seq: u64,
+ pub created_at_ns: u64,
+ pub root: ChunkDescriptor,
+ pub checksum: String,
+}
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ChunkDescriptor {
+ pub file: String,
+ pub root_path: String,
+ pub start_path: Option<String>,
+ pub end_path: Option<String>,
+ pub file_state_count: u32,
+ pub checksum: String,
+ pub chunks: Vec<ChunkDescriptor>,
+}
+impl ProtocolState {
+ /// Validate the closed protocol version value.
+ pub fn validate(&self) -> Result<()> {
+ Ok(())
+ }
+}
+impl PartitionsManifest {
+ /// Validate routing and directory marker invariants.
+ pub fn validate(&self) -> Result<()> {
+ validate_version("partitions manifest", self.version)?;
+ ensure!(self.epoch > 0, "partitions epoch must be positive");
+ ensure!(self.vbuckets == VBUCKETS, "invalid vbucket count");
+ ensure!(!self.partitions.is_empty(), "partitions must not be empty");
+ for entry in self.partitions.values() {
+ entry.validate()?;
+ }
+ validate_routes(&self.routes, self.partitions.keys().copied())?;
+ let mut operation_ids = HashSet::new();
+ for event in &self.directory_events {
+ ensure!(
+ operation_ids.insert(event.op_id),
+ "duplicate directory op_id"
+ );
+ event.validate()?;
+ for position in &event.positions {
+ ensure!(
+ position.epoch == self.epoch,
+ "marker epoch does not match manifest"
+ );
+ ensure!(
+ self.partitions
+ .get(&position.partition_id)
+ .is_some_and(|entry| entry.state == PartitionState::Stable),
+ "marker partition is unavailable"
+ );
+ }
+ }
+ ensure!(
+ self.partitions
+ .values()
+ .all(|entry| entry.state == PartitionState::Stable),
+ "partition manifest contains an unsupported state"
+ );
+ Ok(())
+ }
+}
+impl PartitionEntry {
+ /// Validate the closed partition state value.
+ pub fn validate(&self) -> Result<()> {
+ Ok(())
+ }
+}
+impl RouteEntry {
+ /// Validate one inclusive vbucket route range.
+ pub fn validate(&self) -> Result<()> {
+ ensure!(
+ self.start <= self.end && self.end < VBUCKETS,
+ "invalid route range"
+ );
+ Ok(())
+ }
+}
+impl PartitionManifest {
+ /// Validate one partition manifest version and epoch.
+ pub fn validate(&self) -> Result<()> {
+ validate_version("partition manifest", self.version)?;
+ ensure!(self.epoch > 0, "partition epoch must be positive");
+ Ok(())
+ }
+}
+impl DirectoryEvent {
+ /// Validate one directory operation and its marker positions.
+ pub fn validate(&self) -> Result<()> {
+ validate_path(&self.source_path)?;
+ match self.operation {
+ DirectoryOperation::RemoveTree | DirectoryOperation::DeleteScope => ensure!(
+ self.destination_path.is_none(),
+ "remove tree has a destination"
+ ),
+ DirectoryOperation::MoveTree => {
+ validate_path(self.destination_path.as_deref().ok_or_else(|| {
+ Error::Serialization("move tree requires a destination".into())
+ })?)?
+ }
+ }
+ let mut partitions = HashSet::new();
+ for position in &self.positions {
+ position.validate()?;
+ ensure!(
+ partitions.insert(position.partition_id),
+ "duplicate marker partition"
+ );
+ }
+ Ok(())
+ }
+}
+impl MarkerPosition {
+ /// Validate one persisted directory marker position.
+ pub fn validate(&self) -> Result<()> {
+ ensure!(self.epoch > 0 && self.seq > 0, "invalid marker position");
+ Ok(())
+ }
+}
+impl ScopeKey {
+ /// Validate an account-scoped partition identity.
+ pub fn validate(&self) -> Result<()> {
+ ensure!(
+ !self.account_id.trim().is_empty() && self.epoch > 0,
+ "invalid scope"
+ );
+ Ok(())
+ }
+ /// Validate this scope against its owning partitions manifest.
+ pub fn validate_against_manifest(&self, manifest: &PartitionsManifest) -> Result<()> {
+ self.validate()?;
+ ensure!(
+ manifest.partitions.contains_key(&self.partition_id),
+ "scope partition is unknown"
+ );
+ ensure!(
+ self.epoch == manifest.epoch,
+ "scope epoch does not match manifest"
+ );
+ Ok(())
+ }
+}
+impl ScopedSeq {
+ /// Validate a scoped sequence value.
+ pub fn validate(&self) -> Result<()> {
+ self.scope.validate()?;
+ ensure!(self.seq > 0, "scoped sequence must be positive");
+ Ok(())
+ }
+}
+impl SegmentRecord {
+ /// Validate a persisted segment record.
+ pub fn validate(&self) -> Result<()> {
+ ensure!(self.seq > 0, "record sequence must be positive");
+ validate_path(&self.path)?;
+ match self.event_type {
+ SegmentEventType::MoveTree => {
+ ensure!(self.op_id.is_some(), "move marker requires an op_id");
+ validate_path(self.destination_path.as_deref().ok_or_else(|| {
+ Error::Serialization("move marker requires a destination".into())
+ })?)?;
+ }
+ SegmentEventType::RemoveTree | SegmentEventType::DeleteScope => {
+ ensure!(self.op_id.is_some(), "remove marker requires an op_id");
+ ensure!(
+ self.destination_path.is_none(),
+ "remove marker has a destination"
+ );
+ }
+ _ => ensure!(
+ self.destination_path.is_none(),
+ "data event has a destination"
+ ),
+ }
+ Ok(())
+ }
+}
+impl SegmentManifest {
+ /// Validate segment continuity and backend progress.
+ pub fn validate(&self) -> Result<()> {
+ validate_version("segment manifest", self.version)?;
+ ensure!(self.next_seq > 0, "next sequence must be positive");
+ let (mut next, mut paths) = (None, HashSet::new());
+ for (index, segment) in self.segments.iter().enumerate() {
+ segment.validate()?;
+ ensure!(paths.insert(&segment.path), "duplicate segment path");
+ if let Some(expected) = next {
+ ensure!(segment.segment_from_seq == expected, "segment sequence gap");
+ }
+ ensure!(
+ segment.segment_to_seq.is_some() || index + 1 == self.segments.len(),
+ "head segment must be last"
+ );
+ next = segment
+ .segment_from_seq
+ .checked_add(u64::from(segment.record_count));
+ ensure!(next.is_some(), "segment sequence overflow");
+ }
+ if let Some(expected) = next {
+ ensure!(
+ self.next_seq == expected,
+ "next sequence does not follow segments"
+ );
+ }
+ for (backend, state) in &self.backend_states {
+ ensure!(!backend.trim().is_empty(), "backend id must not be empty");
+ state.validate(self.next_seq - 1)?;
+ }
+ Ok(())
+ }
+}
+impl SegmentDescriptor {
+ /// Validate one segment summary.
+ pub fn validate(&self) -> Result<()> {
+ ensure!(
+ !self.path.is_empty() && self.segment_from_seq > 0,
+ "invalid segment identity"
+ );
+ ensure!(self.byte_len >= 6, "segment byte length is too small");
+ match self.segment_to_seq {
+ Some(to) => {
+ ensure!(to >= self.segment_from_seq, "segment range is invalid");
+ ensure!(
+ self.path
+ == format!(
+ "{SEALED_SEGMENT_FILE_PREFIX}{:020}-{to:020}{SEGMENT_FILE_EXTENSION}",
+ self.segment_from_seq
+ ),
+ "sealed segment path does not match range"
+ );
+ ensure!(
+ u64::from(self.record_count) == to - self.segment_from_seq + 1,
+ "sealed segment count does not match range"
+ );
+ ensure!(
+ self.record_count <= MAX_SEGMENT_RECORDS as u32,
+ "sealed segment has too many records"
+ );
+ validate_checksum(self.checksum.as_deref().ok_or_else(|| {
+ Error::Serialization("sealed segment requires a checksum".into())
+ })?)?;
+ }
+ None => {
+ ensure!(
+ self.path
+ == format!(
+ "{HEAD_SEGMENT_FILE_PREFIX}{:020}{SEGMENT_FILE_EXTENSION}",
+ self.segment_from_seq
+ ),
+ "head segment path does not match start sequence"
+ );
+ ensure!(self.checksum.is_none(), "head segment has a checksum");
+ }
+ }
+ ensure!(
+ self.segment_from_seq
+ .checked_add(u64::from(self.record_count))
+ .is_some(),
+ "segment sequence overflow"
+ );
+ Ok(())
+ }
+}
+impl BackendState {
+ /// Validate backend progress against the committed sequence.
+ pub fn validate(&self, committed_seq: u64) -> Result<()> {
+ ensure!(
+ self.synced_seq <= committed_seq,
+ "backend progress is ahead"
+ );
+ Ok(())
+ }
+}
+impl FileState {
+ /// Validate a checkpoint file state against its owning scope.
+ pub fn validate(&self, expected_scope: &ScopeKey) -> Result<()> {
+ validate_path(&self.path)?;
+ self.latest_seq.validate()?;
+ ensure!(
+ &self.latest_seq.scope == expected_scope,
+ "file state crosses checkpoint scope"
+ );
+ validate_tombstone(self.deleted)
+ }
+}
+impl CheckpointNode {
+ /// Validate one checkpoint radix tree and its tombstones.
+ pub fn validate(&self) -> Result<()> {
+ ensure!(
+ self.latest_seq.is_none_or(|seq| seq > 0),
+ "invalid checkpoint node sequence"
+ );
+ validate_tombstone(self.deleted)?;
+ ensure!(
+ self.deleted.is_none() || self.latest_seq.is_some(),
+ "tombstone requires a sequence"
+ );
+ let mut previous = None;
+ for child in &self.children {
+ child.validate()?;
+ ensure!(
+ previous.is_none_or(|fragment| fragment < child.path_fragment.as_str()),
+ "checkpoint children are not strictly ordered"
+ );
+ previous = Some(child.path_fragment.as_str());
+ }
+ Ok(())
+ }
+}
+impl CheckpointsManifest {
+ /// Validate the checkpoint publication pointer.
+ pub fn validate(&self) -> Result<()> {
+ validate_version("checkpoints manifest", self.version)?;
+ if let Some(latest) = &self.latest_checkpoint {
+ latest.validate()?;
+ }
+ Ok(())
+ }
+}
+impl LatestCheckpoint {
+ /// Validate one published checkpoint reference.
+ pub fn validate(&self) -> Result<()> {
+ let name = self.path.strip_suffix('/').ok_or_else(|| {
+ Error::Serialization("checkpoint pointer must reference a directory".into())
+ })?;
+ ensure!(
+ is_checkpoint_directory_name(name),
+ "invalid checkpoint reference"
+ );
+ Ok(())
+ }
+}
+
+/// Return whether a single directory name identifies a published checkpoint.
+pub(crate) fn is_checkpoint_directory_name(name: &str) -> bool {
+ !name.contains('/') && name.starts_with(CHECKPOINT_DIR_PREFIX)
+}
+
+impl CheckpointManifest {
+ /// Validate a checkpoint manifest and its chunk tree.
+ pub fn validate(&self) -> Result<()> {
+ validate_version("checkpoint manifest", self.version)?;
+ ensure!(self.epoch > 0, "checkpoint epoch must be positive");
+ ensure!(
+ self.checkpoint_from_seq == 1 && self.checkpoint_to_seq == 0
+ || self.checkpoint_from_seq > 0
+ && self.checkpoint_from_seq <= self.checkpoint_to_seq,
+ "invalid checkpoint sequence range"
+ );
+ validate_checksum(&self.checksum)?;
+ self.root.validate()
+ }
+ /// Validate this checkpoint against its owning scope.
+ pub fn validate_for_scope(&self, scope: &ScopeKey) -> Result<()> {
+ self.validate()?;
+ scope.validate()?;
+ ensure!(
+ self.partition_id == scope.partition_id,
+ "checkpoint partition does not match scope"
+ );
+ ensure!(
+ self.epoch == scope.epoch,
+ "checkpoint epoch does not match scope"
+ );
+ Ok(())
+ }
+}
+impl ChunkDescriptor {
+ /// Validate a checkpoint chunk tree, including unique files and ranges.
+ pub fn validate(&self) -> Result<()> {
+ self.validate_inner(true, &mut HashSet::new())
+ }
+ /// Validate one chunk and recurse into its direct children.
+ fn validate_inner(&self, root: bool, files: &mut HashSet<String>) -> Result<()> {
+ ensure!(
+ !self.file.is_empty() && files.insert(self.file.clone()),
+ "invalid chunk file"
+ );
+ validate_path(&self.root_path)?;
+ ensure!(
+ self.file_state_count <= MAX_CHECKPOINT_NODES as u32,
+ "chunk has too many file states"
+ );
+ validate_checksum(&self.checksum)?;
+ let (range_start, range_end) =
+ match (root, self.start_path.as_deref(), self.end_path.as_deref()) {
+ (true, None, None) => (self.root_path.as_str(), None),
+ (true, _, _) => {
+ return Err(Error::Serialization(
+ "root chunk must not have bounds".into(),
+ ));
+ }
+ (false, Some(start), end) => {
+ if let Some(end) = end {
+ ensure!(start < end, "chunk range is reversed");
+ }
+ (start, end)
+ }
+ (false, None, _) => {
+ return Err(Error::Serialization(
+ "non-root chunk requires start bound".into(),
+ ));
+ }
+ };
+ let mut expected_start = Some(range_start);
+ for child in &self.chunks {
+ child.validate_inner(false, files)?;
+ let child_start = child.start_path.as_deref().unwrap();
+ ensure!(
+ expected_start.is_some_and(|start| start == child_start),
+ "chunk ranges are not contiguous"
+ );
+ expected_start = child.end_path.as_deref();
+ }
+ if !self.chunks.is_empty() {
+ ensure!(
+ expected_start == range_end,
+ "chunk ranges are not contiguous"
+ );
+ }
+ Ok(())
+ }
+}
+/// Validate a supported persisted format version.
+fn validate_version(name: &str, version: u32) -> Result<()> {
+ ensure!(
+ version == FORMAT_VERSION,
+ "unsupported {name} version {version}"
+ );
+ Ok(())
+}
+/// Validate a complete logical path.
+fn validate_path(path: &str) -> Result<()> {
+ ensure!(path.starts_with('/'), "logical path must be absolute");
+ Ok(())
+}
+/// Validate a lowercase SHA-256 hexadecimal checksum.
+fn validate_checksum(checksum: &str) -> Result<()> {
+ ensure!(
+ checksum.len() == 64
+ && checksum
+ .bytes()
+ .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)),
+ "invalid SHA-256 checksum"
+ );
+ Ok(())
+}
+/// Validate an optional deletion tombstone.
+fn validate_tombstone(deleted: Option<bool>) -> Result<()> {
+ ensure!(deleted != Some(false), "tombstone may only be true");
+ Ok(())
+}
+
+#[cfg(test)]
+#[path = "tests/test_model.rs"]
+mod tests;
diff --git a/crates/ragfs/src/multibackend/provider/cache.rs b/crates/ragfs/src/multibackend/provider/cache.rs
new file mode 100644
index 000000000..88161ba58
--- /dev/null
+++ b/crates/ragfs/src/multibackend/provider/cache.rs
@@ -0,0 +1,1555 @@
+//! CacheRuntime-backed V2 multi-write persistence.
+
+use std::collections::{BTreeMap, HashSet};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use bytes::Bytes;
+use serde::Deserialize;
+use uuid::Uuid;
+
+use crate::cache_runtime::{
+ CacheError, CacheOperation, CacheRuntime, ScriptDefinition, ScriptRequest, ScriptValue,
+};
+use crate::core::errors::{Error, Result};
+use crate::core::internal_names::is_multiwrite_internal_path;
+use crate::multibackend::codec::{decode_segment, encode_segment, sha256_hex};
+use crate::multibackend::constants::{
+ HEAD_SEGMENT_FILE_PREFIX, MAX_SEGMENT_RECORDS, SEALED_SEGMENT_FILE_PREFIX,
+ SEGMENT_FILE_EXTENSION,
+};
+use crate::multibackend::meta::MetadataStore;
+use crate::multibackend::model::{
+ BackendState, CheckpointsManifest, FlushResult, MarkerPosition, PartitionContext,
+ PartitionsManifest, PendingEvent, PendingEventKind, ScopeKey, SegmentDescriptor,
+ SegmentEventType, SegmentManifest, SegmentRecord,
+};
+use crate::multibackend::router::AccountRouter;
+
+use super::{read_sealed_segment, select_record_range, MultiWriteProvider};
+
+const FLUSH_SCRIPT_ID: &str = "multiwrite.flush.v1";
+const SEAL_SCRIPT_ID: &str = "multiwrite.seal.v1";
+const ADVANCE_SCRIPT_ID: &str = "multiwrite.advance.v1";
+
+const FLUSH_SCRIPT: &str = r#"
+if redis.call("EXISTS", KEYS[1]) == 0 then
+ return { "not_found" }
+end
+if redis.call("HGET", KEYS[1], "account") ~= ARGV[1]
+ or redis.call("HGET", KEYS[1], "partition") ~= ARGV[2]
+ or redis.call("HGET", KEYS[1], "epoch") ~= ARGV[3] then
+ return { "scope_mismatch" }
+end
+local requested = #ARGV - 6
+if ARGV[4] == "1" and requested ~= 1 then
+ return { "invalid", "marker flush requires exactly one record" }
+end
+local owner = redis.call("GET", KEYS[3])
+if ARGV[6] == "1" then
+ if owner ~= ARGV[5] then
+ return { "fenced" }
+ end
+elseif owner and owner ~= ARGV[5] then
+ return { "busy" }
+else
+ redis.call("SET", KEYS[3], ARGV[5], "PX", 30000)
+end
+redis.call("PEXPIRE", KEYS[3], 30000)
+local marker_seq = tonumber(redis.call("HGET", KEYS[1], "marker_seq"))
+if marker_seq ~= 0 then
+ return { "seal_required" }
+end
+local head_len = redis.call("LLEN", KEYS[2])
+local capacity = 8192 - head_len
+if capacity <= 0 then
+ return { "seal_required" }
+end
+local count = math.min(requested, capacity)
+local next_seq = tonumber(redis.call("HGET", KEYS[1], "next_seq"))
+local first_seq = next_seq
+for index = 1, count do
+ local record = cjson.decode(ARGV[index + 6])
+ record.seq = next_seq
+ redis.call("RPUSH", KEYS[2], cjson.encode(record))
+ next_seq = next_seq + 1
+end
+if ARGV[4] == "1" then
+ marker_seq = next_seq - 1
+end
+local next_seq_text = string.format("%.0f", next_seq)
+local marker_seq_text = string.format("%.0f", marker_seq)
+redis.call("HSET", KEYS[1], "next_seq", next_seq_text,
+ "marker_seq", marker_seq_text)
+local needs_seal = 0
+if redis.call("LLEN", KEYS[2]) == 8192 or marker_seq ~= 0 then
+ needs_seal = 1
+end
+if needs_seal == 0 then
+ redis.call("DEL", KEYS[3])
+else
+ redis.call("PEXPIRE", KEYS[3], 30000)
+end
+return { "ok", string.format("%.0f", first_seq),
+ string.format("%.0f", next_seq - 1), count, needs_seal }
+"#;
+
+const SEAL_SCRIPT: &str = r#"
+local mode = ARGV[1]
+if mode == "bootstrap" then
+ if redis.call("EXISTS", KEYS[1]) ~= 0 then
+ return { "exists" }
+ end
+ redis.call("DEL", KEYS[1])
+ redis.call("HSET", KEYS[1],
+ "account", ARGV[2],
+ "partition", ARGV[3],
+ "epoch", ARGV[4],
+ "version", ARGV[5],
+ "next_seq", ARGV[6],
+ "marker_seq", "0",
+ "segments", ARGV[7],
+ "backend_states", ARGV[8])
+ redis.call("DEL", KEYS[2])
+ redis.call("DEL", KEYS[3])
+ return { "ok" }
+end
+if redis.call("EXISTS", KEYS[1]) == 0 then
+ return { "not_found" }
+end
+if redis.call("HGET", KEYS[1], "account") ~= ARGV[2]
+ or redis.call("HGET", KEYS[1], "partition") ~= ARGV[3]
+ or redis.call("HGET", KEYS[1], "epoch") ~= ARGV[4] then
+ return { "scope_mismatch" }
+end
+if mode == "read" then
+ return {
+ "ok",
+ redis.call("HGET", KEYS[1], "account"),
+ redis.call("HGET", KEYS[1], "partition"),
+ redis.call("HGET", KEYS[1], "epoch"),
+ redis.call("HGET", KEYS[1], "version"),
+ redis.call("HGET", KEYS[1], "next_seq"),
+ redis.call("HGET", KEYS[1], "marker_seq"),
+ redis.call("HGET", KEYS[1], "segments"),
+ redis.call("HGET", KEYS[1], "backend_states")
+ }
+end
+local segments = cjson.decode(redis.call("HGET", KEYS[1], "segments"))
+if mode == "commit" then
+ local first_seq = tonumber(ARGV[5])
+ local last_seq = tonumber(ARGV[6])
+ local count = tonumber(ARGV[7])
+ local descriptor = cjson.decode(ARGV[8])
+ local owner = ARGV[9]
+ local keep_owner = ARGV[10]
+ for _, current in ipairs(segments) do
+ if current.path == descriptor.path then
+ if current.segment_from_seq == descriptor.segment_from_seq
+ and current.segment_to_seq == descriptor.segment_to_seq
+ and current.record_count == descriptor.record_count
+ and current.byte_len == descriptor.byte_len
+ and current.checksum == descriptor.checksum then
+ if owner ~= "" and redis.call("GET", KEYS[3]) == owner then
+ if keep_owner == "1" then
+ redis.call("PEXPIRE", KEYS[3], 30000)
+ else
+ redis.call("DEL", KEYS[3])
+ end
+ end
+ return { "already" }
+ end
+ return { "invalid", "sealed descriptor path conflicts" }
+ end
+ end
+ local current_owner = redis.call("GET", KEYS[3])
+ if owner == "" then
+ if current_owner then
+ return { "busy" }
+ end
+ elseif current_owner ~= owner then
+ return { "fenced" }
+ end
+ if #segments > 0
+ and segments[#segments].segment_to_seq + 1 ~= first_seq then
+ return { "invalid", "sealed descriptor is not contiguous" }
+ end
+ if count ~= last_seq - first_seq + 1 then
+ return { "invalid", "sealed descriptor count is invalid" }
+ end
+ if redis.call("LLEN", KEYS[2]) < count then
+ return { "changed" }
+ end
+ for index = 1, count do
+ if redis.call("LINDEX", KEYS[2], index - 1) ~= ARGV[index + 10] then
+ return { "changed" }
+ end
+ end
+ segments[#segments + 1] = descriptor
+ redis.call("HSET", KEYS[1], "segments", cjson.encode(segments))
+ redis.call("LTRIM", KEYS[2], count, -1)
+ local marker_seq = tonumber(redis.call("HGET", KEYS[1], "marker_seq"))
+ if marker_seq ~= 0 and marker_seq <= last_seq then
+ redis.call("HSET", KEYS[1], "marker_seq", "0")
+ end
+ if owner ~= "" then
+ if keep_owner == "1" then
+ redis.call("PEXPIRE", KEYS[3], 30000)
+ else
+ redis.call("DEL", KEYS[3])
+ end
+ end
+ return { "ok" }
+end
+if mode == "prune" then
+ local requested = cjson.decode(ARGV[5])
+ local requested_set = {}
+ for _, path in ipairs(requested) do
+ if requested_set[path] then
+ return { "invalid", "prune paths contain duplicates" }
+ end
+ requested_set[path] = true
+ end
+ local removed = {}
+ local kept = {}
+ for _, descriptor in ipairs(segments) do
+ if requested_set[descriptor.path] then
+ removed[#removed + 1] = descriptor
+ requested_set[descriptor.path] = nil
+ else
+ kept[#kept + 1] = descriptor
+ end
+ end
+ for path, _ in pairs(requested_set) do
+ return { "invalid", "sealed segment is not referenced: " .. path }
+ end
+ for index = 2, #kept do
+ if kept[index].segment_from_seq ~= kept[index - 1].segment_to_seq + 1 then
+ return { "invalid", "prune would create a segment gap" }
+ end
+ end
+ local head = redis.call("LINDEX", KEYS[2], 0)
+ if head and #kept > 0 then
+ local first_head = cjson.decode(head)
+ if first_head.seq ~= kept[#kept].segment_to_seq + 1 then
+ return { "invalid", "prune would create a head gap" }
+ end
+ elseif not head and #kept > 0 then
+ local next_seq = tonumber(redis.call("HGET", KEYS[1], "next_seq"))
+ if next_seq ~= kept[#kept].segment_to_seq + 1 then
+ return { "invalid", "prune would detach next_seq" }
+ end
+ end
+ local encoded_kept = "[]"
+ if #kept > 0 then
+ encoded_kept = cjson.encode(kept)
+ end
+ local encoded_removed = "[]"
+ if #removed > 0 then
+ encoded_removed = cjson.encode(removed)
+ end
+ redis.call("HSET", KEYS[1], "segments", encoded_kept)
+ return { "ok", encoded_removed }
+end
+return { "invalid", "unknown seal operation" }
+"#;
+
+const ADVANCE_SCRIPT: &str = r#"
+if redis.call("EXISTS", KEYS[1]) == 0 then
+ return { "not_found" }
+end
+if redis.call("HGET", KEYS[1], "account") ~= ARGV[1]
+ or redis.call("HGET", KEYS[1], "partition") ~= ARGV[2]
+ or redis.call("HGET", KEYS[1], "epoch") ~= ARGV[3] then
+ return { "scope_mismatch" }
+end
+local states = cjson.decode(redis.call("HGET", KEYS[1], "backend_states"))
+local state = states[ARGV[4]]
+if not state then
+ return { "invalid", "unknown backend id: " .. ARGV[4] }
+end
+local synced_seq = tonumber(ARGV[5])
+if synced_seq ~= 0 and synced_seq < state.synced_seq then
+ return { "invalid", "backend progress cannot regress" }
+end
+local next_seq = tonumber(redis.call("HGET", KEYS[1], "next_seq"))
+if synced_seq >= next_seq then
+ return { "invalid", "backend progress must remain below next_seq" }
+end
+state.synced_seq = synced_seq
+redis.call("HSET", KEYS[1], "backend_states", cjson.encode(states))
+return { "ok" }
+"#;
+
+const SCRIPT_DEFINITIONS: &[ScriptDefinition] = &[
+ ScriptDefinition {
+ id: FLUSH_SCRIPT_ID,
+ redis_lua: FLUSH_SCRIPT,
+ },
+ ScriptDefinition {
+ id: SEAL_SCRIPT_ID,
+ redis_lua: SEAL_SCRIPT,
+ },
+ ScriptDefinition {
+ id: ADVANCE_SCRIPT_ID,
+ redis_lua: ADVANCE_SCRIPT,
+ },
+];
+
+/// Cache keys owned by one account partition.
+pub(crate) struct CachePartitionKeys {
+ pub(crate) state: String,
+ pub(crate) head: String,
+ pub(crate) flush_gate: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct CacheState {
+ scope: ScopeKey,
+ version: u32,
+ next_seq: u64,
+ marker_seq: u64,
+ segments: Vec<SegmentDescriptor>,
+ backend_states: BTreeMap<String, BackendState>,
+}
+
+struct FlushScriptResult {
+ first_seq: u64,
+ last_seq: u64,
+ appended: usize,
+ needs_seal: bool,
+}
+
+enum FlushScriptOutcome {
+ Appended(FlushScriptResult),
+ SealRequired,
+ Busy,
+}
+
+/// Persists V2 mutable state through CacheRuntime and immutable segments on the primary filesystem.
+pub struct CacheProvider {
+ store: Arc<MetadataStore>,
+ router: AccountRouter,
+ runtime: Arc<CacheRuntime>,
+ namespace: String,
+}
+
+impl CacheProvider {
+ /// Creates a cache-backed metadata provider.
+ pub fn new(
+ store: Arc<MetadataStore>,
+ runtime: Arc<CacheRuntime>,
+ namespace: String,
+ ) -> Result<Self> {
+ Self::build(store, runtime, namespace)
+ }
+
+ /// Validates runtime capabilities and registers stable provider scripts.
+ fn build(
+ store: Arc<MetadataStore>,
+ runtime: Arc<CacheRuntime>,
+ namespace: String,
+ ) -> Result<Self> {
+ Self::validate_namespace(&namespace)?;
+ runtime
+ .require_operations(&[CacheOperation::Lrange, CacheOperation::ExecuteScript])
+ .map_err(|error| {
+ Error::config(format!(
+ "Cache-backed multi-write provider is incompatible: {error}"
+ ))
+ })?;
+ for definition in SCRIPT_DEFINITIONS {
+ runtime.register_script(*definition).map_err(|error| {
+ Error::config(format!(
+ "Cache-backed multi-write script registration failed: {error}"
+ ))
+ })?;
+ }
+ Ok(Self {
+ router: AccountRouter::new(store.clone()),
+ store,
+ runtime,
+ namespace,
+ })
+ }
+
+ /// Validate the Redis hash tag namespace used by multi-write cache keys.
+ fn validate_namespace(namespace: &str) -> Result<()> {
+ if namespace.trim().is_empty() || namespace.contains(['{', '}']) {
+ return Err(Error::config(
+ "cache-backed multi-write namespace must be non-empty and must not contain '{' or '}'",
+ ));
+ }
+ Ok(())
+ }
+
+ /// Builds deterministic same-slot keys for one validated scope.
+ pub(crate) fn keys_for_scope(&self, scope: &ScopeKey) -> Result<CachePartitionKeys> {
+ scope.validate()?;
+ if matches!(scope.account_id.as_str(), "." | "..")
+ || scope.account_id.contains('/')
+ || scope
+ .account_id
+ .bytes()
+ .any(|byte| byte.is_ascii_control() || matches!(byte, b'{' | b'}'))
+ {
+ return Err(Error::invalid_path(
+ "Cache-backed multi-write account id contains an invalid key character",
+ ));
+ }
+ let prefix = format!(
+ "{{{}}}:ov:multiwrite:{}:{}",
+ self.namespace, scope.account_id, scope.partition_id
+ );
+ Ok(CachePartitionKeys {
+ state: format!("{prefix}:state"),
+ head: format!("{prefix}:head"),
+ flush_gate: format!("{prefix}:flush-gate"),
+ })
+ }
+
+ /// Validates normal flush input before routing or cache access.
+ fn validate_flush_events(events: &[PendingEvent]) -> Result<()> {
+ let first = events
+ .first()
+ .ok_or_else(|| Error::invalid_operation("flush requires at least one event"))?;
+ for event in events {
+ if event.account_id != first.account_id {
+ return Err(Error::invalid_operation(
+ "flush events must belong to one account",
+ ));
+ }
+ if is_multiwrite_internal_path(&event.path) {
+ return Err(Error::invalid_path(format!(
+ "cannot flush internal path: {}",
+ event.path
+ )));
+ }
+ match event.kind {
+ PendingEventKind::Data(SegmentEventType::Write | SegmentEventType::Remove) => {}
+ _ => {
+ return Err(Error::invalid_operation(
+ "normal flush accepts only write and remove events",
+ ))
+ }
+ }
+ if event.destination_path.is_some() {
+ return Err(Error::invalid_operation(
+ "normal data event cannot have a destination",
+ ));
+ }
+ }
+ Ok(())
+ }
+
+ /// Routes every event and requires one current partition scope.
+ async fn route_events(&self, events: &[PendingEvent]) -> Result<PartitionContext> {
+ let mut selected = None;
+ for event in events {
+ let current = self.router.route(&event.account_id, &event.path).await?;
+ if selected
+ .as_ref()
+ .is_some_and(|existing: &PartitionContext| existing.scope != current.scope)
+ {
+ return Err(Error::invalid_operation(
+ "flush events route to different partition scopes",
+ ));
+ }
+ selected.get_or_insert(current);
+ }
+ selected.ok_or_else(|| Error::invalid_operation("flush requires at least one event"))
+ }
+
+ /// Validates a scope against its current account partitions manifest.
+ async fn validate_scope(&self, scope: &ScopeKey) -> Result<()> {
+ self.keys_for_scope(scope)?;
+ let path = self.store.paths().account_manifest(&scope.account_id)?.0;
+ let manifest: PartitionsManifest = self.store.read_json(&path).await?;
+ manifest.validate()?;
+ scope.validate_against_manifest(&manifest)
+ }
+
+ /// Converts one pending event into a validated record template.
+ fn record(event: &PendingEvent) -> Result<SegmentRecord> {
+ let event_type = match event.kind {
+ PendingEventKind::Data(kind) => kind,
+ PendingEventKind::DeleteAccount => {
+ return Err(Error::invalid_operation(
+ "account deletion is not a segment record",
+ ))
+ }
+ };
+ let marker = matches!(
+ event_type,
+ SegmentEventType::RemoveTree
+ | SegmentEventType::MoveTree
+ | SegmentEventType::DeleteScope
+ );
+ let record = SegmentRecord {
+ seq: 1,
+ event_type,
+ path: event.path.clone(),
+ op_id: marker.then_some(event.operation_id),
+ destination_path: event.destination_path.clone(),
+ };
+ record.validate()?;
+ encode_segment(std::slice::from_ref(&record))?;
+ Ok(record)
+ }
+
+ /// Validates one marker against its caller-selected scope.
+ fn validate_marker(&self, scope: &ScopeKey, event: &PendingEvent) -> Result<()> {
+ if event.account_id != scope.account_id {
+ return Err(Error::invalid_operation(
+ "marker account does not match supplied scope",
+ ));
+ }
+ match event.kind {
+ PendingEventKind::Data(
+ SegmentEventType::RemoveTree
+ | SegmentEventType::MoveTree
+ | SegmentEventType::DeleteScope,
+ ) => {}
+ _ => {
+ return Err(Error::invalid_operation(
+ "append_marker accepts only directory markers",
+ ))
+ }
+ }
+ if is_multiwrite_internal_path(&event.path) {
+ return Err(Error::invalid_path(format!(
+ "cannot append marker for internal path: {}",
+ event.path
+ )));
+ }
+ if !matches!(
+ event.kind,
+ PendingEventKind::Data(SegmentEventType::DeleteScope)
+ ) {
+ self.store
+ .paths()
+ .backend_path(&scope.account_id, &event.path)?;
+ }
+ if let Some(destination) = &event.destination_path {
+ if is_multiwrite_internal_path(destination) {
+ return Err(Error::invalid_path(format!(
+ "cannot append marker for internal destination: {destination}"
+ )));
+ }
+ self.store
+ .paths()
+ .backend_path(&scope.account_id, destination)?;
+ }
+ Self::record(event).map(|_| ())
+ }
+
+ /// Executes one registered script and requires an array result.
+ async fn execute_array(
+ &self,
+ script_id: &str,
+ keys: &CachePartitionKeys,
+ args: Vec<Bytes>,
+ ) -> Result<Vec<ScriptValue>> {
+ let result = self
+ .runtime
+ .execute_script(ScriptRequest {
+ script_id: script_id.to_string(),
+ keys: vec![
+ keys.state.clone(),
+ keys.head.clone(),
+ keys.flush_gate.clone(),
+ ],
+ args,
+ })
+ .await
+ .and_then(|result| result.decode())
+ .map_err(|error| Self::cache_error(script_id, error))?;
+ match result {
+ ScriptValue::Array(values) => Ok(values),
+ other => Err(Error::Serialization(format!(
+ "Cache-backed multi-write {script_id} returned non-array result: {other:?}"
+ ))),
+ }
+ }
+
+ /// Executes one bounded atomic append attempt.
+ async fn flush_once(
+ &self,
+ scope: &ScopeKey,
+ records: &[SegmentRecord],
+ marker: bool,
+ owner: &str,
+ continuing: bool,
+ ) -> Result<FlushScriptOutcome> {
+ let keys = self.keys_for_scope(scope)?;
+ let mut args = Self::scope_args(scope);
+ args.push(Bytes::from(if marker { "1" } else { "0" }));
+ args.push(Bytes::from(owner.to_string()));
+ args.push(Bytes::from(if continuing { "1" } else { "0" }));
+ for record in records {
+ args.push(Bytes::from(serde_json::to_vec(record)?));
+ }
+ let values = self.execute_array(FLUSH_SCRIPT_ID, &keys, args).await?;
+ match Self::status(&values, "flush")? {
+ "ok" => Ok(FlushScriptOutcome::Appended(FlushScriptResult {
+ first_seq: Self::u64_value(&values, 1, "flush first_seq")?,
+ last_seq: Self::u64_value(&values, 2, "flush last_seq")?,
+ appended: Self::usize_value(&values, 3, "flush appended count")?,
+ needs_seal: Self::u64_value(&values, 4, "flush seal flag")? == 1,
+ })),
+ "seal_required" => Ok(FlushScriptOutcome::SealRequired),
+ "busy" => Ok(FlushScriptOutcome::Busy),
+ "fenced" => Err(Error::would_block(
+ "cache-backed flush owner was fenced by another writer",
+ )),
+ "not_found" => Err(Error::not_found(keys.state)),
+ "scope_mismatch" => Err(Error::invalid_operation(
+ "Cache-backed multi-write state scope does not match",
+ )),
+ "invalid" => Err(Self::script_invalid(&values, "flush")),
+ other => Err(Self::unknown_status("flush", other)),
+ }
+ }
+
+ /// Revalidates routing and executes one append while holding the account manifest lock.
+ async fn flush_routed_once(
+ &self,
+ events: &[PendingEvent],
+ expected_scope: Option<&ScopeKey>,
+ records: &[SegmentRecord],
+ owner: &str,
+ continuing: bool,
+ ) -> Result<(PartitionContext, FlushScriptOutcome)> {
+ let account_id = &events
+ .first()
+ .ok_or_else(|| Error::invalid_operation("flush requires at least one event"))?
+ .account_id;
+ let manifest_path = self.store.paths().account_manifest(account_id)?.0;
+ let lock_timeout = self
+ .store
+ .pathlock_manager()
+ .default_lock_timeout()
+ .max(std::time::Duration::from_secs(30));
+ let lease = self
+ .store
+ .pathlock_manager()
+ .acquire_exact(&manifest_path, lock_timeout, None)
+ .await?;
+ let operation = async {
+ let route = self.route_events(events).await?;
+ if expected_scope.is_some_and(|scope| scope != &route.scope) {
+ return Err(Error::would_block(
+ "multi-write route changed during cache-backed flush",
+ ));
+ }
+ let outcome = self
+ .flush_once(&route.scope, records, false, owner, continuing)
+ .await?;
+ Ok((route, outcome))
+ }
+ .await;
+ let release = self
+ .store
+ .pathlock_manager()
+ .release(&lease)
+ .await
+ .map_err(Error::from);
+ match (operation, release) {
+ (Err(error), _) => Err(error),
+ (Ok(_), Err(error)) => Err(error),
+ (Ok(result), Ok(())) => Ok(result),
+ }
+ }
+
+ /// Revalidates a supplied scope and appends one marker under its manifest lock.
+ async fn flush_marker_once(
+ &self,
+ scope: &ScopeKey,
+ record: &SegmentRecord,
+ owner: &str,
+ continuing: bool,
+ ) -> Result<FlushScriptOutcome> {
+ let manifest_path = self.store.paths().account_manifest(&scope.account_id)?.0;
+ let lock_timeout = self
+ .store
+ .pathlock_manager()
+ .default_lock_timeout()
+ .max(std::time::Duration::from_secs(30));
+ let lease = self
+ .store
+ .pathlock_manager()
+ .acquire_exact(&manifest_path, lock_timeout, None)
+ .await?;
+ let operation = async {
+ self.validate_scope(scope).await?;
+ self.flush_once(scope, std::slice::from_ref(record), true, owner, continuing)
+ .await
+ }
+ .await;
+ let release = self
+ .store
+ .pathlock_manager()
+ .release(&lease)
+ .await
+ .map_err(Error::from);
+ match (operation, release) {
+ (Err(error), _) => Err(error),
+ (Ok(_), Err(error)) => Err(error),
+ (Ok(result), Ok(())) => Ok(result),
+ }
+ }
+
+ /// Reads one stable cache state and head-list snapshot.
+ async fn snapshot(
+ &self,
+ scope: &ScopeKey,
+ ) -> Result<(CacheState, Vec<SegmentRecord>, Vec<Bytes>)> {
+ self.validate_scope(scope).await?;
+ let keys = self.keys_for_scope(scope)?;
+ loop {
+ let before = self.read_state_once(scope, &keys).await?;
+ let raw_records = self
+ .runtime
+ .lrange(&keys.head, 0, -1)
+ .await
+ .map_err(|error| Self::cache_error("read head", error))?;
+ let after = self.read_state_once(scope, &keys).await?;
+ if before == after {
+ let records = raw_records
+ .iter()
+ .map(|raw| {
+ serde_json::from_slice::<SegmentRecord>(raw)
+ .map_err(|error| Error::Serialization(error.to_string()))
+ })
+ .collect::<Result<Vec<_>>>()?;
+ Self::validate_snapshot(&before, &records)?;
+ return Ok((before, records, raw_records));
+ }
+ }
+ }
+
+ /// Reads one cache state value through the shared seal script.
+ async fn read_state_once(
+ &self,
+ scope: &ScopeKey,
+ keys: &CachePartitionKeys,
+ ) -> Result<CacheState> {
+ let mut args = vec![Bytes::from_static(b"read")];
+ args.extend(Self::scope_args(scope));
+ let values = self.execute_array(SEAL_SCRIPT_ID, keys, args).await?;
+ match Self::status(&values, "read state")? {
+ "ok" => {
+ let state = CacheState {
+ scope: ScopeKey {
+ account_id: Self::string_value(&values, 1, "state account")?.to_string(),
+ partition_id: Self::u32_value(&values, 2, "state partition")?,
+ epoch: Self::u64_value(&values, 3, "state epoch")?,
+ },
+ version: Self::u32_value(&values, 4, "state version")?,
+ next_seq: Self::u64_value(&values, 5, "state next_seq")?,
+ marker_seq: Self::u64_value(&values, 6, "state marker_seq")?,
+ segments: Self::json_value(&values, 7, "state segments")?,
+ backend_states: Self::json_value(&values, 8, "state backend states")?,
+ };
+ Self::validate_state(scope, &state)?;
+ Ok(state)
+ }
+ "not_found" => Err(Error::not_found(keys.state.clone())),
+ "scope_mismatch" => Err(Error::invalid_operation(
+ "Cache-backed multi-write state scope does not match",
+ )),
+ other => Err(Self::unknown_status("read state", other)),
+ }
+ }
+
+ /// Validates cache state fields that do not depend on the mutable head.
+ fn validate_state(scope: &ScopeKey, state: &CacheState) -> Result<()> {
+ if &state.scope != scope {
+ return Err(Error::Serialization(
+ "Cache-backed multi-write state has the wrong scope".to_string(),
+ ));
+ }
+ if state.version != 1 || state.next_seq == 0 {
+ return Err(Error::Serialization(
+ "Cache-backed multi-write state has invalid version or next_seq".to_string(),
+ ));
+ }
+ let mut previous_to = None;
+ let mut paths = HashSet::new();
+ for descriptor in &state.segments {
+ descriptor.validate()?;
+ if descriptor.segment_to_seq.is_none() || !paths.insert(&descriptor.path) {
+ return Err(Error::Serialization(
+ "cache state contains an invalid sealed descriptor".to_string(),
+ ));
+ }
+ if previous_to.is_some_and(|to| descriptor.segment_from_seq != to + 1) {
+ return Err(Error::Serialization(
+ "cache sealed descriptors are not contiguous".to_string(),
+ ));
+ }
+ previous_to = descriptor.segment_to_seq;
+ }
+ for (backend, backend_state) in &state.backend_states {
+ if backend.trim().is_empty() {
+ return Err(Error::Serialization(
+ "cache backend id must not be empty".to_string(),
+ ));
+ }
+ backend_state.validate(state.next_seq - 1)?;
+ }
+ if state.marker_seq >= state.next_seq {
+ return Err(Error::Serialization(
+ "cache marker is ahead of next_seq".to_string(),
+ ));
+ }
+ Ok(())
+ }
+
+ /// Validates head ordering and its continuity with cache manifest state.
+ fn validate_snapshot(state: &CacheState, records: &[SegmentRecord]) -> Result<()> {
+ if records.len() > MAX_SEGMENT_RECORDS {
+ return Err(Error::Serialization(
+ "cache head contains too many records".to_string(),
+ ));
+ }
+ for (index, record) in records.iter().enumerate() {
+ record.validate()?;
+ if index > 0 && record.seq != records[index - 1].seq + 1 {
+ return Err(Error::Serialization(
+ "cache head contains a sequence gap".to_string(),
+ ));
+ }
+ }
+ if let Some(first) = records.first() {
+ if state
+ .segments
+ .last()
+ .and_then(|descriptor| descriptor.segment_to_seq)
+ .is_some_and(|last| first.seq != last + 1)
+ {
+ return Err(Error::Serialization(
+ "cache head does not follow sealed segments".to_string(),
+ ));
+ }
+ if records.last().unwrap().seq.checked_add(1) != Some(state.next_seq) {
+ return Err(Error::Serialization(
+ "cache head does not end at next_seq".to_string(),
+ ));
+ }
+ } else if state
+ .segments
+ .last()
+ .and_then(|descriptor| descriptor.segment_to_seq)
+ .is_some_and(|last| last.checked_add(1) != Some(state.next_seq))
+ {
+ return Err(Error::Serialization(
+ "cache next_seq does not follow sealed segments".to_string(),
+ ));
+ }
+ if state.marker_seq != 0
+ && records.last().map(|record| record.seq) != Some(state.marker_seq)
+ {
+ return Err(Error::Serialization(
+ "cache marker is not the final head record".to_string(),
+ ));
+ }
+ Ok(())
+ }
+
+ /// Converts a stable snapshot into the public validated manifest.
+ fn manifest(state: CacheState, records: &[SegmentRecord]) -> Result<SegmentManifest> {
+ let mut manifest = SegmentManifest {
+ version: state.version,
+ next_seq: state.next_seq,
+ segments: state.segments,
+ backend_states: state.backend_states,
+ };
+ if let Some(first) = records.first() {
+ let bytes = encode_segment(records)?;
+ manifest.segments.push(SegmentDescriptor {
+ path: Self::head_name(first.seq),
+ segment_from_seq: first.seq,
+ segment_to_seq: None,
+ record_count: records.len() as u32,
+ byte_len: bytes.len() as u64,
+ checksum: None,
+ });
+ }
+ manifest.validate()?;
+ Ok(manifest)
+ }
+
+ /// Persists one immutable segment snapshot on the shared raw primary.
+ async fn persist_sealed(
+ &self,
+ scope: &ScopeKey,
+ records: &[SegmentRecord],
+ ) -> Result<SegmentDescriptor> {
+ let first = records
+ .first()
+ .ok_or_else(|| Error::invalid_operation("cannot seal an empty cache head"))?
+ .seq;
+ let last = records.last().unwrap().seq;
+ let bytes = encode_segment(records)?;
+ let descriptor = SegmentDescriptor {
+ path: Self::sealed_name(first, last),
+ segment_from_seq: first,
+ segment_to_seq: Some(last),
+ record_count: records.len() as u32,
+ byte_len: bytes.len() as u64,
+ checksum: Some(sha256_hex(&bytes)),
+ };
+ let directory = self
+ .store
+ .paths()
+ .segments_dir(&scope.account_id, scope.partition_id)?
+ .0;
+ self.store
+ .write_immutable(&format!("{directory}/{}", descriptor.path), &bytes)
+ .await?;
+ Ok(descriptor)
+ }
+
+ /// Commits one immutable descriptor and trims its exact cache head prefix.
+ pub(crate) async fn commit_seal(
+ &self,
+ scope: &ScopeKey,
+ raw_records: &[Bytes],
+ descriptor: &SegmentDescriptor,
+ owner: Option<&str>,
+ keep_owner: bool,
+ ) -> Result<()> {
+ let keys = self.keys_for_scope(scope)?;
+ let mut args = vec![Bytes::from_static(b"commit")];
+ args.extend(Self::scope_args(scope));
+ args.push(Bytes::from(descriptor.segment_from_seq.to_string()));
+ args.push(Bytes::from(descriptor.segment_to_seq.unwrap().to_string()));
+ args.push(Bytes::from(descriptor.record_count.to_string()));
+ args.push(Bytes::from(serde_json::to_vec(descriptor)?));
+ args.push(Bytes::from(owner.unwrap_or_default().to_string()));
+ args.push(Bytes::from(if keep_owner { "1" } else { "0" }));
+ args.extend(raw_records.iter().cloned());
+ let values = self.execute_array(SEAL_SCRIPT_ID, &keys, args).await?;
+ match Self::status(&values, "seal")? {
+ "ok" | "already" => Ok(()),
+ "not_found" => Err(Error::not_found(keys.state)),
+ "busy" => Err(Error::would_block("cache head is owned by an active flush")),
+ "fenced" => Err(Error::would_block(
+ "cache seal owner was fenced by another writer",
+ )),
+ "changed" => Err(Error::would_block(
+ "cache head changed while committing a sealed prefix",
+ )),
+ "scope_mismatch" => Err(Error::invalid_operation(
+ "Cache-backed multi-write state scope does not match",
+ )),
+ "invalid" => Err(Self::script_invalid(&values, "seal")),
+ other => Err(Self::unknown_status("seal", other)),
+ }
+ }
+
+ /// Seals one snapshot while enforcing an optional flush-owner fence.
+ async fn seal_head_with_owner(
+ &self,
+ scope: &ScopeKey,
+ owner: Option<&str>,
+ keep_owner: bool,
+ ) -> Result<Option<SegmentDescriptor>> {
+ let (_, records, raw_records) = self.snapshot(scope).await?;
+ if records.is_empty() {
+ return Ok(None);
+ }
+ let directory = self
+ .store
+ .paths()
+ .segments_dir(&scope.account_id, scope.partition_id)?
+ .0;
+ let path = format!(
+ "{directory}/{}",
+ Self::sealed_name(records.first().unwrap().seq, records.last().unwrap().seq)
+ );
+ let lease = self
+ .store
+ .pathlock_manager()
+ .acquire_exact(
+ &path,
+ self.store
+ .pathlock_manager()
+ .default_lock_timeout()
+ .max(std::time::Duration::from_secs(30)),
+ None,
+ )
+ .await?;
+ let operation = async {
+ let descriptor = self.persist_sealed(scope, &records).await?;
+ self.commit_seal(scope, &raw_records, &descriptor, owner, keep_owner)
+ .await?;
+ Ok(Some(descriptor))
+ }
+ .await;
+ let release = self
+ .store
+ .pathlock_manager()
+ .release(&lease)
+ .await
+ .map_err(Error::from);
+ match operation {
+ Ok(result) => release.map(|()| result),
+ Err(error) => Err(error),
+ }
+ }
+
+ /// Returns whether one marker is the end of a referenced sealed segment.
+ async fn marker_is_sealed(&self, scope: &ScopeKey, seq: u64) -> Result<bool> {
+ Ok(self
+ .read_manifest(scope)
+ .await?
+ .segments
+ .iter()
+ .any(|descriptor| descriptor.segment_to_seq == Some(seq)))
+ }
+
+ /// Returns the deterministic mutable head filename.
+ fn head_name(from_seq: u64) -> String {
+ format!("{HEAD_SEGMENT_FILE_PREFIX}{from_seq:020}{SEGMENT_FILE_EXTENSION}")
+ }
+
+ /// Returns the deterministic immutable segment filename.
+ fn sealed_name(from_seq: u64, to_seq: u64) -> String {
+ format!("{SEALED_SEGMENT_FILE_PREFIX}{from_seq:020}-{to_seq:020}{SEGMENT_FILE_EXTENSION}")
+ }
+
+ /// Rebuild missing mutable state from the latest checkpoint and retained segments.
+ async fn recovery_manifest(
+ &self,
+ scope: &ScopeKey,
+ mut manifest: SegmentManifest,
+ ) -> Result<SegmentManifest> {
+ manifest.validate()?;
+ for state in manifest.backend_states.values_mut() {
+ state.synced_seq = 0;
+ }
+ manifest.segments.clear();
+ manifest.next_seq = match self
+ .store
+ .read_json::<CheckpointsManifest>(
+ &self
+ .store
+ .paths()
+ .checkpoints_manifest(&scope.account_id, scope.partition_id)?
+ .0,
+ )
+ .await
+ {
+ Ok(pointer) => {
+ pointer.validate()?;
+ pointer.latest_checkpoint.map_or(1, |latest| {
+ latest.checkpoint_to_seq.checked_add(1).unwrap_or(u64::MAX)
+ })
+ }
+ Err(Error::NotFound(_)) => 1,
+ Err(error) => return Err(error),
+ };
+ let directory = self
+ .store
+ .paths()
+ .segments_dir(&scope.account_id, scope.partition_id)?
+ .0;
+ let mut candidates = match self.store.list_directory(&directory).await {
+ Ok(entries) => entries
+ .into_iter()
+ .filter_map(|entry| {
+ (!entry.is_dir)
+ .then(|| Self::sealed_range(&entry.name).map(|range| (range, entry.name)))
+ .flatten()
+ })
+ .collect::<Vec<_>>(),
+ Err(Error::NotFound(_)) => Vec::new(),
+ Err(error) => return Err(error),
+ };
+ candidates.sort_by_key(|(range, _)| *range);
+ for ((from, to), path) in candidates {
+ if to < manifest.next_seq {
+ continue;
+ }
+ if from != manifest.next_seq {
+ break;
+ }
+ let bytes = self
+ .store
+ .read_bytes(&format!("{directory}/{path}"))
+ .await?;
+ let decoded = decode_segment(&bytes)?;
+ if decoded.truncated_tail
+ || decoded.records.first().map(|record| record.seq) != Some(from)
+ || decoded.records.last().map(|record| record.seq) != Some(to)
+ {
+ return Err(Error::Serialization(
+ "retained cache segment does not match its filename".to_string(),
+ ));
+ }
+ manifest.segments.push(SegmentDescriptor {
+ path,
+ segment_from_seq: from,
+ segment_to_seq: Some(to),
+ record_count: decoded.records.len() as u32,
+ byte_len: bytes.len() as u64,
+ checksum: Some(sha256_hex(&bytes)),
+ });
+ manifest.next_seq = to
+ .checked_add(1)
+ .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?;
+ }
+ manifest.validate()?;
+ Ok(manifest)
+ }
+
+ /// Parse one deterministic sealed segment filename.
+ fn sealed_range(name: &str) -> Option<(u64, u64)> {
+ let range = name
+ .strip_prefix(SEALED_SEGMENT_FILE_PREFIX)?
+ .strip_suffix(SEGMENT_FILE_EXTENSION)?;
+ let (from, to) = range.split_once('-')?;
+ Some((from.parse().ok()?, to.parse().ok()?))
+ }
+
+ /// Encodes account partition identity as script arguments.
+ fn scope_args(scope: &ScopeKey) -> Vec<Bytes> {
+ vec![
+ Bytes::from(scope.account_id.clone()),
+ Bytes::from(scope.partition_id.to_string()),
+ Bytes::from(scope.epoch.to_string()),
+ ]
+ }
+
+ /// Returns the leading UTF-8 status from one script result.
+ fn status<'a>(values: &'a [ScriptValue], operation: &str) -> Result<&'a str> {
+ Self::string_value(values, 0, operation)
+ }
+
+ /// Reads one UTF-8 byte result at the requested index.
+ fn string_value<'a>(values: &'a [ScriptValue], index: usize, field: &str) -> Result<&'a str> {
+ match values.get(index) {
+ Some(ScriptValue::Bytes(value)) => std::str::from_utf8(value).map_err(|error| {
+ Error::Serialization(format!(
+ "Cache-backed multi-write {field} is not UTF-8: {error}"
+ ))
+ }),
+ other => Err(Error::Serialization(format!(
+ "Cache-backed multi-write {field} has invalid type: {other:?}"
+ ))),
+ }
+ }
+
+ /// Reads one non-negative integer script result as u64.
+ fn u64_value(values: &[ScriptValue], index: usize, field: &str) -> Result<u64> {
+ match values.get(index) {
+ Some(ScriptValue::Integer(value)) => u64::try_from(*value).map_err(|_| {
+ Error::Serialization(format!("Cache-backed multi-write {field} is negative"))
+ }),
+ Some(ScriptValue::Bytes(value)) => std::str::from_utf8(value)
+ .ok()
+ .and_then(|value| value.parse().ok())
+ .ok_or_else(|| {
+ Error::Serialization(format!(
+ "Cache-backed multi-write {field} is not an integer"
+ ))
+ }),
+ other => Err(Error::Serialization(format!(
+ "Cache-backed multi-write {field} has invalid type: {other:?}"
+ ))),
+ }
+ }
+
+ /// Reads one script integer as u32.
+ fn u32_value(values: &[ScriptValue], index: usize, field: &str) -> Result<u32> {
+ u32::try_from(Self::u64_value(values, index, field)?).map_err(|_| {
+ Error::Serialization(format!("Cache-backed multi-write {field} exceeds u32"))
+ })
+ }
+
+ /// Reads one script integer as usize.
+ fn usize_value(values: &[ScriptValue], index: usize, field: &str) -> Result<usize> {
+ usize::try_from(Self::u64_value(values, index, field)?).map_err(|_| {
+ Error::Serialization(format!("Cache-backed multi-write {field} exceeds usize"))
+ })
+ }
+
+ /// Deserializes one JSON byte result.
+ fn json_value<T>(values: &[ScriptValue], index: usize, field: &str) -> Result<T>
+ where
+ T: for<'de> Deserialize<'de>,
+ {
+ match values.get(index) {
+ Some(ScriptValue::Bytes(value)) => serde_json::from_slice(value)
+ .map_err(|error| Error::Serialization(format!("{field}: {error}"))),
+ other => Err(Error::Serialization(format!(
+ "Cache-backed multi-write {field} has invalid type: {other:?}"
+ ))),
+ }
+ }
+
+ /// Builds a validated operation error from a script response.
+ fn script_invalid(values: &[ScriptValue], operation: &str) -> Error {
+ match Self::string_value(values, 1, operation) {
+ Ok(message) => Error::invalid_operation(message),
+ Err(error) => error,
+ }
+ }
+
+ /// Builds an error for an unknown script status.
+ fn unknown_status(operation: &str, status: &str) -> Error {
+ Error::Serialization(format!(
+ "Cache-backed multi-write {operation} returned unknown status: {status}"
+ ))
+ }
+
+ /// Maps cache runtime failures into existing RAGFS error categories.
+ fn cache_error(operation: &str, error: CacheError) -> Error {
+ match error {
+ CacheError::Timeout(message) => {
+ Error::Timeout(format!("Cache-backed multi-write {operation}: {message}"))
+ }
+ CacheError::Unavailable(message) => {
+ Error::Network(format!("Cache-backed multi-write {operation}: {message}"))
+ }
+ CacheError::Closed => Error::Network(format!(
+ "Cache-backed multi-write {operation}: runtime is closed"
+ )),
+ CacheError::InvalidData(message) => {
+ Error::Serialization(format!("Cache-backed multi-write {operation}: {message}"))
+ }
+ other => Error::internal(format!("Cache-backed multi-write {operation}: {other}")),
+ }
+ }
+}
+
+#[async_trait]
+impl MultiWriteProvider for CacheProvider {
+ /// Atomically installs reconstructed state only when cache state is absent.
+ async fn bootstrap_manifest(&self, scope: &ScopeKey, manifest: SegmentManifest) -> Result<()> {
+ self.validate_scope(scope).await?;
+ match self.read_manifest(scope).await {
+ Ok(_) => return Ok(()),
+ Err(Error::NotFound(_)) => {}
+ Err(error) => return Err(error),
+ }
+ let manifest = self.recovery_manifest(scope, manifest).await?;
+ let keys = self.keys_for_scope(scope)?;
+ let mut args = vec![Bytes::from_static(b"bootstrap")];
+ args.extend(Self::scope_args(scope));
+ args.push(Bytes::from(manifest.version.to_string()));
+ args.push(Bytes::from(manifest.next_seq.to_string()));
+ args.push(Bytes::from(serde_json::to_vec(&manifest.segments)?));
+ args.push(Bytes::from(serde_json::to_vec(&manifest.backend_states)?));
+ let values = self.execute_array(SEAL_SCRIPT_ID, &keys, args).await?;
+ match Self::status(&values, "bootstrap")? {
+ "ok" | "exists" => self.read_manifest(scope).await.map(|_| ()),
+ "invalid" => Err(Self::script_invalid(&values, "bootstrap")),
+ other => Err(Self::unknown_status("bootstrap", other)),
+ }
+ }
+
+ /// Flushes normal events through bounded atomic cache appends.
+ async fn flush(
+ &self,
+ _route_hint: Option<&PartitionContext>,
+ events: Vec<PendingEvent>,
+ ) -> Result<FlushResult> {
+ Self::validate_flush_events(&events)?;
+ let records = events
+ .iter()
+ .map(Self::record)
+ .collect::<Result<Vec<_>>>()?;
+ let mut offset = 0;
+ let mut route = None;
+ let mut continuing = false;
+ let mut first_seq = None;
+ let mut last_seq = None;
+ let mut sealed_segments = Vec::new();
+ let owner = Uuid::new_v4().to_string();
+ while offset < records.len() {
+ let (current_route, outcome) = self
+ .flush_routed_once(
+ &events,
+ route.as_ref().map(|route: &PartitionContext| &route.scope),
+ &records[offset..],
+ &owner,
+ continuing,
+ )
+ .await?;
+ match outcome {
+ FlushScriptOutcome::SealRequired => {
+ route.get_or_insert(current_route);
+ continuing = true;
+ if let Some(descriptor) = self
+ .seal_head_with_owner(&route.as_ref().unwrap().scope, Some(&owner), true)
+ .await?
+ {
+ if !sealed_segments
+ .iter()
+ .any(|sealed: &SegmentDescriptor| sealed.path == descriptor.path)
+ {
+ sealed_segments.push(descriptor);
+ }
+ }
+ }
+ FlushScriptOutcome::Busy => {
+ tokio::time::sleep(std::time::Duration::from_millis(1)).await;
+ }
+ FlushScriptOutcome::Appended(result) => {
+ if result.appended == 0 {
+ return Err(Error::Serialization(
+ "cache-backed flush appended no records".to_string(),
+ ));
+ }
+ route.get_or_insert(current_route);
+ first_seq.get_or_insert(result.first_seq);
+ last_seq = Some(result.last_seq);
+ offset += result.appended;
+ if result.needs_seal {
+ let keep_owner = offset < records.len();
+ if let Some(descriptor) = self
+ .seal_head_with_owner(
+ &route.as_ref().unwrap().scope,
+ Some(&owner),
+ keep_owner,
+ )
+ .await?
+ {
+ if !sealed_segments
+ .iter()
+ .any(|sealed| sealed.path == descriptor.path)
+ {
+ sealed_segments.push(descriptor);
+ }
+ }
+ continuing = keep_owner;
+ } else if offset < records.len() {
+ return Err(Error::Serialization(
+ "cache-backed flush released its owner before completing the batch"
+ .to_string(),
+ ));
+ }
+ }
+ }
+ }
+ Ok(FlushResult {
+ scope: route.unwrap().scope,
+ first_seq: first_seq.unwrap(),
+ last_seq: last_seq.unwrap(),
+ sealed_segments,
+ })
+ }
+
+ /// Reads and validates committed cache head records in one half-open range.
+ async fn read_committed_head(
+ &self,
+ scope: &ScopeKey,
+ from_seq_inclusive: u64,
+ to_seq_exclusive: u64,
+ ) -> Result<Vec<SegmentRecord>> {
+ let (state, records, _) = self.snapshot(scope).await?;
+ if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > state.next_seq {
+ return Err(Error::invalid_operation(
+ "committed head range is outside the manifest",
+ ));
+ }
+ let head_from = records
+ .first()
+ .map(|record| record.seq)
+ .unwrap_or(state.next_seq);
+ if from_seq_inclusive < head_from {
+ return Err(Error::invalid_operation(
+ "committed head range starts before the current cache head",
+ ));
+ }
+ let expected_len = usize::try_from(to_seq_exclusive - from_seq_inclusive)
+ .map_err(|_| Error::invalid_operation("committed head range is too large"))?;
+ let selected = records
+ .into_iter()
+ .filter(|record| record.seq >= from_seq_inclusive && record.seq < to_seq_exclusive)
+ .collect::<Vec<_>>();
+ if selected.len() != expected_len
+ || selected.iter().enumerate().any(|(index, record)| {
+ record.seq
+ != from_seq_inclusive
+ .checked_add(index as u64)
+ .unwrap_or(u64::MAX)
+ })
+ {
+ return Err(Error::Serialization(
+ "cache committed head range is incomplete".to_string(),
+ ));
+ }
+ Ok(selected)
+ }
+
+ /// Reads one complete committed range from sealed storage and the cache head.
+ async fn read_committed_range(
+ &self,
+ scope: &ScopeKey,
+ from_seq_inclusive: u64,
+ to_seq_exclusive: u64,
+ ) -> Result<Vec<SegmentRecord>> {
+ let (state, head, _) = self.snapshot(scope).await?;
+ if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > state.next_seq {
+ return Err(Error::invalid_operation(
+ "committed range is outside the manifest",
+ ));
+ }
+ let mut records = Vec::new();
+ for descriptor in &state.segments {
+ if descriptor.segment_from_seq >= to_seq_exclusive
+ || descriptor
+ .segment_to_seq
+ .is_some_and(|to| to < from_seq_inclusive)
+ {
+ continue;
+ }
+ records.extend(read_sealed_segment(&self.store, scope, descriptor).await?);
+ }
+ records.extend(head);
+ select_record_range(records, from_seq_inclusive, to_seq_exclusive)
+ }
+
+ /// Reads one immutable sealed segment named by a validated descriptor.
+ async fn read_sealed_segment(
+ &self,
+ scope: &ScopeKey,
+ descriptor: &SegmentDescriptor,
+ ) -> Result<Vec<SegmentRecord>> {
+ read_sealed_segment(&self.store, scope, descriptor).await
+ }
+
+ /// Returns validated sealed state plus a derived mutable-head descriptor.
+ async fn read_manifest(&self, scope: &ScopeKey) -> Result<SegmentManifest> {
+ let (state, records, _) = self.snapshot(scope).await?;
+ Self::manifest(state, &records)
+ }
+
+ /// Appends one caller-scoped marker and seals it before returning.
+ async fn append_marker(
+ &self,
+ scope: &ScopeKey,
+ event: &PendingEvent,
+ ) -> Result<MarkerPosition> {
+ self.validate_scope(scope).await?;
+ self.validate_marker(scope, event)?;
+ let record = Self::record(event)?;
+ let owner = Uuid::new_v4().to_string();
+ let mut continuing = false;
+ loop {
+ match self
+ .flush_marker_once(scope, &record, &owner, continuing)
+ .await?
+ {
+ FlushScriptOutcome::SealRequired => {
+ continuing = true;
+ self.seal_head_with_owner(scope, Some(&owner), true).await?;
+ }
+ FlushScriptOutcome::Busy => {
+ tokio::time::sleep(std::time::Duration::from_millis(1)).await;
+ }
+ FlushScriptOutcome::Appended(result) => {
+ if result.appended != 1 || !result.needs_seal {
+ return Err(Error::Serialization(
+ "cache marker append returned an invalid result".to_string(),
+ ));
+ }
+ self.seal_head_with_owner(scope, Some(&owner), false)
+ .await?;
+ if !self.marker_is_sealed(scope, result.first_seq).await? {
+ return Err(Error::Serialization(
+ "cache marker is not referenced by a sealed segment".to_string(),
+ ));
+ }
+ return Ok(MarkerPosition {
+ partition_id: scope.partition_id,
+ epoch: scope.epoch,
+ seq: result.first_seq,
+ });
+ }
+ }
+ }
+ }
+
+ /// Advances one backend's monotonic progress atomically through CacheRuntime.
+ async fn advance_backend_state(
+ &self,
+ scope: &ScopeKey,
+ backend_id: &str,
+ synced_seq: u64,
+ ) -> Result<()> {
+ self.validate_scope(scope).await?;
+ if backend_id.trim().is_empty() {
+ return Err(Error::invalid_operation("backend id must not be empty"));
+ }
+ let keys = self.keys_for_scope(scope)?;
+ let mut args = Self::scope_args(scope);
+ args.push(Bytes::from(backend_id.to_string()));
+ args.push(Bytes::from(synced_seq.to_string()));
+ let values = self.execute_array(ADVANCE_SCRIPT_ID, &keys, args).await?;
+ match Self::status(&values, "advance backend")? {
+ "ok" => Ok(()),
+ "not_found" => Err(Error::not_found(keys.state)),
+ "scope_mismatch" => Err(Error::invalid_operation(
+ "Cache-backed multi-write state scope does not match",
+ )),
+ "invalid" => Err(Self::script_invalid(&values, "advance backend")),
+ other => Err(Self::unknown_status("advance backend", other)),
+ }
+ }
+
+ /// Atomically validates and removes exactly the requested sealed descriptors.
+ async fn prune_sealed(
+ &self,
+ scope: &ScopeKey,
+ removable_paths: &[String],
+ ) -> Result<Vec<SegmentDescriptor>> {
+ let path = self.store.paths().account_manifest(&scope.account_id)?.0;
+ let lease = self
+ .store
+ .pathlock_manager()
+ .acquire_exact(
+ &path,
+ self.store
+ .pathlock_manager()
+ .default_lock_timeout()
+ .max(std::time::Duration::from_secs(30)),
+ None,
+ )
+ .await?;
+ let operation = async {
+ if !self.store.gc_scope_is_safe(scope).await? {
+ return Ok(Vec::new());
+ }
+ let keys = self.keys_for_scope(scope)?;
+ let mut args = vec![Bytes::from_static(b"prune")];
+ args.extend(Self::scope_args(scope));
+ args.push(Bytes::from(serde_json::to_vec(removable_paths)?));
+ let values = self.execute_array(SEAL_SCRIPT_ID, &keys, args).await?;
+ match Self::status(&values, "prune")? {
+ "ok" => Self::json_value(&values, 1, "pruned descriptors"),
+ "not_found" => Err(Error::not_found(keys.state)),
+ "scope_mismatch" => Err(Error::invalid_operation(
+ "Cache-backed multi-write state scope does not match",
+ )),
+ "invalid" => Err(Self::script_invalid(&values, "prune")),
+ other => Err(Self::unknown_status("prune", other)),
+ }
+ }
+ .await;
+ let release = self
+ .store
+ .pathlock_manager()
+ .release(&lease)
+ .await
+ .map_err(Error::from);
+ match operation {
+ Ok(result) => release.map(|()| result),
+ Err(error) => Err(error),
+ }
+ }
+}
diff --git a/crates/ragfs/src/multibackend/provider/filesystem.rs b/crates/ragfs/src/multibackend/provider/filesystem.rs
new file mode 100644
index 000000000..89840e632
--- /dev/null
+++ b/crates/ragfs/src/multibackend/provider/filesystem.rs
@@ -0,0 +1,767 @@
+//! Filesystem-backed V2 multi-write persistence.
+
+use std::collections::{BTreeSet, HashSet};
+use std::sync::Arc;
+
+use async_trait::async_trait;
+
+use crate::core::errors::{Error, Result};
+use crate::core::internal_names::is_multiwrite_internal_path;
+use crate::lock::{OwnedPathLockLease, PathLockKind, PathLockRequest};
+use crate::multibackend::codec::{decode_segment, encode_segment, sha256_hex};
+use crate::multibackend::constants::{
+ HEAD_SEGMENT_FILE_PREFIX, MAX_SEGMENT_RECORDS, SEALED_SEGMENT_FILE_PREFIX,
+ SEGMENT_FILE_EXTENSION,
+};
+use crate::multibackend::meta::MetadataStore;
+use crate::multibackend::model::{
+ FlushResult, MarkerPosition, PartitionContext, PartitionsManifest, PendingEvent,
+ PendingEventKind, ScopeKey, SegmentDescriptor, SegmentEventType, SegmentManifest,
+ SegmentRecord,
+};
+use crate::multibackend::router::AccountRouter;
+
+use super::{read_sealed_segment, select_record_range, MultiWriteProvider};
+
+/// Persists V2 manifests and segment blobs on the primary filesystem.
+pub struct FilesystemProvider {
+ store: Arc<MetadataStore>,
+ router: AccountRouter,
+}
+
+impl FilesystemProvider {
+ /// Creates a filesystem-backed metadata provider.
+ pub fn new(store: Arc<MetadataStore>) -> Self {
+ Self {
+ router: AccountRouter::new(store.clone()),
+ store,
+ }
+ }
+
+ /// Validates normal flush input before any route or storage access.
+ fn validate_flush_events(events: &[PendingEvent]) -> Result<()> {
+ let first = events
+ .first()
+ .ok_or_else(|| Error::invalid_operation("flush requires at least one event"))?;
+ for event in events {
+ if event.account_id != first.account_id {
+ return Err(Error::invalid_operation(
+ "flush events must belong to one account",
+ ));
+ }
+ if is_multiwrite_internal_path(&event.path) {
+ return Err(Error::invalid_path(format!(
+ "cannot flush internal path: {}",
+ event.path
+ )));
+ }
+ match event.kind {
+ PendingEventKind::Data(SegmentEventType::Write | SegmentEventType::Remove) => {}
+ _ => {
+ return Err(Error::invalid_operation(
+ "normal flush accepts only write and remove events",
+ ))
+ }
+ }
+ if event.destination_path.is_some() {
+ return Err(Error::invalid_operation(
+ "normal data event cannot have a destination",
+ ));
+ }
+ }
+ Ok(())
+ }
+
+ /// Routes every event and requires one current partition scope.
+ async fn route_events(&self, events: &[PendingEvent]) -> Result<PartitionContext> {
+ let mut selected = None;
+ for event in events {
+ let current = self.router.route(&event.account_id, &event.path).await?;
+ if selected
+ .as_ref()
+ .is_some_and(|existing: &PartitionContext| existing.scope != current.scope)
+ {
+ return Err(Error::invalid_operation(
+ "flush events route to different partition scopes",
+ ));
+ }
+ selected.get_or_insert(current);
+ }
+ selected.ok_or_else(|| Error::invalid_operation("flush requires at least one event"))
+ }
+
+ /// Reads and validates the partitions manifest owning one scope.
+ async fn validate_scope(&self, scope: &ScopeKey) -> Result<()> {
+ let path = self.store.paths().account_manifest(&scope.account_id)?.0;
+ let manifest: PartitionsManifest = self.store.read_json(&path).await?;
+ manifest.validate()?;
+ scope.validate_against_manifest(&manifest)
+ }
+
+ /// Reads and validates one segment manifest without acquiring locks.
+ async fn load_manifest(&self, scope: &ScopeKey) -> Result<SegmentManifest> {
+ self.validate_scope(scope).await?;
+ let path = self
+ .store
+ .paths()
+ .segment_manifest(&scope.account_id, scope.partition_id)?
+ .0;
+ let manifest: SegmentManifest = self.store.read_json(&path).await?;
+ manifest.validate()?;
+ Ok(manifest)
+ }
+
+ /// Returns the current mutable head descriptor.
+ fn head(manifest: &SegmentManifest) -> Option<&SegmentDescriptor> {
+ manifest
+ .segments
+ .last()
+ .filter(|segment| segment.segment_to_seq.is_none())
+ }
+
+ /// Returns the deterministic mutable head filename.
+ fn head_name(from_seq: u64) -> String {
+ format!("{HEAD_SEGMENT_FILE_PREFIX}{from_seq:020}{SEGMENT_FILE_EXTENSION}")
+ }
+
+ /// Returns the deterministic immutable sealed filename.
+ fn sealed_name(from_seq: u64, to_seq: u64) -> String {
+ format!("{SEALED_SEGMENT_FILE_PREFIX}{from_seq:020}-{to_seq:020}{SEGMENT_FILE_EXTENSION}")
+ }
+
+ /// Returns one logical segment blob path.
+ fn segment_path(&self, scope: &ScopeKey, name: &str) -> Result<String> {
+ let directory = self
+ .store
+ .paths()
+ .segments_dir(&scope.account_id, scope.partition_id)?
+ .0;
+ Ok(format!("{directory}/{name}"))
+ }
+
+ /// Builds the complete Exact-lock set for one append candidate.
+ fn append_lock_requests(
+ &self,
+ scope: &ScopeKey,
+ manifest: &SegmentManifest,
+ event_count: usize,
+ force_final_seal: bool,
+ ) -> Result<Vec<PathLockRequest>> {
+ let mut paths = BTreeSet::from([self
+ .store
+ .paths()
+ .segment_manifest(&scope.account_id, scope.partition_id)?
+ .0]);
+ paths.insert(self.store.paths().account_manifest(&scope.account_id)?.0);
+ let mut remaining = event_count;
+ let mut next_seq = manifest.next_seq;
+ if next_seq.checked_add(event_count as u64).is_none() {
+ return Err(Error::invalid_operation("multi-write sequence overflow"));
+ }
+ let mut count = 0;
+ let mut segment_from_seq = next_seq;
+ if let Some(head) = Self::head(manifest) {
+ paths.insert(self.segment_path(scope, &head.path)?);
+ count = head.record_count as usize;
+ segment_from_seq = head.segment_from_seq;
+ }
+
+ if count == MAX_SEGMENT_RECORDS {
+ paths.insert(
+ self.segment_path(scope, &Self::sealed_name(segment_from_seq, next_seq - 1))?,
+ );
+ count = 0;
+ segment_from_seq = next_seq;
+ }
+ while remaining > 0 {
+ let appended = remaining.min(MAX_SEGMENT_RECORDS - count);
+ remaining -= appended;
+ next_seq = next_seq
+ .checked_add(appended as u64)
+ .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?;
+ count += appended;
+ if count == MAX_SEGMENT_RECORDS || (force_final_seal && remaining == 0) {
+ paths.insert(
+ self.segment_path(scope, &Self::sealed_name(segment_from_seq, next_seq - 1))?,
+ );
+ count = 0;
+ segment_from_seq = next_seq;
+ }
+ }
+ if count > 0 {
+ let name = if force_final_seal {
+ Self::sealed_name(segment_from_seq, next_seq - 1)
+ } else {
+ Self::head_name(segment_from_seq)
+ };
+ paths.insert(self.segment_path(scope, &name)?);
+ }
+ Ok(paths
+ .into_iter()
+ .map(|path| PathLockRequest {
+ path,
+ kind: PathLockKind::Exact,
+ })
+ .collect())
+ }
+
+ /// Builds the Exact-lock set for a manifest-only state mutation.
+ fn manifest_lock_requests(&self, scope: &ScopeKey) -> Result<Vec<PathLockRequest>> {
+ let mut paths = vec![PathLockRequest {
+ path: self
+ .store
+ .paths()
+ .segment_manifest(&scope.account_id, scope.partition_id)?
+ .0,
+ kind: PathLockKind::Exact,
+ }];
+ paths.push(PathLockRequest {
+ path: self.store.paths().account_manifest(&scope.account_id)?.0,
+ kind: PathLockKind::Exact,
+ });
+ Ok(paths)
+ }
+
+ /// Acquires one all-or-nothing Exact-lock batch.
+ async fn acquire(&self, requests: &[PathLockRequest]) -> Result<OwnedPathLockLease> {
+ let manager = self.store.pathlock_manager();
+ Ok(manager
+ .acquire_batch(requests, std::time::Duration::from_secs(5), None)
+ .await?)
+ }
+
+ /// Releases a lock lease while preserving any operation failure.
+ async fn release<T>(&self, lease: &OwnedPathLockLease, operation: Result<T>) -> Result<T> {
+ let release = self
+ .store
+ .pathlock_manager()
+ .release(lease)
+ .await
+ .map_err(Error::from);
+ merge_operation_and_release(operation, release)
+ }
+
+ /// Reads and validates only the committed prefix of the current head.
+ async fn committed_head(
+ &self,
+ scope: &ScopeKey,
+ manifest: &SegmentManifest,
+ ) -> Result<Vec<SegmentRecord>> {
+ let Some(head) = Self::head(manifest) else {
+ return Ok(Vec::new());
+ };
+ if head.path != Self::head_name(head.segment_from_seq) {
+ return Err(Error::Serialization(
+ "head segment path is not deterministic".to_string(),
+ ));
+ }
+ let bytes = self
+ .store
+ .read_bytes(&self.segment_path(scope, &head.path)?)
+ .await?;
+ let committed_len = usize::try_from(head.byte_len)
+ .map_err(|_| Error::Serialization("head byte length is too large".to_string()))?;
+ if bytes.len() < committed_len {
+ return Err(Error::Serialization(
+ "head segment is shorter than its committed byte length".to_string(),
+ ));
+ }
+ let decoded = decode_segment(&bytes[..committed_len])?;
+ if decoded.truncated_tail
+ || decoded.valid_bytes != head.byte_len
+ || decoded.records.len() != head.record_count as usize
+ {
+ return Err(Error::Serialization(
+ "head segment committed prefix does not match its descriptor".to_string(),
+ ));
+ }
+ for (index, record) in decoded.records.iter().enumerate() {
+ let expected = head
+ .segment_from_seq
+ .checked_add(index as u64)
+ .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?;
+ if record.seq != expected {
+ return Err(Error::Serialization(
+ "head segment contains a sequence gap".to_string(),
+ ));
+ }
+ }
+ Ok(decoded.records)
+ }
+
+ /// Converts one pending event into its persisted segment record.
+ fn record(event: &PendingEvent, seq: u64) -> Result<SegmentRecord> {
+ let event_type = match event.kind {
+ PendingEventKind::Data(kind) => kind,
+ PendingEventKind::DeleteAccount => {
+ return Err(Error::invalid_operation(
+ "account deletion is not a segment record",
+ ))
+ }
+ };
+ let marker = matches!(
+ event_type,
+ SegmentEventType::RemoveTree
+ | SegmentEventType::MoveTree
+ | SegmentEventType::DeleteScope
+ );
+ let record = SegmentRecord {
+ seq,
+ event_type,
+ path: event.path.clone(),
+ op_id: marker.then_some(event.operation_id),
+ destination_path: event.destination_path.clone(),
+ };
+ record.validate()?;
+ Ok(record)
+ }
+
+ /// Persists one immutable sealed segment and returns its descriptor.
+ async fn persist_sealed(
+ &self,
+ scope: &ScopeKey,
+ manifest: &SegmentManifest,
+ records: &[SegmentRecord],
+ ) -> Result<SegmentDescriptor> {
+ let first = records
+ .first()
+ .ok_or_else(|| Error::invalid_operation("cannot seal an empty segment"))?
+ .seq;
+ let last = records.last().unwrap().seq;
+ let bytes = encode_segment(records)?;
+ let name = Self::sealed_name(first, last);
+ let path = self.segment_path(scope, &name)?;
+ if !manifest.segments.iter().any(|segment| segment.path == name) {
+ self.store.remove_file(&path).await?;
+ }
+ self.store.write_immutable(&path, &bytes).await?;
+ Ok(SegmentDescriptor {
+ path: name,
+ segment_from_seq: first,
+ segment_to_seq: Some(last),
+ record_count: records.len() as u32,
+ byte_len: bytes.len() as u64,
+ checksum: Some(sha256_hex(&bytes)),
+ })
+ }
+
+ /// Publishes one mutable head and returns its descriptor.
+ async fn persist_head(
+ &self,
+ scope: &ScopeKey,
+ records: &[SegmentRecord],
+ ) -> Result<SegmentDescriptor> {
+ let first = records
+ .first()
+ .ok_or_else(|| Error::invalid_operation("cannot publish an empty head"))?
+ .seq;
+ let bytes = encode_segment(records)?;
+ let name = Self::head_name(first);
+ self.store
+ .write_bytes(&self.segment_path(scope, &name)?, &bytes)
+ .await?;
+ Ok(SegmentDescriptor {
+ path: name,
+ segment_from_seq: first,
+ segment_to_seq: None,
+ record_count: records.len() as u32,
+ byte_len: bytes.len() as u64,
+ checksum: None,
+ })
+ }
+
+ /// Appends records to a locked manifest and publishes blobs before the manifest.
+ async fn commit_events(
+ &self,
+ scope: &ScopeKey,
+ mut manifest: SegmentManifest,
+ events: &[PendingEvent],
+ force_final_seal: bool,
+ ) -> Result<(SegmentManifest, Vec<SegmentDescriptor>)> {
+ let mut records = self.committed_head(scope, &manifest).await?;
+ let mut sealed = Vec::new();
+ if records.len() == MAX_SEGMENT_RECORDS {
+ manifest.segments.pop();
+ let descriptor = self.persist_sealed(scope, &manifest, &records).await?;
+ manifest.segments.push(descriptor.clone());
+ sealed.push(descriptor);
+ records.clear();
+ }
+
+ for (index, event) in events.iter().enumerate() {
+ let seq = manifest.next_seq;
+ manifest.next_seq = manifest
+ .next_seq
+ .checked_add(1)
+ .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?;
+ records.push(Self::record(event, seq)?);
+ let seal_now = records.len() == MAX_SEGMENT_RECORDS
+ || (force_final_seal && index + 1 == events.len());
+ if seal_now {
+ if Self::head(&manifest).is_some() {
+ manifest.segments.pop();
+ }
+ let descriptor = self.persist_sealed(scope, &manifest, &records).await?;
+ manifest.segments.push(descriptor.clone());
+ sealed.push(descriptor);
+ records.clear();
+ }
+ }
+
+ if !records.is_empty() {
+ let descriptor = self.persist_head(scope, &records).await?;
+ if Self::head(&manifest).is_some() {
+ manifest.segments.pop();
+ }
+ manifest.segments.push(descriptor);
+ }
+ manifest.validate()?;
+ let path = self
+ .store
+ .paths()
+ .segment_manifest(&scope.account_id, scope.partition_id)?
+ .0;
+ self.store
+ .publish_json(&path, &manifest, SegmentManifest::validate)
+ .await?;
+ Ok((manifest, sealed))
+ }
+
+ /// Returns whether the locked append snapshot still matches its candidate.
+ fn append_snapshot_matches(
+ candidate_scope: &ScopeKey,
+ current_scope: &ScopeKey,
+ candidate_manifest: &SegmentManifest,
+ current_manifest: &SegmentManifest,
+ candidate_requests: &[PathLockRequest],
+ current_requests: &[PathLockRequest],
+ ) -> bool {
+ candidate_scope == current_scope
+ && candidate_manifest == current_manifest
+ && candidate_requests == current_requests
+ }
+
+ /// Validates one marker against its caller-selected scope.
+ fn validate_marker(&self, scope: &ScopeKey, event: &PendingEvent) -> Result<()> {
+ if event.account_id != scope.account_id {
+ return Err(Error::invalid_operation(
+ "marker account does not match supplied scope",
+ ));
+ }
+ match event.kind {
+ PendingEventKind::Data(
+ SegmentEventType::RemoveTree
+ | SegmentEventType::MoveTree
+ | SegmentEventType::DeleteScope,
+ ) => {}
+ _ => {
+ return Err(Error::invalid_operation(
+ "append_marker accepts only directory markers",
+ ))
+ }
+ }
+ if is_multiwrite_internal_path(&event.path) {
+ return Err(Error::invalid_path(format!(
+ "cannot append marker for internal path: {}",
+ event.path
+ )));
+ }
+ if !matches!(
+ event.kind,
+ PendingEventKind::Data(SegmentEventType::DeleteScope)
+ ) {
+ self.store
+ .paths()
+ .backend_path(&scope.account_id, &event.path)?;
+ }
+ if let Some(destination) = &event.destination_path {
+ if is_multiwrite_internal_path(destination) {
+ return Err(Error::invalid_path(format!(
+ "cannot append marker for internal destination: {destination}"
+ )));
+ }
+ self.store
+ .paths()
+ .backend_path(&scope.account_id, destination)?;
+ }
+ Self::record(event, 1).map(|_| ())
+ }
+}
+
+#[async_trait]
+impl MultiWriteProvider for FilesystemProvider {
+ /// Confirms that filesystem state exists for the account partition.
+ async fn bootstrap_manifest(&self, scope: &ScopeKey, manifest: SegmentManifest) -> Result<()> {
+ manifest.validate()?;
+ self.load_manifest(scope).await.map(|_| ())
+ }
+
+ /// Flushes normal events after validating their current partition route.
+ async fn flush(
+ &self,
+ _route_hint: Option<&PartitionContext>,
+ events: Vec<PendingEvent>,
+ ) -> Result<FlushResult> {
+ Self::validate_flush_events(&events)?;
+ loop {
+ let candidate = self.route_events(&events).await?;
+ let candidate_manifest = self.load_manifest(&candidate.scope).await?;
+ let requests = self.append_lock_requests(
+ &candidate.scope,
+ &candidate_manifest,
+ events.len(),
+ false,
+ )?;
+ let lease = self.acquire(&requests).await?;
+ let operation = async {
+ let current = self.route_events(&events).await?;
+ let manifest = self.load_manifest(&current.scope).await?;
+ let current_requests =
+ self.append_lock_requests(&current.scope, &manifest, events.len(), false)?;
+ if !Self::append_snapshot_matches(
+ &candidate.scope,
+ &current.scope,
+ &candidate_manifest,
+ &manifest,
+ &requests,
+ &current_requests,
+ ) {
+ return Ok(None);
+ }
+ let first_seq = manifest.next_seq;
+ let (_, sealed_segments) = self
+ .commit_events(&current.scope, manifest, &events, false)
+ .await?;
+ let last_seq = first_seq
+ .checked_add(events.len() as u64 - 1)
+ .ok_or_else(|| Error::Serialization("segment sequence overflow".to_string()))?;
+ Ok(Some(FlushResult {
+ scope: current.scope,
+ first_seq,
+ last_seq,
+ sealed_segments,
+ }))
+ }
+ .await;
+ if let Some(result) = self.release(&lease, operation).await? {
+ return Ok(result);
+ }
+ }
+ }
+
+ /// Reads committed head records in the requested half-open sequence range.
+ async fn read_committed_head(
+ &self,
+ scope: &ScopeKey,
+ from_seq_inclusive: u64,
+ to_seq_exclusive: u64,
+ ) -> Result<Vec<SegmentRecord>> {
+ let manifest = self.load_manifest(scope).await?;
+ if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > manifest.next_seq {
+ return Err(Error::invalid_operation(
+ "committed head range is outside the manifest",
+ ));
+ }
+ let records = self.committed_head(scope, &manifest).await?;
+ Ok(records
+ .into_iter()
+ .filter(|record| record.seq >= from_seq_inclusive && record.seq < to_seq_exclusive)
+ .collect())
+ }
+
+ /// Reads one complete committed range from sealed segments and the current head.
+ async fn read_committed_range(
+ &self,
+ scope: &ScopeKey,
+ from_seq_inclusive: u64,
+ to_seq_exclusive: u64,
+ ) -> Result<Vec<SegmentRecord>> {
+ let manifest = self.load_manifest(scope).await?;
+ if from_seq_inclusive > to_seq_exclusive || to_seq_exclusive > manifest.next_seq {
+ return Err(Error::invalid_operation(
+ "committed range is outside the manifest",
+ ));
+ }
+ let mut records = Vec::new();
+ for descriptor in &manifest.segments {
+ if descriptor.segment_from_seq >= to_seq_exclusive
+ || descriptor
+ .segment_to_seq
+ .is_some_and(|to| to < from_seq_inclusive)
+ {
+ continue;
+ }
+ if descriptor.segment_to_seq.is_some() {
+ records.extend(read_sealed_segment(&self.store, scope, descriptor).await?);
+ } else {
+ records.extend(self.committed_head(scope, &manifest).await?);
+ }
+ }
+ select_record_range(records, from_seq_inclusive, to_seq_exclusive)
+ }
+
+ /// Reads one immutable sealed segment named by a validated descriptor.
+ async fn read_sealed_segment(
+ &self,
+ scope: &ScopeKey,
+ descriptor: &SegmentDescriptor,
+ ) -> Result<Vec<SegmentRecord>> {
+ read_sealed_segment(&self.store, scope, descriptor).await
+ }
+
+ /// Returns the current segment state for one scope.
+ async fn read_manifest(&self, scope: &ScopeKey) -> Result<SegmentManifest> {
+ self.load_manifest(scope).await
+ }
+
+ /// Appends and seals one directory marker in the caller-selected scope.
+ async fn append_marker(
+ &self,
+ scope: &ScopeKey,
+ event: &PendingEvent,
+ ) -> Result<MarkerPosition> {
+ self.validate_marker(scope, event)?;
+ loop {
+ let candidate_manifest = self.load_manifest(scope).await?;
+ let requests = self.append_lock_requests(scope, &candidate_manifest, 1, true)?;
+ let lease = self.acquire(&requests).await?;
+ let operation = async {
+ let manifest = self.load_manifest(scope).await?;
+ let current_requests = self.append_lock_requests(scope, &manifest, 1, true)?;
+ if !Self::append_snapshot_matches(
+ scope,
+ scope,
+ &candidate_manifest,
+ &manifest,
+ &requests,
+ &current_requests,
+ ) {
+ return Ok(None);
+ }
+ let seq = manifest.next_seq;
+ self.commit_events(scope, manifest, std::slice::from_ref(event), true)
+ .await?;
+ Ok(Some(MarkerPosition {
+ partition_id: scope.partition_id,
+ epoch: scope.epoch,
+ seq,
+ }))
+ }
+ .await;
+ if let Some(position) = self.release(&lease, operation).await? {
+ return Ok(position);
+ }
+ }
+ }
+
+ /// Advances one backup's progress while its catch-up lease is held.
+ async fn advance_backend_state(
+ &self,
+ scope: &ScopeKey,
+ backend_id: &str,
+ synced_seq: u64,
+ ) -> Result<()> {
+ let requests = self.manifest_lock_requests(scope)?;
+ let lease = self.acquire(&requests).await?;
+ let operation = async {
+ let mut manifest = self.load_manifest(scope).await?;
+ let state = manifest.backend_states.get_mut(backend_id).ok_or_else(|| {
+ Error::invalid_operation(format!("unknown backend id: {backend_id}"))
+ })?;
+ if synced_seq != 0 && synced_seq < state.synced_seq {
+ return Err(Error::invalid_operation("backend progress cannot regress"));
+ }
+ if synced_seq >= manifest.next_seq {
+ return Err(Error::invalid_operation(
+ "backend progress must remain below next_seq",
+ ));
+ }
+ state.synced_seq = synced_seq;
+ let path = self
+ .store
+ .paths()
+ .segment_manifest(&scope.account_id, scope.partition_id)?
+ .0;
+ self.store
+ .publish_json(&path, &manifest, SegmentManifest::validate)
+ .await
+ }
+ .await;
+ self.release(&lease, operation).await
+ }
+
+ /// Atomically removes selected sealed descriptors and returns the removed set.
+ async fn prune_sealed(
+ &self,
+ scope: &ScopeKey,
+ removable_paths: &[String],
+ ) -> Result<Vec<SegmentDescriptor>> {
+ let requested = removable_paths.iter().collect::<HashSet<_>>();
+ if requested.len() != removable_paths.len() {
+ return Err(Error::invalid_operation(
+ "prune paths must not contain duplicates",
+ ));
+ }
+ let requests = self.manifest_lock_requests(scope)?;
+ let lease = self.acquire(&requests).await?;
+ let operation = async {
+ if !self.store.gc_scope_is_safe(scope).await? {
+ return Ok(Vec::new());
+ }
+ let mut manifest = self.load_manifest(scope).await?;
+ for path in removable_paths {
+ let descriptor = manifest
+ .segments
+ .iter()
+ .find(|segment| segment.path == *path)
+ .ok_or_else(|| {
+ Error::invalid_operation(format!(
+ "sealed segment is not referenced: {path}"
+ ))
+ })?;
+ if descriptor.segment_to_seq.is_none() {
+ return Err(Error::invalid_operation(format!(
+ "cannot prune mutable head: {path}"
+ )));
+ }
+ }
+ let mut removed = Vec::new();
+ manifest.segments.retain(|segment| {
+ if requested.contains(&segment.path) {
+ removed.push(segment.clone());
+ false
+ } else {
+ true
+ }
+ });
+ manifest.validate()?;
+ let path = self
+ .store
+ .paths()
+ .segment_manifest(&scope.account_id, scope.partition_id)?
+ .0;
+ self.store
+ .publish_json(&path, &manifest, SegmentManifest::validate)
+ .await?;
+ Ok(removed)
+ }
+ .await;
+ self.release(&lease, operation).await
+ }
+}
+
+/// Preserves an operation error when lock release also fails.
+fn merge_operation_and_release<T>(operation: Result<T>, release: Result<()>) -> Result<T> {
+ match (operation, release) {
+ (Ok(value), Ok(())) => Ok(value),
+ (Ok(_), Err(error)) => Err(error),
+ (Err(error), Ok(())) => Err(error),
+ (Err(error), Err(release_error)) => Err(Error::internal(format!(
+ "{error}; lock release failed: {release_error}"
+ ))),
+ }
+}
+
+#[cfg(test)]
+#[path = "tests/test_filesystem.rs"]
+mod tests;
diff --git a/crates/ragfs/src/multibackend/provider/mod.rs b/crates/ragfs/src/multibackend/provider/mod.rs
new file mode 100644
index 000000000..1f9911728
--- /dev/null
+++ b/crates/ragfs/src/multibackend/provider/mod.rs
@@ -0,0 +1,179 @@
+//! Persistence providers for V2 multi-write state.
+
+use async_trait::async_trait;
+
+use crate::core::errors::{Error, Result};
+use crate::multibackend::codec::{decode_segment, sha256_hex};
+use crate::multibackend::meta::MetadataStore;
+use crate::multibackend::model::{
+ FlushResult, MarkerPosition, PartitionContext, PartitionState, PartitionsManifest,
+ PendingEvent, ScopeKey, SegmentDescriptor, SegmentManifest, SegmentRecord,
+};
+
+mod filesystem;
+#[cfg(feature = "cache")]
+mod cache;
+
+pub use filesystem::FilesystemProvider;
+#[cfg(feature = "cache")]
+pub use cache::CacheProvider;
+
+/// Persists and reads partitioned multi-write segment state.
+#[async_trait]
+pub trait MultiWriteProvider: Send + Sync {
+ /// Creates missing provider state without replacing an existing manifest.
+ async fn bootstrap_manifest(
+ &self,
+ scope: &ScopeKey,
+ manifest: SegmentManifest,
+ ) -> Result<()>;
+
+ /// Flushes normal events after validating their current partition route.
+ async fn flush(
+ &self,
+ route_hint: Option<&PartitionContext>,
+ events: Vec<PendingEvent>,
+ ) -> Result<FlushResult>;
+
+ /// Reads committed head records in the requested half-open sequence range.
+ async fn read_committed_head(
+ &self,
+ scope: &ScopeKey,
+ from_seq_inclusive: u64,
+ to_seq_exclusive: u64,
+ ) -> Result<Vec<SegmentRecord>>;
+
+ /// Reads committed records across sealed segments and the current head.
+ async fn read_committed_range(
+ &self,
+ scope: &ScopeKey,
+ from_seq_inclusive: u64,
+ to_seq_exclusive: u64,
+ ) -> Result<Vec<SegmentRecord>>;
+
+ /// Reads one immutable sealed segment named by a validated descriptor.
+ async fn read_sealed_segment(
+ &self,
+ scope: &ScopeKey,
+ descriptor: &SegmentDescriptor,
+ ) -> Result<Vec<SegmentRecord>>;
+
+ /// Returns the current segment state for one scope.
+ async fn read_manifest(&self, scope: &ScopeKey) -> Result<SegmentManifest>;
+
+ /// Appends and seals one directory marker in the caller-selected scope.
+ async fn append_marker(
+ &self,
+ scope: &ScopeKey,
+ event: &PendingEvent,
+ ) -> Result<MarkerPosition>;
+
+ /// Advances one backup's monotonic progress while its catch-up lease is held.
+ async fn advance_backend_state(
+ &self,
+ scope: &ScopeKey,
+ backend_id: &str,
+ synced_seq: u64,
+ ) -> Result<()>;
+
+ /// Atomically removes selected sealed descriptors and returns the removed set.
+ async fn prune_sealed(
+ &self,
+ scope: &ScopeKey,
+ removable_paths: &[String],
+ ) -> Result<Vec<SegmentDescriptor>>;
+
+}
+
+/// Bootstrap every stable provider partition from its raw-primary manifest.
+pub(crate) async fn bootstrap_account(
+ provider: &dyn MultiWriteProvider,
+ store: &MetadataStore,
+ account_id: &str,
+ partitions: &PartitionsManifest,
+) -> Result<()> {
+ for (&partition_id, partition) in &partitions.partitions {
+ if partition.state != PartitionState::Stable {
+ continue;
+ }
+ let scope = ScopeKey {
+ account_id: account_id.to_string(),
+ partition_id,
+ epoch: partitions.epoch,
+ };
+ let path = store.paths().segment_manifest(account_id, partition_id)?.0;
+ provider.bootstrap_manifest(&scope, store.read_json(&path).await?).await?;
+ }
+ Ok(())
+}
+
+/// Read and validate one immutable segment through the shared metadata store.
+async fn read_sealed_segment(
+ store: &MetadataStore,
+ scope: &ScopeKey,
+ descriptor: &SegmentDescriptor,
+) -> Result<Vec<SegmentRecord>> {
+ let directory = store
+ .paths()
+ .segments_dir(&scope.account_id, scope.partition_id)?
+ .0;
+ let bytes = store
+ .read_bytes(&format!("{directory}/{}", descriptor.path))
+ .await?;
+ if bytes.len() as u64 != descriptor.byte_len
+ || sha256_hex(&bytes) != descriptor.checksum.as_deref().unwrap_or_default()
+ {
+ return Err(Error::Serialization(
+ "sealed segment bytes do not match descriptor".to_string(),
+ ));
+ }
+ let decoded = decode_segment(&bytes)?;
+ if decoded.truncated_tail
+ || decoded.valid_bytes != descriptor.byte_len
+ || decoded.records.len() != descriptor.record_count as usize
+ || decoded.records.iter().enumerate().any(|(index, record)| {
+ record.seq
+ != descriptor
+ .segment_from_seq
+ .checked_add(index as u64)
+ .unwrap_or(u64::MAX)
+ })
+ {
+ return Err(Error::Serialization(
+ "sealed segment records do not match descriptor".to_string(),
+ ));
+ }
+ Ok(decoded.records)
+}
+
+/// Select and validate one complete half-open sequence range.
+fn select_record_range(
+ records: impl IntoIterator<Item = SegmentRecord>,
+ from_seq_inclusive: u64,
+ to_seq_exclusive: u64,
+) -> Result<Vec<SegmentRecord>> {
+ let range_len = to_seq_exclusive
+ .checked_sub(from_seq_inclusive)
+ .ok_or_else(|| Error::invalid_operation("committed range is reversed"))?;
+ let expected_len = usize::try_from(range_len)
+ .map_err(|_| Error::invalid_operation("committed range is too large"))?;
+ let selected = records
+ .into_iter()
+ .filter(|record| {
+ record.seq >= from_seq_inclusive && record.seq < to_seq_exclusive
+ })
+ .collect::<Vec<_>>();
+ if selected.len() != expected_len
+ || selected.iter().enumerate().any(|(index, record)| {
+ record.seq
+ != from_seq_inclusive
+ .checked_add(index as u64)
+ .unwrap_or(u64::MAX)
+ })
+ {
+ return Err(Error::Serialization(
+ "committed record range is incomplete".to_string(),
+ ));
+ }
+ Ok(selected)
+}
diff --git a/crates/ragfs/src/multibackend/provider/tests/test_filesystem.rs b/crates/ragfs/src/multibackend/provider/tests/test_filesystem.rs
new file mode 100644
index 000000000..636eb4381
--- /dev/null
+++ b/crates/ragfs/src/multibackend/provider/tests/test_filesystem.rs
@@ -0,0 +1,72 @@
+use super::*;
+use crate::core::FileSystem;
+use crate::lock::{MemoryPathLockProvider, PathLockConfig, PathLockManager};
+use crate::plugins::memfs::MemFileSystem;
+
+/// Verifies flush publishes a contiguous range that committed reads can recover.
+#[tokio::test]
+async fn flush_publishes_a_complete_committed_range() {
+ let raw_primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let pathlock_manager = Arc::new(PathLockManager::new(
+ raw_primary.clone(),
+ Arc::new(MemoryPathLockProvider::new()),
+ PathLockConfig::default(),
+ ));
+ let store = Arc::new(MetadataStore::new(raw_primary, pathlock_manager, "/local").unwrap());
+ let partitions = store.initialize_account("account", 1, &[]).await.unwrap();
+ let provider = FilesystemProvider::new(store);
+ let scope = ScopeKey {
+ account_id: "account".to_string(),
+ partition_id: 0,
+ epoch: partitions.epoch,
+ };
+
+ let result = provider
+ .flush(
+ None,
+ vec![
+ PendingEvent {
+ operation_id: uuid::Uuid::from_u128(1),
+ account_id: "account".to_string(),
+ kind: PendingEventKind::Data(SegmentEventType::Write),
+ path: "/local/account/first.txt".to_string(),
+ destination_path: None,
+ route_hint: None,
+ },
+ PendingEvent {
+ operation_id: uuid::Uuid::from_u128(2),
+ account_id: "account".to_string(),
+ kind: PendingEventKind::Data(SegmentEventType::Remove),
+ path: "/local/account/second.txt".to_string(),
+ destination_path: None,
+ route_hint: None,
+ },
+ ],
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(result.scope, scope);
+ assert_eq!((result.first_seq, result.last_seq), (1, 2));
+ let records = provider.read_committed_range(&scope, 1, 3).await.unwrap();
+ assert_eq!(
+ records,
+ vec![
+ SegmentRecord {
+ seq: 1,
+ event_type: SegmentEventType::Write,
+ path: "/local/account/first.txt".to_string(),
+ op_id: None,
+ destination_path: None,
+ },
+ SegmentRecord {
+ seq: 2,
+ event_type: SegmentEventType::Remove,
+ path: "/local/account/second.txt".to_string(),
+ op_id: None,
+ destination_path: None,
+ },
+ ]
+ );
+ assert!(provider.read_committed_range(&scope, 2, 4).await.is_err());
+}
diff --git a/crates/ragfs/src/multibackend/router.rs b/crates/ragfs/src/multibackend/router.rs
new file mode 100644
index 000000000..209df71f2
--- /dev/null
+++ b/crates/ragfs/src/multibackend/router.rs
@@ -0,0 +1,274 @@
+//! Logical path mapping and account partition routing.
+
+use std::collections::HashSet;
+use std::sync::Arc;
+
+use xxhash_rust::xxh3::xxh3_64;
+
+use crate::core::errors::{Error, Result};
+use crate::multibackend::constants::{
+ CHECKPOINTS_DIR, MANIFEST_FILE, MULTIWRITE_MOUNT_PREFIX, MULTIWRITE_PROTOCOL_FILE,
+ PARTITIONS_DIR, SEGMENTS_DIR, SYSTEM_DIR, VBUCKETS,
+};
+use crate::multibackend::meta::MetadataStore;
+use crate::multibackend::model::{PartitionContext, PartitionsManifest, RouteEntry, ScopeKey};
+
+/// Maps canonical logical metadata paths to raw backend paths.
+#[derive(Debug, Clone)]
+pub struct MultiWritePaths {
+ mount_prefix: String,
+}
+
+impl MultiWritePaths {
+ /// Create a path mapper for one canonical absolute mount prefix.
+ pub fn new(mount_prefix: &str) -> Result<Self> {
+ validate_canonical_path(mount_prefix)?;
+ if mount_prefix != MULTIWRITE_MOUNT_PREFIX {
+ return Err(Error::invalid_path(format!(
+ "unsupported multi-write mount prefix: {mount_prefix}"
+ )));
+ }
+ Ok(Self {
+ mount_prefix: mount_prefix.to_string(),
+ })
+ }
+
+ /// Return logical and backend paths for the mount protocol manifest.
+ pub fn mount_protocol(&self) -> (String, String) {
+ self.pair(&format!(
+ "{}/{SYSTEM_DIR}/{MULTIWRITE_PROTOCOL_FILE}",
+ self.mount_prefix
+ ))
+ }
+
+ /// Return logical and backend paths for an account partitions directory.
+ pub fn account_partitions(&self, account_id: &str) -> Result<(String, String)> {
+ validate_account(account_id)?;
+ Ok(self.pair(&format!(
+ "{}/{account_id}/{SYSTEM_DIR}/{PARTITIONS_DIR}",
+ self.mount_prefix
+ )))
+ }
+
+ /// Return logical and backend paths for an account partitions manifest.
+ pub fn account_manifest(&self, account_id: &str) -> Result<(String, String)> {
+ let (logical, _) = self.account_partitions(account_id)?;
+ Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}")))
+ }
+
+ /// Return logical and backend paths for one partition manifest.
+ pub fn partition_manifest(
+ &self,
+ account_id: &str,
+ partition_id: u32,
+ ) -> Result<(String, String)> {
+ let (logical, _) = self.account_partitions(account_id)?;
+ Ok(self.pair(&format!("{logical}/{partition_id}/{MANIFEST_FILE}")))
+ }
+
+ /// Return logical and backend paths for one partition segments directory.
+ pub fn segments_dir(&self, account_id: &str, partition_id: u32) -> Result<(String, String)> {
+ let (logical, _) = self.account_partitions(account_id)?;
+ Ok(self.pair(&format!("{logical}/{partition_id}/{SEGMENTS_DIR}")))
+ }
+
+ /// Return logical and backend paths for one segment manifest.
+ pub fn segment_manifest(
+ &self,
+ account_id: &str,
+ partition_id: u32,
+ ) -> Result<(String, String)> {
+ let (logical, _) = self.segments_dir(account_id, partition_id)?;
+ Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}")))
+ }
+
+ /// Return logical and backend paths for one partition checkpoints directory.
+ pub fn checkpoints_dir(&self, account_id: &str, partition_id: u32) -> Result<(String, String)> {
+ let (logical, _) = self.account_partitions(account_id)?;
+ Ok(self.pair(&format!("{logical}/{partition_id}/{CHECKPOINTS_DIR}")))
+ }
+
+ /// Return logical and backend paths for one checkpoints manifest.
+ pub fn checkpoints_manifest(
+ &self,
+ account_id: &str,
+ partition_id: u32,
+ ) -> Result<(String, String)> {
+ let (logical, _) = self.checkpoints_dir(account_id, partition_id)?;
+ Ok(self.pair(&format!("{logical}/{MANIFEST_FILE}")))
+ }
+
+ /// Convert a canonical logical account path to its raw backend path.
+ pub fn backend_path(&self, account_id: &str, logical_path: &str) -> Result<String> {
+ validate_account(account_id)?;
+ let account_root = format!("{}/{account_id}", self.mount_prefix);
+ if logical_path != account_root
+ && !logical_path
+ .strip_prefix(&account_root)
+ .is_some_and(|suffix| suffix.starts_with('/'))
+ {
+ return Err(Error::invalid_path(format!(
+ "logical path does not belong to account '{account_id}': {logical_path}"
+ )));
+ }
+ self.raw_backend_path(logical_path)
+ }
+
+ /// Convert any canonical logical mount path to its raw backend path.
+ pub fn raw_backend_path(&self, logical_path: &str) -> Result<String> {
+ validate_canonical_path(logical_path)?;
+ let suffix = logical_path
+ .strip_prefix(&self.mount_prefix)
+ .filter(|suffix| suffix.starts_with('/'))
+ .ok_or_else(|| {
+ Error::invalid_path(format!(
+ "logical path is outside mount '{}': {logical_path}",
+ self.mount_prefix
+ ))
+ })?;
+ Ok(suffix.to_string())
+ }
+
+ /// Hash a canonical full logical account path into its fixed vbucket.
+ pub fn vbucket(&self, account_id: &str, logical_path: &str) -> Result<u32> {
+ self.backend_path(account_id, logical_path)?;
+ Ok((xxh3_64(logical_path.as_bytes()) & u64::from(VBUCKETS - 1)) as u32)
+ }
+
+ /// Pair one validated logical path with its raw backend form.
+ fn pair(&self, logical_path: &str) -> (String, String) {
+ (
+ logical_path.to_string(),
+ logical_path[self.mount_prefix.len()..].to_string(),
+ )
+ }
+}
+
+/// Routes account paths through the current persisted partitions manifest.
+pub struct AccountRouter {
+ store: Arc<MetadataStore>,
+}
+
+impl AccountRouter {
+ /// Create an account router backed by the supplied metadata store.
+ pub fn new(store: Arc<MetadataStore>) -> Self {
+ Self { store }
+ }
+
+ /// Route one canonical logical account path to its partition context.
+ pub async fn route(&self, account_id: &str, logical_path: &str) -> Result<PartitionContext> {
+ let bucket = self.store.paths().vbucket(account_id, logical_path)?;
+ let manifest_path = self.store.paths().account_manifest(account_id)?.0;
+ let manifest: PartitionsManifest = self.store.read_json(&manifest_path).await?;
+ manifest.validate()?;
+ let index = manifest.routes.partition_point(|route| route.end < bucket);
+ let route = manifest
+ .routes
+ .get(index)
+ .ok_or_else(|| Error::Serialization("vbucket route is missing".to_string()))?;
+ let segment_dir = self
+ .store
+ .paths()
+ .segments_dir(account_id, route.partition)?
+ .1;
+ Ok(PartitionContext {
+ scope: ScopeKey {
+ account_id: account_id.to_string(),
+ partition_id: route.partition,
+ epoch: manifest.epoch,
+ },
+ segment_dir,
+ })
+ }
+}
+
+/// Validate compressed complete routes against the known partition identifiers.
+pub fn validate_routes(
+ routes: &[RouteEntry],
+ partition_ids: impl IntoIterator<Item = u32>,
+) -> Result<()> {
+ let partition_ids = partition_ids.into_iter().collect::<HashSet<_>>();
+ let mut next = 0;
+ let mut previous_owner = None;
+ for route in routes {
+ route.validate()?;
+ if route.start != next {
+ return Err(Error::Serialization(
+ "routes are not contiguous".to_string(),
+ ));
+ }
+ if !partition_ids.contains(&route.partition) {
+ return Err(Error::Serialization("route owner is unknown".to_string()));
+ }
+ if previous_owner == Some(route.partition) {
+ return Err(Error::Serialization(
+ "adjacent routes have the same owner".to_string(),
+ ));
+ }
+ next = route.end + 1;
+ previous_owner = Some(route.partition);
+ }
+ if next != VBUCKETS {
+ return Err(Error::Serialization(
+ "routes do not cover all vbuckets".to_string(),
+ ));
+ }
+ Ok(())
+}
+
+/// Build deterministic balanced routes for the requested initial partition count.
+pub fn build_initial_routes(partition_count: u32) -> Result<Vec<RouteEntry>> {
+ if !(1..=VBUCKETS).contains(&partition_count) {
+ return Err(Error::Serialization(format!(
+ "partition count must be between 1 and {VBUCKETS}"
+ )));
+ }
+ let owners = (0..VBUCKETS)
+ .map(|bucket| { ((u64::from(bucket) * u64::from(partition_count)) / u64::from(VBUCKETS)) as u32})
+ .collect::<Vec<_>>();
+ let routes = compress_owners(&owners);
+ validate_routes(&routes, 0..partition_count)?;
+ Ok(routes)
+}
+
+/// Compress a complete owner table into inclusive route ranges.
+pub(crate) fn compress_owners(owners: &[u32]) -> Vec<RouteEntry> {
+ let mut routes = Vec::new();
+ let mut start = 0;
+ for index in 1..=owners.len() {
+ if index == owners.len() || owners[index] != owners[start] {
+ routes.push(RouteEntry {
+ start: start as u32,
+ end: index as u32 - 1,
+ partition: owners[start],
+ });
+ start = index;
+ }
+ }
+ routes
+}
+
+/// Validate an account identifier as one canonical path component.
+fn validate_account(account_id: &str) -> Result<()> {
+ if account_id.is_empty() || account_id == "." || account_id == ".." || account_id.contains('/')
+ {
+ return Err(Error::invalid_path(format!(
+ "invalid account identifier: {account_id}"
+ )));
+ }
+ Ok(())
+}
+
+/// Validate an absolute path without empty, dot, or trailing components.
+fn validate_canonical_path(path: &str) -> Result<()> {
+ if !path.starts_with('/')
+ || path.len() <= 1
+ || path.ends_with('/')
+ || path.split('/').skip(1).any(|part| {
+ part.is_empty() || part == "." || part == ".."
+ })
+ {
+ return Err(Error::invalid_path(format!("non-canonical path: {path}")));
+ }
+ Ok(())
+}
diff --git a/crates/ragfs/src/multibackend/runtime.rs b/crates/ragfs/src/multibackend/runtime.rs
new file mode 100644
index 000000000..c9f2e16b8
--- /dev/null
+++ b/crates/ragfs/src/multibackend/runtime.rs
@@ -0,0 +1,755 @@
+//! Foreground event queue and metadata flush worker for V2 multi-write.
+
+use std::collections::{HashMap, HashSet};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::{Arc, Mutex as StdMutex};
+use std::time::Duration;
+
+use tokio::sync::{mpsc, watch};
+use tokio::task::JoinHandle;
+use tracing::{debug, error};
+
+use crate::core::context::{FsContextInner, FS_CTX};
+use crate::core::errors::{Error, Result};
+use crate::core::filesystem::FileSystem;
+use crate::multibackend::catch_up::{CatchUpTarget, CatchUpWorker};
+use crate::multibackend::checkpoint::{CheckpointReader, CheckpointWorker};
+use crate::multibackend::codec::{decode_segment, sha256_hex};
+use crate::multibackend::constants::SYSTEM_DIR;
+use crate::multibackend::meta::{MetadataStore, MultiWriteWorker};
+use crate::multibackend::model::{
+ DirectoryEvent, DirectoryOperation, MarkerPosition, PartitionContext, PartitionState,
+ PartitionsManifest, PendingEvent, PendingEventKind, ScopeKey, SegmentEventType,
+};
+use crate::multibackend::provider::{bootstrap_account, MultiWriteProvider};
+use crate::multibackend::router::AccountRouter;
+
+/// Read active account ids from the encrypted or plain primary registry.
+pub(crate) async fn read_active_accounts(primary: &dyn FileSystem) -> Result<HashSet<String>> {
+ let context = Arc::new(
+ FsContextInner::new(SYSTEM_DIR)
+ .with_bypass_cache(true)
+ .with_auto_pathlock_disabled(),
+ );
+ let bytes = match FS_CTX
+ .scope(context, primary.read("/_system/accounts.json", 0, 0))
+ .await
+ {
+ Ok(bytes) => bytes,
+ Err(Error::NotFound(_)) => return Ok(HashSet::new()),
+ Err(error) => return Err(error),
+ };
+ let value: serde_json::Value = serde_json::from_slice(&bytes)?;
+ let accounts = value
+ .get("accounts")
+ .and_then(serde_json::Value::as_object)
+ .ok_or_else(|| Error::Serialization("accounts.json lacks an accounts object".into()))?;
+ Ok(accounts
+ .iter()
+ .filter(|(_, account)| {
+ account
+ .get("deletion")
+ .is_none_or(serde_json::Value::is_null)
+ })
+ .map(|(account_id, _)| account_id.clone())
+ .collect())
+}
+
+/// Owns the non-blocking foreground sender and its background flush worker.
+pub struct MultiWriteRuntime {
+ sender: StdMutex<Option<mpsc::UnboundedSender<PendingEvent>>>,
+ activation: watch::Sender<Option<bool>>,
+ import_worker: StdMutex<Option<JoinHandle<Result<()>>>>,
+ store: Arc<MetadataStore>,
+ catch_up_cancelled: watch::Sender<bool>,
+ catch_up_wake: Arc<tokio::sync::Notify>,
+ running: Arc<AtomicBool>,
+ pending_events: Arc<AtomicUsize>,
+}
+
+impl MultiWriteRuntime {
+ /// Assemble runtime tasks without allowing background work to start.
+ pub(crate) async fn prepare(
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+ initial_partitions: u32,
+ backup_names: Vec<String>,
+ primary: Option<Arc<dyn FileSystem>>,
+ targets: Vec<CatchUpTarget>,
+ checkpoint_interval: Duration,
+ ) -> Self {
+ let (sender, receiver) = mpsc::unbounded_channel();
+ let running = Arc::new(AtomicBool::new(true));
+ let worker_running = running.clone();
+ let pending_events = Arc::new(AtomicUsize::new(0));
+ let worker_pending_events = pending_events.clone();
+ let (activation, _) = watch::channel(None);
+ let (catch_up_cancelled, _) = watch::channel(false);
+ let catch_up_wake = Arc::new(tokio::sync::Notify::new());
+ let flush_primary = primary.clone();
+ let checkpoint_state = primary.as_ref().map(|primary| {
+ Arc::new(CheckpointWorker::new(
+ primary.clone(),
+ store.clone(),
+ provider.clone(),
+ ))
+ });
+ let checkpoint_reader = Arc::new(CheckpointReader::new(store.clone()));
+ if let Some(worker) = checkpoint_state {
+ let cancellation = catch_up_cancelled.subscribe();
+ let mut activated = activation.subscribe();
+ drop(tokio::spawn(async move {
+ if wait_for_start(&mut activated).await {
+ worker.run(checkpoint_interval, cancellation).await;
+ }
+ }));
+ }
+ let catch_up_workers = primary.map_or_else(Vec::new, |primary| {
+ targets
+ .into_iter()
+ .map(|target| {
+ Arc::new(
+ CatchUpWorker::new(
+ target.backend_id,
+ primary.clone(),
+ target.backend,
+ store.clone(),
+ provider.clone(),
+ )
+ .with_checkpoint_consumer(checkpoint_reader.clone())
+ .with_cancellation(catch_up_cancelled.subscribe()),
+ )
+ })
+ .collect::<Vec<_>>()
+ });
+ let worker = FlushWorker::new(
+ store.clone(),
+ provider,
+ initial_partitions,
+ backup_names,
+ flush_primary,
+ );
+ let mut activated = activation.subscribe();
+ drop(tokio::spawn(async move {
+ if wait_for_start(&mut activated).await {
+ worker.recover_directory_events().await;
+ if *activated.borrow() == Some(true) {
+ worker.run(receiver, worker_pending_events).await;
+ }
+ }
+ worker_running.store(false, Ordering::SeqCst);
+ }));
+ for worker in catch_up_workers {
+ let mut cancelled = catch_up_cancelled.subscribe();
+ let wake = catch_up_wake.clone();
+ let mut activated = activation.subscribe();
+ drop(tokio::spawn(async move {
+ if !wait_for_start(&mut activated).await {
+ return;
+ }
+ while !*cancelled.borrow() {
+ if let Err(error) = worker.run_once().await {
+ debug!(error = %error, "multi-write catch-up round failed");
+ }
+ tokio::select! {
+ _ = tokio::time::sleep(Duration::from_secs(300)) => {}
+ _ = wake.notified() => {}
+ changed = cancelled.changed() => {
+ if changed.is_err() || *cancelled.borrow() {
+ break;
+ }
+ }
+ }
+ }
+ }));
+ }
+ Self {
+ sender: StdMutex::new(Some(sender)),
+ activation,
+ import_worker: StdMutex::new(None),
+ store,
+ catch_up_cancelled,
+ catch_up_wake,
+ running,
+ pending_events,
+ }
+ }
+
+ /// Attach the full-data import task to this runtime's lifecycle.
+ pub(crate) fn set_import_worker(
+ &self,
+ worker: impl std::future::Future<Output = Result<()>> + Send + 'static,
+ ) {
+ let mut activated = self.activation.subscribe();
+ let worker = tokio::spawn(async move {
+ if wait_for_start(&mut activated).await {
+ worker.await
+ } else {
+ Ok(())
+ }
+ });
+ *self
+ .import_worker
+ .lock()
+ .unwrap_or_else(|error| error.into_inner()) = Some(worker);
+ }
+
+ /// Release prepared runtime work after the mount becomes visible.
+ pub(crate) fn activate(&self) {
+ self.activation.send_replace(Some(true));
+ }
+
+ /// Submit one event without waiting for metadata work or taking business locks.
+ pub fn submit(&self, event: PendingEvent) {
+ let failed = {
+ let sender = self
+ .sender
+ .lock()
+ .unwrap_or_else(|error| error.into_inner());
+ match sender.as_ref() {
+ Some(sender) => {
+ self.pending_events.fetch_add(1, Ordering::Relaxed);
+ sender.send(event).err().map(|error| {
+ self.pending_events.fetch_sub(1, Ordering::Relaxed);
+ error.0
+ })
+ }
+ None => Some(event),
+ }
+ };
+ if let Some(event) = failed {
+ self.store.record_worker_error(MultiWriteWorker::Flush);
+ error!(
+ operation_id = %event.operation_id,
+ account = %event.account_id,
+ event_kind = ?event.kind,
+ "multi-write event queue is closed"
+ );
+ }
+ self.catch_up_wake.notify_waiters();
+ }
+
+ /// Return events still waiting in the foreground channel.
+ pub(crate) fn pending_event_count(&self) -> usize {
+ self.pending_events.load(Ordering::Relaxed)
+ }
+
+ /// Return whether the flush worker has not exited yet.
+ pub fn is_running(&self) -> bool {
+ self.running.load(Ordering::SeqCst)
+ }
+}
+
+impl Drop for MultiWriteRuntime {
+ /// Signal background tasks when the owning wrapper is dropped.
+ fn drop(&mut self) {
+ self.sender
+ .lock()
+ .unwrap_or_else(|error| error.into_inner())
+ .take();
+ self.activation.send_replace(Some(false));
+ if let Some(worker) = self
+ .import_worker
+ .lock()
+ .unwrap_or_else(|error| error.into_inner())
+ .take()
+ {
+ worker.abort();
+ }
+ self.catch_up_cancelled.send_replace(true);
+ self.catch_up_wake.notify_waiters();
+ }
+}
+
+/// Wait until a prepared runtime gate is started or stopped.
+async fn wait_for_start(gate: &mut watch::Receiver<Option<bool>>) -> bool {
+ gate.wait_for(|state| state.is_some())
+ .await
+ .is_ok_and(|state| *state == Some(true))
+}
+
+#[cfg(test)]
+#[path = "tests/test_runtime.rs"]
+mod tests;
+
+/// Drains queued events and persists normal file events by account and scope.
+pub struct FlushWorker {
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+ primary: Option<Arc<dyn FileSystem>>,
+ router: AccountRouter,
+ initial_partitions: u32,
+ backup_names: Vec<String>,
+}
+
+impl FlushWorker {
+ /// Create a worker using one metadata store and persistence provider.
+ pub fn new(
+ store: Arc<MetadataStore>,
+ provider: Arc<dyn MultiWriteProvider>,
+ initial_partitions: u32,
+ backup_names: Vec<String>,
+ primary: Option<Arc<dyn FileSystem>>,
+ ) -> Self {
+ Self {
+ router: AccountRouter::new(store.clone()),
+ store,
+ provider,
+ primary,
+ initial_partitions,
+ backup_names,
+ }
+ }
+
+ /// Drain the receiver until every sender is closed and queued event is consumed.
+ pub async fn run(
+ mut self,
+ mut receiver: mpsc::UnboundedReceiver<PendingEvent>,
+ pending_events: Arc<AtomicUsize>,
+ ) {
+ while let Some(first) = receiver.recv().await {
+ let mut batch = vec![first];
+ while let Ok(event) = receiver.try_recv() {
+ batch.push(event);
+ }
+ let attempted = batch.len();
+ self.flush_batch(batch).await;
+ pending_events.fetch_sub(attempted, Ordering::Relaxed);
+ }
+ }
+
+ /// Flush one batch while recording and dropping failed events.
+ async fn flush_batch(&mut self, batch: Vec<PendingEvent>) {
+ let mut failed = false;
+ let mut accounts = HashMap::<String, Vec<PendingEvent>>::new();
+ for event in batch {
+ if matches!(event.kind, PendingEventKind::DeleteAccount) {
+ accounts
+ .entry(SYSTEM_DIR.to_string())
+ .or_default()
+ .push(Self::delete_scope_event(event));
+ continue;
+ }
+ accounts
+ .entry(event.account_id.clone())
+ .or_default()
+ .push(event);
+ }
+
+ for (account, events) in accounts {
+ match self.account_exists(&account).await {
+ Ok(false) => continue,
+ Err(error) => {
+ failed = true;
+ error!(account = %account, error = %error,
+ "multi-write primary account check failed");
+ continue;
+ }
+ Ok(true) => {}
+ }
+ match self
+ .store
+ .initialize_account(&account, self.initial_partitions, &self.backup_names)
+ .await
+ {
+ Ok(partitions) => {
+ if let Err(error) = bootstrap_account(
+ self.provider.as_ref(),
+ self.store.as_ref(),
+ &account,
+ &partitions,
+ )
+ .await
+ {
+ failed = true;
+ error!(account = %account, error = %error,
+ "multi-write provider bootstrap failed");
+ continue;
+ }
+ }
+ Err(error) => {
+ failed = true;
+ error!(
+ account = %account,
+ error = %error,
+ "multi-write account initialization failed"
+ );
+ continue;
+ }
+ }
+
+ let mut groups = HashMap::<ScopeKey, (PartitionContext, Vec<PendingEvent>)>::new();
+ for mut event in events {
+ match event.kind {
+ PendingEventKind::Data(SegmentEventType::Write | SegmentEventType::Remove) => {
+ match self.router.route(&account, &event.path).await {
+ Ok(route) => {
+ event.route_hint = Some(route.clone());
+ groups
+ .entry(route.scope.clone())
+ .or_insert_with(|| (route, Vec::new()))
+ .1
+ .push(event);
+ }
+ Err(error) => {
+ failed = true;
+ error!(
+ operation_id = %event.operation_id,
+ account = %account,
+ event_kind = ?event.kind,
+ error = %error,
+ "multi-write event routing failed"
+ );
+ }
+ }
+ }
+ PendingEventKind::Data(
+ SegmentEventType::RemoveTree
+ | SegmentEventType::MoveTree
+ | SegmentEventType::DeleteScope,
+ ) => {
+ if let Err(error) = self.flush_directory_event(&account, &event).await {
+ failed = true;
+ error!(
+ operation_id = %event.operation_id,
+ account = %account,
+ event_kind = ?event.kind,
+ error = %error,
+ "multi-write directory event persistence failed"
+ );
+ }
+ }
+ PendingEventKind::DeleteAccount => unreachable!(),
+ }
+ }
+
+ for (scope, (route, events)) in groups {
+ if let Err(error) = self.provider.flush(Some(&route), events).await {
+ failed = true;
+ error!(
+ account = %account,
+ partition = scope.partition_id,
+ epoch = scope.epoch,
+ error = %error,
+ "multi-write metadata flush failed"
+ );
+ }
+ }
+ }
+ if failed {
+ self.store.record_worker_error(MultiWriteWorker::Flush);
+ }
+ }
+
+ /// Convert one account-root deletion into a durable system-scope marker.
+ fn delete_scope_event(mut event: PendingEvent) -> PendingEvent {
+ event.account_id = SYSTEM_DIR.to_string();
+ event.kind = PendingEventKind::Data(SegmentEventType::DeleteScope);
+ event.route_hint = None;
+ event
+ }
+
+ /// Check account membership through the user registry when the primary view is available.
+ async fn account_exists(&self, account_id: &str) -> Result<bool> {
+ if account_id == SYSTEM_DIR {
+ return Ok(true);
+ }
+ let Some(primary) = &self.primary else {
+ return self.store.account_exists(account_id).await;
+ };
+ Ok(read_active_accounts(primary.as_ref())
+ .await?
+ .contains(account_id))
+ }
+
+ /// Register and persist one directory marker to every captured stable partition.
+ async fn flush_directory_event(&self, account: &str, event: &PendingEvent) -> Result<()> {
+ let scopes = self.register_directory_event(account, event).await?;
+ let mut failed = false;
+ for scope in scopes {
+ let result = match self.find_or_append_marker(&scope, event).await {
+ Ok(position) => {
+ self.backfill_position(account, event.operation_id, position)
+ .await
+ }
+ Err(error) => Err(error),
+ };
+ if let Err(error) = result {
+ failed = true;
+ error!(
+ operation_id = %event.operation_id, account = %account,
+ partition = scope.partition_id, epoch = scope.epoch, error = %error,
+ "multi-write directory marker scope remains incomplete"
+ );
+ }
+ }
+ if failed {
+ Err(Error::internal("directory marker scopes remain incomplete"))
+ } else {
+ Ok(())
+ }
+ }
+
+ /// Register one operation idempotently and return its missing stable scopes.
+ async fn register_directory_event(
+ &self,
+ account: &str,
+ event: &PendingEvent,
+ ) -> Result<Vec<ScopeKey>> {
+ let persisted = Self::directory_event(event)?;
+ self.store
+ .update_partitions_manifest(account, move |manifest| {
+ let index = manifest
+ .directory_events
+ .iter()
+ .position(|candidate| candidate.op_id == persisted.op_id);
+ if let Some(index) = index {
+ let existing = &manifest.directory_events[index];
+ if existing.operation != persisted.operation
+ || existing.source_path != persisted.source_path
+ || existing.destination_path != persisted.destination_path
+ {
+ return Err(Error::invalid_operation(
+ "directory op_id was reused with different content",
+ ));
+ }
+ } else {
+ manifest.directory_events.push(persisted);
+ }
+ let event = manifest
+ .directory_events
+ .iter()
+ .find(|candidate| candidate.op_id == event.operation_id)
+ .unwrap();
+ Ok(Self::missing_scopes(account, manifest, &event.positions))
+ })
+ .await
+ }
+
+ /// Return Stable scopes that do not yet have a marker position.
+ fn missing_scopes(
+ account: &str,
+ manifest: &PartitionsManifest,
+ positions: &[MarkerPosition],
+ ) -> Vec<ScopeKey> {
+ let completed = positions
+ .iter()
+ .map(|position| position.partition_id)
+ .collect::<HashSet<_>>();
+ manifest
+ .partitions
+ .iter()
+ .filter(|(id, partition)| {
+ partition.state == PartitionState::Stable && !completed.contains(id)
+ })
+ .map(|(&partition_id, _)| ScopeKey {
+ account_id: account.to_string(),
+ partition_id,
+ epoch: manifest.epoch,
+ })
+ .collect()
+ }
+
+ /// Convert one queued directory marker into its persisted notification.
+ fn directory_event(event: &PendingEvent) -> Result<DirectoryEvent> {
+ let operation = match event.kind {
+ PendingEventKind::Data(SegmentEventType::RemoveTree) => DirectoryOperation::RemoveTree,
+ PendingEventKind::Data(SegmentEventType::MoveTree) => DirectoryOperation::MoveTree,
+ PendingEventKind::Data(SegmentEventType::DeleteScope) => {
+ DirectoryOperation::DeleteScope
+ }
+ _ => {
+ return Err(Error::invalid_operation(
+ "directory notification requires a tree event",
+ ))
+ }
+ };
+ Ok(DirectoryEvent {
+ op_id: event.operation_id,
+ operation,
+ source_path: event.path.clone(),
+ destination_path: event.destination_path.clone(),
+ positions: Vec::new(),
+ })
+ }
+
+ /// Idempotently persist one marker position under the manifest Exact lock.
+ async fn backfill_position(
+ &self,
+ account: &str,
+ operation_id: uuid::Uuid,
+ position: MarkerPosition,
+ ) -> Result<()> {
+ self.store
+ .update_partitions_manifest(account, move |manifest| {
+ let event = manifest
+ .directory_events
+ .iter_mut()
+ .find(|event| event.op_id == operation_id)
+ .ok_or_else(|| Error::not_found(operation_id.to_string()))?;
+ if let Some(existing) = event
+ .positions
+ .iter()
+ .find(|existing| existing.partition_id == position.partition_id)
+ {
+ if existing != &position {
+ return Err(Error::invalid_operation(
+ "conflicting directory marker positions",
+ ));
+ }
+ return Ok(());
+ }
+ event.positions.push(position);
+ event
+ .positions
+ .sort_by_key(|position| position.partition_id);
+ Ok(())
+ })
+ .await
+ }
+
+ /// Recover every incomplete notification discoverable before queue processing starts.
+ async fn recover_directory_events(&self) {
+ let accounts = match self.store.initialized_accounts().await {
+ Ok(accounts) => accounts,
+ Err(error) => {
+ self.store.record_worker_error(MultiWriteWorker::Flush);
+ error!(error = %error, "multi-write directory recovery discovery failed");
+ return;
+ }
+ };
+ for account in accounts {
+ match self.recover_account(&account).await {
+ Ok(()) | Err(Error::NotFound(_)) => {}
+ Err(error) => {
+ self.store.record_worker_error(MultiWriteWorker::Flush);
+ error!(account = %account, error = %error,
+ "multi-write directory recovery failed");
+ }
+ }
+ }
+ }
+
+ /// Recover all currently missing marker positions for one account.
+ async fn recover_account(&self, account: &str) -> Result<()> {
+ let manifest_path = self.store.paths().account_manifest(account)?.0;
+ let manifest: PartitionsManifest = self.store.read_json(&manifest_path).await?;
+ manifest.validate()?;
+ let mut failed = false;
+ for persisted in &manifest.directory_events {
+ let event = PendingEvent {
+ operation_id: persisted.op_id,
+ account_id: account.to_string(),
+ kind: PendingEventKind::Data(match persisted.operation {
+ DirectoryOperation::RemoveTree => SegmentEventType::RemoveTree,
+ DirectoryOperation::MoveTree => SegmentEventType::MoveTree,
+ DirectoryOperation::DeleteScope => SegmentEventType::DeleteScope,
+ }),
+ path: persisted.source_path.clone(),
+ destination_path: persisted.destination_path.clone(),
+ route_hint: None,
+ };
+ for scope in Self::missing_scopes(account, &manifest, &persisted.positions) {
+ let result = self.find_or_append_marker(&scope, &event).await;
+ match result {
+ Ok(position) => {
+ if let Err(error) = self
+ .backfill_position(account, event.operation_id, position)
+ .await
+ {
+ failed = true;
+ error!(operation_id = %event.operation_id, account = %account,
+ partition = scope.partition_id, error = %error,
+ "multi-write recovered marker position update failed");
+ }
+ }
+ Err(error) => {
+ failed = true;
+ error!(operation_id = %event.operation_id, account = %account,
+ partition = scope.partition_id, error = %error,
+ "multi-write directory marker recovery scope failed");
+ }
+ }
+ }
+ }
+ if failed {
+ Err(Error::internal(
+ "directory marker recovery remains incomplete",
+ ))
+ } else {
+ Ok(())
+ }
+ }
+
+ /// Reuse a retained sealed marker or append it when none exists.
+ async fn find_or_append_marker(
+ &self,
+ scope: &ScopeKey,
+ event: &PendingEvent,
+ ) -> Result<MarkerPosition> {
+ match self.find_marker(scope, event).await? {
+ Some(position) => Ok(position),
+ None => self.provider.append_marker(scope, event).await,
+ }
+ }
+
+ /// Find one operation marker in currently retained sealed segments.
+ async fn find_marker(
+ &self,
+ scope: &ScopeKey,
+ event: &PendingEvent,
+ ) -> Result<Option<MarkerPosition>> {
+ let manifest = self.provider.read_manifest(scope).await?;
+ let directory = self
+ .store
+ .paths()
+ .segments_dir(&scope.account_id, scope.partition_id)?
+ .0;
+ for descriptor in manifest
+ .segments
+ .iter()
+ .filter(|segment| segment.segment_to_seq.is_some())
+ {
+ let bytes = self
+ .store
+ .read_bytes(&format!("{directory}/{}", descriptor.path))
+ .await?;
+ if bytes.len() as u64 != descriptor.byte_len
+ || descriptor.checksum.as_deref() != Some(&sha256_hex(&bytes))
+ {
+ return Err(Error::Serialization(
+ "sealed marker segment mismatch".to_string(),
+ ));
+ }
+ let decoded = decode_segment(&bytes)?;
+ if decoded.truncated_tail || decoded.valid_bytes != descriptor.byte_len {
+ return Err(Error::Serialization(
+ "sealed marker segment is truncated".to_string(),
+ ));
+ }
+ if let Some(record) = decoded
+ .records
+ .iter()
+ .find(|record| record.op_id == Some(event.operation_id))
+ {
+ let PendingEventKind::Data(kind) = event.kind else {
+ unreachable!()
+ };
+ if record.event_type != kind
+ || record.path != event.path
+ || record.destination_path != event.destination_path
+ {
+ return Err(Error::Serialization(
+ "directory op_id marker mismatch".to_string(),
+ ));
+ }
+ return Ok(Some(MarkerPosition {
+ partition_id: scope.partition_id,
+ epoch: scope.epoch,
+ seq: record.seq,
+ }));
+ }
+ }
+ Ok(None)
+ }
+}
diff --git a/crates/ragfs/src/multibackend/tests/test_catch_up.rs b/crates/ragfs/src/multibackend/tests/test_catch_up.rs
new file mode 100644
index 000000000..2ee8a5a33
--- /dev/null
+++ b/crates/ragfs/src/multibackend/tests/test_catch_up.rs
@@ -0,0 +1,89 @@
+use super::*;
+
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use crate::core::context::{FsContextInner, FS_CTX};
+use crate::core::encryption_wrapper::EncryptionWrappedFS;
+use crate::core::{MountableFS, PluginConfig};
+use crate::crypto;
+use crate::lock::{MemoryPathLockProvider, PathLockConfig, PathLockManager};
+use crate::plugins::memfs::MemFileSystem;
+use crate::plugins::MemFSPlugin;
+
+/// Build one encrypted MemFS backup and expose its raw mounted storage.
+async fn encrypted_backup() -> (Arc<MountableFS>, EncryptionWrappedFS) {
+ let raw = Arc::new(MountableFS::new());
+ raw.register_plugin(MemFSPlugin).await;
+ raw.mount(PluginConfig::single_backend(
+ "memfs",
+ "/mem",
+ HashMap::new(),
+ ))
+ .await
+ .unwrap();
+ let manager = Arc::new(PathLockManager::new(
+ raw.clone() as Arc<dyn FileSystem>,
+ Arc::new(MemoryPathLockProvider::new()),
+ PathLockConfig::default(),
+ ));
+ raw.set_pathlock_manager(manager.clone()).await;
+ let encrypted = EncryptionWrappedFS::new(
+ raw.clone(),
+ [7_u8; 32],
+ crypto::PROVIDER_LOCAL,
+ manager,
+ "/mem".to_string(),
+ );
+ (raw, encrypted)
+}
+
+/// Verify encrypted writes preserve their historical physical byte count.
+#[tokio::test]
+async fn encrypted_write_returns_physical_length() {
+ let (raw, encrypted) = encrypted_backup().await;
+ let data = b"plain";
+ let written = FS_CTX
+ .scope(
+ Arc::new(FsContextInner::new("acct")),
+ encrypted.write("/mem/file.txt", data, 0, WriteFlag::Create),
+ )
+ .await
+ .unwrap();
+ let stored = raw.read("/mem/file.txt", 0, 0).await.unwrap();
+
+ assert_eq!(written, stored.len() as u64);
+ assert!(written > data.len() as u64);
+}
+
+/// Verify catch-up accepts an encrypted backend's physical byte count.
+#[tokio::test]
+async fn reconcile_file_accepts_encrypted_write_length() {
+ let primary = MemFileSystem::new();
+ primary
+ .ensure_parent_dirs("/mem/file.txt", 0o755)
+ .await
+ .unwrap();
+ primary
+ .write("/mem/file.txt", b"plain", 0, WriteFlag::Create)
+ .await
+ .unwrap();
+ let (_, encrypted) = encrypted_backup().await;
+
+ FS_CTX
+ .scope(
+ Arc::new(FsContextInner::new("acct")),
+ reconcile_file(&primary, &encrypted, "/mem/file.txt"),
+ )
+ .await
+ .unwrap();
+
+ let copied = FS_CTX
+ .scope(
+ Arc::new(FsContextInner::new("acct")),
+ encrypted.read("/mem/file.txt", 0, 0),
+ )
+ .await
+ .unwrap();
+ assert_eq!(copied, b"plain");
+}
diff --git a/crates/ragfs/src/multibackend/tests/test_codec.rs b/crates/ragfs/src/multibackend/tests/test_codec.rs
new file mode 100644
index 000000000..2dc542fea
--- /dev/null
+++ b/crates/ragfs/src/multibackend/tests/test_codec.rs
@@ -0,0 +1,35 @@
+use super::*;
+use uuid::Uuid;
+
+/// Verifies segment round trips and recovery from an incomplete final frame.
+#[test]
+fn segment_round_trip_recovers_complete_prefix_from_truncated_tail() {
+ let records = vec![
+ SegmentRecord {
+ seq: 1,
+ event_type: SegmentEventType::Write,
+ path: "/local/account/file.txt".to_string(),
+ op_id: None,
+ destination_path: None,
+ },
+ SegmentRecord {
+ seq: 2,
+ event_type: SegmentEventType::MoveTree,
+ path: "/local/account/source".to_string(),
+ op_id: Some(Uuid::from_u128(7)),
+ destination_path: Some("/local/account/destination".to_string()),
+ },
+ ];
+
+ let encoded = encode_segment(&records).unwrap();
+ let decoded = decode_segment(&encoded).unwrap();
+ assert_eq!(decoded.records, records);
+ assert_eq!(decoded.valid_bytes, encoded.len() as u64);
+ assert!(!decoded.truncated_tail);
+
+ let first_record_bytes = encode_segment(&records[..1]).unwrap();
+ let truncated = decode_segment(&encoded[..encoded.len() - 1]).unwrap();
+ assert_eq!(truncated.records, records[..1]);
+ assert_eq!(truncated.valid_bytes, first_record_bytes.len() as u64);
+ assert!(truncated.truncated_tail);
+}
diff --git a/crates/ragfs/src/multibackend/tests/test_model.rs b/crates/ragfs/src/multibackend/tests/test_model.rs
new file mode 100644
index 000000000..1f6678110
--- /dev/null
+++ b/crates/ragfs/src/multibackend/tests/test_model.rs
@@ -0,0 +1,45 @@
+use super::*;
+use std::collections::BTreeMap;
+use uuid::Uuid;
+
+/// Verifies complete routing and directory marker ownership invariants.
+#[test]
+fn partitions_manifest_rejects_route_gaps_and_stale_markers() {
+ let manifest = PartitionsManifest {
+ version: 1,
+ epoch: 3,
+ vbuckets: VBUCKETS,
+ partitions: BTreeMap::from([(
+ 0,
+ PartitionEntry {
+ state: PartitionState::Stable,
+ },
+ )]),
+ routes: vec![RouteEntry {
+ start: 0,
+ end: VBUCKETS - 1,
+ partition: 0,
+ }],
+ directory_events: vec![DirectoryEvent {
+ op_id: Uuid::from_u128(1),
+ operation: DirectoryOperation::RemoveTree,
+ source_path: "/local/account/removed".to_string(),
+ destination_path: None,
+ positions: vec![MarkerPosition {
+ partition_id: 0,
+ epoch: 3,
+ seq: 1,
+ }],
+ }],
+ };
+
+ assert!(manifest.validate().is_ok());
+
+ let mut route_gap = manifest.clone();
+ route_gap.routes[0].start = 1;
+ assert!(route_gap.validate().is_err());
+
+ let mut stale_marker = manifest;
+ stale_marker.directory_events[0].positions[0].epoch = 2;
+ assert!(stale_marker.validate().is_err());
+}
diff --git a/crates/ragfs/src/multibackend/tests/test_runtime.rs b/crates/ragfs/src/multibackend/tests/test_runtime.rs
new file mode 100644
index 000000000..e9523af26
--- /dev/null
+++ b/crates/ragfs/src/multibackend/tests/test_runtime.rs
@@ -0,0 +1,97 @@
+use super::*;
+use crate::core::{FileSystem, WriteFlag};
+use crate::lock::{MemoryPathLockProvider, PathLockConfig, PathLockManager};
+use crate::multibackend::provider::FilesystemProvider;
+use crate::plugins::memfs::MemFileSystem;
+
+/// Verifies deleted registry entries are excluded while null deletions stay active.
+#[tokio::test]
+async fn active_accounts_exclude_completed_deletions() {
+ let primary = MemFileSystem::new();
+ let registry = serde_json::json!({
+ "accounts": {
+ "active": {},
+ "pending": {"deletion": null},
+ "deleted": {"deletion": {"deleted_at_ns": 1}}
+ }
+ });
+ primary
+ .ensure_parent_dirs("/_system/accounts.json", 0o755)
+ .await
+ .unwrap();
+ primary
+ .write(
+ "/_system/accounts.json",
+ &serde_json::to_vec(&registry).unwrap(),
+ 0,
+ WriteFlag::Create,
+ )
+ .await
+ .unwrap();
+
+ let accounts = read_active_accounts(&primary).await.unwrap();
+ assert_eq!(
+ accounts,
+ HashSet::from(["active".to_string(), "pending".to_string()])
+ );
+}
+
+/// Verifies delete events are persisted in the system scope and remove backup data.
+#[tokio::test]
+async fn delete_account_event_removes_backup_scope() {
+ let primary: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ let backup: Arc<dyn FileSystem> = Arc::new(MemFileSystem::new());
+ primary
+ .ensure_parent_dirs("/_system/accounts.json", 0o755)
+ .await
+ .unwrap();
+ primary
+ .write(
+ "/_system/accounts.json",
+ br#"{"accounts":{"account":{}}}"#,
+ 0,
+ WriteFlag::Create,
+ )
+ .await
+ .unwrap();
+ backup.mkdir("/account", 0o755).await.unwrap();
+
+ let manager = Arc::new(PathLockManager::new(
+ primary.clone(),
+ Arc::new(MemoryPathLockProvider::new()),
+ PathLockConfig::default(),
+ ));
+ let store = Arc::new(MetadataStore::new(primary.clone(), manager, "/local").unwrap());
+ let provider = Arc::new(FilesystemProvider::new(store.clone()));
+ let catch_up = Arc::new(CatchUpWorker::new(
+ "backup".to_string(),
+ primary.clone(),
+ backup.clone(),
+ store.clone(),
+ provider.clone(),
+ ));
+ let mut worker = FlushWorker::new(
+ store,
+ provider,
+ 1,
+ vec!["backup".to_string()],
+ Some(primary),
+ );
+
+ worker
+ .flush_batch(vec![PendingEvent {
+ operation_id: uuid::Uuid::from_u128(1),
+ account_id: "account".to_string(),
+ kind: PendingEventKind::DeleteAccount,
+ path: "/local/account".to_string(),
+ destination_path: None,
+ route_hint: None,
+ }])
+ .await;
+ catch_up.run_once().await.unwrap();
+
+ assert!(matches!(
+ backup.stat("/account").await.unwrap_err(),
+ Error::NotFound(_)
+ ));
+}
diff --git a/crates/ragfs/src/multibackend/types.rs b/crates/ragfs/src/multibackend/types.rs
index 78bb71d66..09a4f53ec 100644
--- a/crates/ragfs/src/multibackend/types.rs
+++ b/crates/ragfs/src/multibackend/types.rs
@@ -2,6 +2,8 @@
use std::sync::Arc;
+#[cfg(feature = "cache")]
+use crate::cache_runtime::CacheRuntime;
use crate::lock::PathLockManager;
/// Encryption context used while assembling one multi-backend mount.
@@ -15,6 +17,9 @@ pub struct MultiBackendBuildContext {
pub pathlock_manager: Arc<PathLockManager>,
/// Mount prefix used to restore manager-visible backend paths.
pub backend_prefix: String,
+ /// Optional shared cache runtime used by cache-backed metadata.
+ #[cfg(feature = "cache")]
+ pub cache_runtime: Option<Arc<CacheRuntime>>,
}
impl MultiBackendBuildContext {
diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rs
index 67b50a7dc..463f7ae3b 100644
--- a/crates/ragfs/src/plugins/localfs/mod.rs
+++ b/crates/ragfs/src/plugins/localfs/mod.rs
@@ -897,6 +897,8 @@ impl FileSystem for LocalFileSystem {
let mut file = options.open(&local_path).map_err(|e| {
if e.kind() == std::io::ErrorKind::NotFound {
Error::NotFound(path.to_string())
+ } else if e.kind() == std::io::ErrorKind::AlreadyExists {
+ Error::AlreadyExists(path.to_string())
} else {
Error::plugin(format!("failed to open file: {}", e))
}
diff --git a/crates/ragfs/tests/cache_wrapper.rs b/crates/ragfs/tests/cache_wrapper.rs
deleted file mode 100644
index 4c31ab288..000000000
--- a/crates/ragfs/tests/cache_wrapper.rs
+++ /dev/null
@@ -1,1478 +0,0 @@
-use async_trait::async_trait;
-use ragfs::cache::{
- CacheDecision, CacheNamespace, CachePolicy, CacheTraversalMode, CachedFileSystem,
-};
-use ragfs::cache_runtime::{CacheRuntime, MemoryMockProvider};
-use ragfs::core::{
- FsContextInner, GrepOptions, GrepResult, MultiWriteWrappedFS, TreeEntry, FS_CTX,
-};
-use ragfs::plugins::MemFileSystem;
-use ragfs::{Error, FileInfo, FileSystem, Result, WriteFlag};
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::Arc;
-use std::time::Duration;
-
-#[derive(Clone)]
-struct CountingFileSystem {
- inner: Arc<MemFileSystem>,
- reads: Arc<AtomicU64>,
- read_dirs: Arc<AtomicU64>,
- stats: Arc<AtomicU64>,
- greps: Arc<AtomicU64>,
- trees: Arc<AtomicU64>,
- read_delay: Duration,
- partial_remove_path: Option<String>,
-}
-
-impl CountingFileSystem {
- fn new() -> Self {
- Self {
- inner: Arc::new(MemFileSystem::new()),
- reads: Arc::new(AtomicU64::new(0)),
- read_dirs: Arc::new(AtomicU64::new(0)),
- stats: Arc::new(AtomicU64::new(0)),
- greps: Arc::new(AtomicU64::new(0)),
- trees: Arc::new(AtomicU64::new(0)),
- read_delay: Duration::ZERO,
- partial_remove_path: None,
- }
- }
-
- fn with_read_delay(mut self, delay: Duration) -> Self {
- self.read_delay = delay;
- self
- }
-
- fn with_partial_remove_failure(mut self, path: &str) -> Self {
- self.partial_remove_path = Some(path.to_string());
- self
- }
-
- fn read_count(&self) -> u64 {
- self.reads.load(Ordering::Relaxed)
- }
-
- fn read_dir_count(&self) -> u64 {
- self.read_dirs.load(Ordering::Relaxed)
- }
-
- fn stat_count(&self) -> u64 {
- self.stats.load(Ordering::Relaxed)
- }
-
- fn grep_count(&self) -> u64 {
- self.greps.load(Ordering::Relaxed)
- }
-
- fn tree_count(&self) -> u64 {
- self.trees.load(Ordering::Relaxed)
- }
-}
-
-#[async_trait]
-impl FileSystem for CountingFileSystem {
- async fn create(&self, path: &str) -> Result<()> {
- self.inner.create(path).await
- }
-
- async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {
- self.inner.mkdir(path, mode).await
- }
-
- async fn remove(&self, path: &str) -> Result<()> {
- self.inner.remove(path).await
- }
-
- async fn remove_all(&self, path: &str) -> Result<()> {
- if let Some(partial_remove_path) = &self.partial_remove_path {
- self.inner.remove(partial_remove_path).await?;
- return Err(Error::internal(format!(
- "remove_all partially removed {partial_remove_path} under {path}"
- )));
- }
- self.inner.remove_all(path).await
- }
-
- async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {
- self.reads.fetch_add(1, Ordering::Relaxed);
- if !self.read_delay.is_zero() {
- tokio::time::sleep(self.read_delay).await;
- }
- self.inner.read(path, offset, size).await
- }
-
- async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {
- self.inner.write(path, data, offset, flags).await
- }
-
- async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {
- self.read_dirs.fetch_add(1, Ordering::Relaxed);
- self.inner.read_dir(path).await
- }
-
- async fn stat(&self, path: &str) -> Result<FileInfo> {
- self.stats.fetch_add(1, Ordering::Relaxed);
- self.inner.stat(path).await
- }
-
- async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {
- self.inner.rename(old_path, new_path).await
- }
-
- async fn chmod(&self, path: &str, mode: u32) -> Result<()> {
- self.inner.chmod(path, mode).await
- }
-
- async fn truncate(&self, path: &str, size: u64) -> Result<()> {
- self.inner.truncate(path, size).await
- }
-
- async fn grep(
- &self,
- path: &str,
- pattern: &str,
- options: GrepOptions<'_>,
- ) -> Result<GrepResult> {
- self.greps.fetch_add(1, Ordering::Relaxed);
- self.inner.grep(path, pattern, options).await
- }
-
- async fn tree_directory(
- &self,
- path: &str,
- show_hidden: bool,
- node_limit: Option<usize>,
- level_limit: Option<usize>,
- ) -> Result<Vec<TreeEntry>> {
- self.trees.fetch_add(1, Ordering::Relaxed);
- self.inner
- .tree_directory(path, show_hidden, node_limit, level_limit)
- .await
- }
-}
-
-fn cached_fs(backend: CountingFileSystem) -> (Arc<CachedFileSystem>, Arc<MemoryMockProvider>) {
- cached_fs_with_policy(backend, CachePolicy::default())
-}
-
-fn cached_fs_with_policy(
- backend: CountingFileSystem,
- policy: CachePolicy,
-) -> (Arc<CachedFileSystem>, Arc<MemoryMockProvider>) {
- let provider = Arc::new(MemoryMockProvider::new());
- let fs = Arc::new(CachedFileSystem::with_runtime(
- Box::new(backend),
- CacheRuntime::memory_with_provider(provider.clone()),
- CacheNamespace::new("test"),
- policy,
- ));
- (fs, provider)
-}
-
-fn cached_fs_with_tracking_provider(
- backend: CountingFileSystem,
- policy: CachePolicy,
-) -> (Arc<CachedFileSystem>, Arc<MemoryMockProvider>) {
- cached_fs_with_tracking_provider_instance(backend, policy, Arc::new(MemoryMockProvider::new()))
-}
-
-fn cached_fs_with_tracking_provider_instance(
- backend: CountingFileSystem,
- policy: CachePolicy,
- provider: Arc<MemoryMockProvider>,
-) -> (Arc<CachedFileSystem>, Arc<MemoryMockProvider>) {
- let fs = Arc::new(CachedFileSystem::with_runtime(
- Box::new(backend),
- CacheRuntime::memory_with_provider(provider.clone()),
- CacheNamespace::new("tracking"),
- policy,
- ));
- (fs, provider)
-}
-
-#[test]
-fn cache_policy_traversal_mode_defaults_to_backend() {
- assert_eq!(
- CachePolicy::default().traversal_mode(),
- CacheTraversalMode::Backend
- );
-}
-
-#[tokio::test]
-async fn cached_filesystem_accepts_the_unified_runtime_without_changing_read_through() {
- let backend = CountingFileSystem::new();
- backend
- .write("/runtime.md", b"runtime", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let fs = CachedFileSystem::with_runtime(
- Box::new(backend),
- CacheRuntime::memory(),
- CacheNamespace::new("runtime"),
- CachePolicy::default(),
- );
-
- assert_eq!(fs.read("/runtime.md", 0, 0).await.unwrap(), b"runtime");
- assert_eq!(fs.read("/runtime.md", 0, 0).await.unwrap(), b"runtime");
- assert_eq!(probe.read_count(), 1);
-}
-
-#[tokio::test]
-async fn unified_runtime_failure_still_falls_back_to_backend_reads() {
- let backend = CountingFileSystem::new();
- backend
- .write("/fallback.txt", b"backend", 0, WriteFlag::Create)
- .await
- .unwrap();
- let runtime = CacheRuntime::memory();
- runtime.close().await.unwrap();
- let cached = CachedFileSystem::with_runtime(
- Box::new(backend),
- runtime,
- CacheNamespace::new("runtime-fail-open"),
- CachePolicy::default(),
- );
-
- assert_eq!(
- cached.read("/fallback.txt", 0, 0).await.unwrap(),
- b"backend"
- );
-}
-
-#[tokio::test]
-async fn default_tree_directory_delegates_to_backend() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend
- .write("/docs/one.md", b"one", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- let entries = fs.tree_directory("/docs", false, None, None).await.unwrap();
-
- assert_eq!(entries.len(), 1);
- assert_eq!(probe.tree_count(), 1);
- assert_eq!(probe.read_dir_count(), 0);
-}
-
-#[tokio::test]
-async fn default_grep_delegates_to_backend() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend
- .write("/docs/one.md", b"needle", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- let result = fs
- .grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .await
- .unwrap();
-
- assert_eq!(result.count, 1);
- assert_eq!(probe.grep_count(), 1);
- assert_eq!(probe.read_dir_count(), 0);
- assert_eq!(probe.read_count(), 0);
-}
-
-#[tokio::test]
-async fn cached_tree_traversal_reuses_directory_cache_after_warmup() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend.mkdir("/docs/sub", 0o755).await.unwrap();
- backend
- .write("/docs/one.md", b"one", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/docs/sub/two.md", b"two", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs_with_policy(
- backend,
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- let first = fs.tree_directory("/docs", false, None, None).await.unwrap();
- assert_eq!(first.len(), 3);
- assert_eq!(probe.tree_count(), 0);
- assert_eq!(probe.read_dir_count(), 2);
-
- let second = fs.tree_directory("/docs", false, None, None).await.unwrap();
- assert_eq!(second.len(), 3);
- assert_eq!(
- probe.read_dir_count(),
- 2,
- "second tree should reuse cached read_dir entries"
- );
-}
-
-#[tokio::test]
-async fn cached_grep_traversal_reuses_directory_and_file_cache_after_warmup() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend.mkdir("/docs/sub", 0o755).await.unwrap();
- backend
- .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs_with_policy(
- backend,
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- let first = fs
- .grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .await
- .unwrap();
- assert_eq!(first.count, 2);
- assert_eq!(probe.grep_count(), 0);
- assert_eq!(probe.read_dir_count(), 2);
- assert_eq!(probe.read_count(), 2);
- assert_eq!(probe.stat_count(), 1);
-
- let second = fs
- .grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .await
- .unwrap();
- assert_eq!(second.count, 2);
- assert_eq!(
- probe.read_dir_count(),
- 2,
- "second grep should reuse cached read_dir entries"
- );
- assert_eq!(
- probe.read_count(),
- 2,
- "second grep should reuse cached file contents"
- );
- assert_eq!(
- probe.stat_count(),
- 2,
- "second grep should only stat the query root, not every cached entry"
- );
-}
-
-#[tokio::test]
-async fn cached_grep_includes_context_from_cached_content() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend
- .write(
- "/docs/a.md",
- b"first\nbefore\nneedle\nafter",
- 0,
- WriteFlag::Create,
- )
- .await
- .unwrap();
- let (fs, _) = cached_fs_with_policy(
- backend,
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- let result = fs
- .grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- before_context: 2,
- after_context: 1,
- ..Default::default()
- },
- )
- .await
- .unwrap();
-
- let matched = &result.matches[0];
- assert_eq!(matched.before_context.as_ref().unwrap()[0].content, "first");
- assert_eq!(
- matched.before_context.as_ref().unwrap()[1].content,
- "before"
- );
- assert_eq!(matched.after_context.as_ref().unwrap()[0].content, "after");
-}
-
-#[tokio::test]
-async fn cached_grep_batches_generation_validation_after_warmup() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend.mkdir("/docs/sub", 0o755).await.unwrap();
- backend
- .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create)
- .await
- .unwrap();
- let (fs, provider) = cached_fs_with_tracking_provider(
- backend,
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- fs.grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .await
- .unwrap();
- provider.reset_observed_reads();
-
- let result = fs
- .grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .await
- .unwrap();
-
- assert_eq!(result.count, 2);
- assert!(
- provider.batch_get_count() > 0,
- "warm cached grep should batch generation validation reads"
- );
-}
-
-#[tokio::test]
-async fn cached_grep_memoizes_generation_keys_within_one_traversal() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend.mkdir("/docs/sub", 0o755).await.unwrap();
- backend
- .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create)
- .await
- .unwrap();
- let (fs, provider) = cached_fs_with_tracking_provider(
- backend,
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- fs.grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .await
- .unwrap();
- provider.reset_observed_reads();
-
- let result = fs
- .grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .await
- .unwrap();
-
- assert_eq!(result.count, 2);
- let subtree_keys = provider
- .observed_read_keys()
- .into_iter()
- .filter(|key| key.contains(":subtree:"))
- .collect::<Vec<_>>();
- let unique = subtree_keys
- .iter()
- .collect::<std::collections::HashSet<_>>();
- assert_eq!(
- subtree_keys.len(),
- unique.len(),
- "one cached grep traversal should not re-read the same generation key"
- );
-}
-
-#[tokio::test]
-async fn cached_grep_scans_cached_files_with_bounded_concurrency() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- for index in 0..8 {
- backend
- .write(
- &format!("/docs/{index}.md"),
- b"needle\nplain",
- 0,
- WriteFlag::Create,
- )
- .await
- .unwrap();
- }
- let provider = Arc::new(MemoryMockProvider::new().with_get_delay(Duration::from_millis(30)));
- let (fs, provider) = cached_fs_with_tracking_provider_instance(
- backend,
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- provider,
- );
-
- fs.grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .await
- .unwrap();
- provider.reset_observed_reads();
-
- let result = fs
- .grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .await
- .unwrap();
-
- assert_eq!(result.count, 8);
- let max_gets = provider.max_concurrent_gets();
- assert!(
- max_gets > 1,
- "warm cached grep should scan cached file reads concurrently"
- );
- assert!(
- max_gets <= 8,
- "cached grep file scan concurrency should stay bounded"
- );
-}
-
-#[tokio::test]
-async fn cached_grep_traversal_matches_default_grep_semantics() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend.mkdir("/docs/sub", 0o755).await.unwrap();
- backend.mkdir("/docs/skip", 0o755).await.unwrap();
- backend
- .write("/docs/a.md", b"Needle\nneedle", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/docs/sub/b.md", b"needle\nmiss", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/docs/skip/c.md", b"needle", 0, WriteFlag::Create)
- .await
- .unwrap();
- let direct = backend.clone();
- let (fs, _) = cached_fs_with_policy(
- backend,
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- let cached = fs
- .grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- case_insensitive: true,
- node_limit: Some(2),
- exclude_path: Some("/docs/skip"),
- level_limit: Some(1),
- ..Default::default()
- },
- )
- .await
- .unwrap();
- let direct_result = direct
- .grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- case_insensitive: true,
- node_limit: Some(2),
- exclude_path: Some("/docs/skip"),
- level_limit: Some(1),
- ..Default::default()
- },
- )
- .await
- .unwrap();
-
- let tuples = |result: &GrepResult| {
- result
- .matches
- .iter()
- .map(|m| (m.file.clone(), m.line, m.content.clone()))
- .collect::<Vec<_>>()
- };
- assert_eq!(tuples(&cached), tuples(&direct_result));
-}
-
-#[tokio::test]
-async fn cached_grep_traversal_falls_back_for_multiwrite_backend() {
- let primary = CountingFileSystem::new();
- primary.mkdir("/docs", 0o755).await.unwrap();
- primary
- .write("/docs/a.md", b"needle", 0, WriteFlag::Create)
- .await
- .unwrap();
- let primary_probe = primary.clone();
- let multiwrite = MultiWriteWrappedFS::builder(Arc::new(primary))
- .build()
- .unwrap();
- let fs = CachedFileSystem::with_runtime(
- Box::new(multiwrite),
- CacheRuntime::memory(),
- CacheNamespace::new("grep-multiwrite"),
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- let ctx = Arc::new(FsContextInner::new("acct"));
- let result = FS_CTX
- .scope(ctx, async {
- fs.grep(
- "/docs",
- "needle",
- GrepOptions {
- recursive: true,
- ..Default::default()
- },
- )
- .await
- .unwrap()
- })
- .await;
-
- assert_eq!(result.count, 1);
- assert_eq!(
- primary_probe.grep_count(),
- 1,
- "multi-write backends must keep their backend grep path"
- );
- assert_eq!(
- primary_probe.read_dir_count(),
- 0,
- "cached traversal must not bypass multi-write grep semantics"
- );
-}
-
-#[tokio::test]
-async fn cached_tree_traversal_matches_default_tree_semantics() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend.mkdir("/docs/sub", 0o755).await.unwrap();
- backend
- .write("/docs/a.md", b"a", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/docs/.hidden.md", b"hidden", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/docs/sub/b.md", b"b", 0, WriteFlag::Create)
- .await
- .unwrap();
- let direct = backend.clone();
- let (fs, _) = cached_fs_with_policy(
- backend,
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- let cached = fs
- .tree_directory("/docs", false, None, Some(1))
- .await
- .unwrap();
- let direct_entries = direct
- .tree_directory("/docs", false, None, Some(1))
- .await
- .unwrap();
- assert_eq!(
- cached
- .iter()
- .map(|entry| entry.rel_path.as_str())
- .collect::<Vec<_>>(),
- direct_entries
- .iter()
- .map(|entry| entry.rel_path.as_str())
- .collect::<Vec<_>>()
- );
-
- let with_hidden = fs.tree_directory("/docs", true, None, None).await.unwrap();
- assert!(
- with_hidden
- .iter()
- .any(|entry| entry.rel_path == ".hidden.md"),
- "show_hidden=true should include hidden files"
- );
-}
-
-#[tokio::test]
-async fn cached_tree_traversal_returns_oversized_directories_without_caching_them() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/large", 0o755).await.unwrap();
- for index in 0..4097 {
- backend
- .write(
- &format!("/large/{index:04}.txt"),
- b"x",
- 0,
- WriteFlag::Create,
- )
- .await
- .unwrap();
- }
- let probe = backend.clone();
- let (fs, _) = cached_fs_with_policy(
- backend,
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- assert_eq!(
- fs.tree_directory("/large", false, None, None)
- .await
- .unwrap()
- .len(),
- 4097
- );
- assert_eq!(
- fs.tree_directory("/large", false, None, None)
- .await
- .unwrap()
- .len(),
- 4097
- );
- assert_eq!(
- probe.read_dir_count(),
- 2,
- "oversized directories should not be cached by tree traversal"
- );
-}
-
-#[tokio::test]
-async fn cached_tree_traversal_falls_back_when_provider_is_unavailable() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend
- .write("/docs/a.md", b"a", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let provider = Arc::new(MemoryMockProvider::new());
- provider.set_unavailable(true);
- let fs = CachedFileSystem::with_runtime(
- Box::new(backend),
- CacheRuntime::memory_with_provider(provider),
- CacheNamespace::new("tree-unavailable"),
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- assert_eq!(
- fs.tree_directory("/docs", false, None, None)
- .await
- .unwrap()
- .len(),
- 1
- );
- assert_eq!(
- fs.tree_directory("/docs", false, None, None)
- .await
- .unwrap()
- .len(),
- 1
- );
- assert_eq!(probe.read_dir_count(), 2);
- assert!(fs.metrics().snapshot().errors >= 1);
-}
-
-#[tokio::test]
-async fn cached_tree_traversal_falls_back_for_multiwrite_backend() {
- let primary = CountingFileSystem::new();
- primary.mkdir("/docs", 0o755).await.unwrap();
- primary
- .write("/docs/a.md", b"a", 0, WriteFlag::Create)
- .await
- .unwrap();
- let primary_probe = primary.clone();
- let multiwrite = MultiWriteWrappedFS::builder(Arc::new(primary))
- .build()
- .unwrap();
- let fs = CachedFileSystem::with_runtime(
- Box::new(multiwrite),
- CacheRuntime::memory(),
- CacheNamespace::new("tree-multiwrite"),
- CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),
- );
-
- let ctx = Arc::new(FsContextInner::new("acct"));
- let entries = FS_CTX
- .scope(ctx, async {
- fs.tree_directory("/docs", false, None, None).await.unwrap()
- })
- .await;
-
- assert_eq!(entries.len(), 1);
- assert_eq!(
- primary_probe.tree_count(),
- 1,
- "multi-write backends must keep their backend tree_directory path"
- );
- assert_eq!(
- primary_probe.read_dir_count(),
- 0,
- "cached traversal must not bypass multi-write tree semantics"
- );
-}
-
-#[test]
-fn cache_policy_supports_explicit_permission_sensitive_prefixes() {
- let policy = CachePolicy::default().with_bypass_prefix("/private");
-
- assert!(!policy.cache_file("/private/secret.md", 10));
- assert!(!policy.cache_directory("/private/docs"));
- assert!(policy.cache_file("/public/.overview.md", 10));
-}
-
-#[test]
-fn cache_policy_marks_high_value_objects_as_preferred() {
- let policy = CachePolicy::default();
-
- assert_eq!(
- policy.file_decision("/docs/.abstract.md", 128),
- CacheDecision::Prefer
- );
- assert_eq!(
- policy.file_decision("/docs/.overview.md", 128),
- CacheDecision::Prefer
- );
- assert_eq!(
- policy.file_decision("/docs/note.md", 128),
- CacheDecision::Cache
- );
- assert_eq!(policy.directory_decision("/docs"), CacheDecision::Prefer);
-}
-
-#[tokio::test]
-async fn full_file_reads_are_read_through_cached_but_range_reads_bypass() {
- let backend = CountingFileSystem::new();
- backend
- .write("/note.md", b"hello world", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- assert_eq!(fs.read("/note.md", 0, 0).await.unwrap(), b"hello world");
- assert_eq!(fs.read("/note.md", 0, 0).await.unwrap(), b"hello world");
- assert_eq!(probe.read_count(), 1);
-
- assert_eq!(fs.read("/note.md", 6, 5).await.unwrap(), b"world");
- assert_eq!(probe.read_count(), 2);
-
- let metrics = fs.metrics().snapshot();
- assert_eq!(metrics.file_hits, 1);
- assert_eq!(metrics.file_misses, 1);
- assert_eq!(metrics.backend_fallbacks, 1);
-}
-
-#[tokio::test]
-async fn read_dir_is_cached_and_parent_changes_invalidate_it() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/docs", 0o755).await.unwrap();
- backend
- .write("/docs/one.md", b"one", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- assert_eq!(fs.read_dir("/docs").await.unwrap().len(), 1);
- assert_eq!(fs.read_dir("/docs").await.unwrap().len(), 1);
- assert_eq!(probe.read_dir_count(), 1);
-
- fs.write("/docs/two.md", b"two", 0, WriteFlag::Create)
- .await
- .unwrap();
- let entries = fs.read_dir("/docs").await.unwrap();
- assert_eq!(entries.len(), 2);
- assert_eq!(probe.read_dir_count(), 2);
-
- let metrics = fs.metrics().snapshot();
- assert_eq!(metrics.read_dir_hits, 1);
- assert_eq!(metrics.read_dir_misses, 2);
- assert!(metrics.invalidations >= 1);
-}
-
-#[tokio::test]
-async fn oversized_directories_bypass_directory_cache() {
- let cacheable = CountingFileSystem::new();
- cacheable.mkdir("/small", 0o755).await.unwrap();
- for index in 0..4096 {
- cacheable
- .write(
- &format!("/small/{index:04}.txt"),
- b"x",
- 0,
- WriteFlag::Create,
- )
- .await
- .unwrap();
- }
- let cacheable_probe = cacheable.clone();
- let (cacheable_fs, _) = cached_fs(cacheable);
-
- assert_eq!(cacheable_fs.read_dir("/small").await.unwrap().len(), 4096);
- assert_eq!(cacheable_fs.read_dir("/small").await.unwrap().len(), 4096);
- assert_eq!(cacheable_probe.read_dir_count(), 1);
-
- let oversized = CountingFileSystem::new();
- oversized.mkdir("/large", 0o755).await.unwrap();
- for index in 0..4097 {
- oversized
- .write(
- &format!("/large/{index:04}.txt"),
- b"x",
- 0,
- WriteFlag::Create,
- )
- .await
- .unwrap();
- }
- let oversized_probe = oversized.clone();
- let (oversized_fs, _) = cached_fs(oversized);
-
- assert_eq!(oversized_fs.read_dir("/large").await.unwrap().len(), 4097);
- assert_eq!(oversized_fs.read_dir("/large").await.unwrap().len(), 4097);
- assert_eq!(oversized_probe.read_dir_count(), 2);
-}
-
-#[tokio::test]
-async fn all_directory_membership_mutations_invalidate_parent_entries() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/root", 0o755).await.unwrap();
- backend.mkdir("/root/old", 0o755).await.unwrap();
- backend
- .write("/root/file.txt", b"file", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);
- assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);
-
- fs.mkdir("/root/new", 0o755).await.unwrap();
- assert_eq!(fs.read_dir("/root").await.unwrap().len(), 3);
-
- fs.rename("/root/old", "/root/moved").await.unwrap();
- let names: Vec<String> = fs
- .read_dir("/root")
- .await
- .unwrap()
- .into_iter()
- .map(|entry| entry.name)
- .collect();
- assert!(names.contains(&"moved".to_string()));
- assert!(!names.contains(&"old".to_string()));
-
- fs.remove("/root/file.txt").await.unwrap();
- assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);
-
- fs.remove_all("/root/new").await.unwrap();
- assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1);
- assert_eq!(probe.read_dir_count(), 5);
-}
-
-#[tokio::test]
-async fn writes_and_deletes_never_leave_stale_file_cache_entries() {
- let backend = CountingFileSystem::new();
- backend
- .write("/value.txt", b"old", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"old");
- fs.write("/value.txt", b"new", 0, WriteFlag::Truncate)
- .await
- .unwrap();
- assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"new");
-
- fs.remove("/value.txt").await.unwrap();
- assert!(fs.read("/value.txt", 0, 0).await.is_err());
- assert_eq!(probe.read_count(), 2);
-}
-
-#[tokio::test]
-async fn full_writes_populate_cache_before_returning() {
- let backend = CountingFileSystem::new();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- fs.write("/fresh.md", b"fresh", 0, WriteFlag::Create)
- .await
- .unwrap();
- assert_eq!(fs.read("/fresh.md", 0, 0).await.unwrap(), b"fresh");
- assert_eq!(probe.read_count(), 0);
-}
-
-#[tokio::test]
-async fn partial_writes_and_truncate_invalidate_cached_file_contents() {
- let backend = CountingFileSystem::new();
- backend
- .write("/partial.txt", b"hello", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hello");
- fs.write("/partial.txt", b"X", 1, WriteFlag::None)
- .await
- .unwrap();
- assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hXllo");
-
- fs.truncate("/partial.txt", 2).await.unwrap();
- assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hX");
- assert_eq!(probe.read_count(), 3);
-}
-
-#[tokio::test]
-async fn file_rename_invalidates_old_and_new_paths() {
- let backend = CountingFileSystem::new();
- backend
- .write("/old.txt", b"old", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/new.txt", b"historical", 0, WriteFlag::Create)
- .await
- .unwrap();
- let direct = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- assert_eq!(fs.read("/old.txt", 0, 0).await.unwrap(), b"old");
- assert_eq!(fs.read("/new.txt", 0, 0).await.unwrap(), b"historical");
- direct.remove("/new.txt").await.unwrap();
-
- fs.rename("/old.txt", "/new.txt").await.unwrap();
-
- assert!(fs.read("/old.txt", 0, 0).await.is_err());
- assert_eq!(fs.read("/new.txt", 0, 0).await.unwrap(), b"old");
-}
-
-#[tokio::test]
-async fn remove_all_generation_rejects_residual_descendant_cache_entries() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/tree", 0o755).await.unwrap();
- backend
- .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create)
- .await
- .unwrap();
- let direct = backend.clone();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- assert_eq!(fs.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old");
- fs.remove_all("/tree").await.unwrap();
-
- direct.mkdir("/tree", 0o755).await.unwrap();
- direct
- .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create)
- .await
- .unwrap();
-
- assert_eq!(fs.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"new");
- assert_eq!(probe.read_count(), 2);
-}
-
-#[tokio::test]
-async fn remove_all_error_invalidates_cached_state_after_partial_mutation() {
- let backend = CountingFileSystem::new().with_partial_remove_failure("/tree/deleted.txt");
- backend.mkdir("/tree", 0o755).await.unwrap();
- backend
- .write("/tree/deleted.txt", b"deleted", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/tree/survivor.txt", b"survivor", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend
- .write("/unrelated.txt", b"unrelated", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- assert_eq!(
- fs.read("/tree/deleted.txt", 0, 0).await.unwrap(),
- b"deleted"
- );
- assert_eq!(
- fs.read("/tree/survivor.txt", 0, 0).await.unwrap(),
- b"survivor"
- );
- assert_eq!(fs.read("/unrelated.txt", 0, 0).await.unwrap(), b"unrelated");
- assert_eq!(fs.read_dir("/tree").await.unwrap().len(), 2);
-
- let error = fs.remove_all("/tree").await.unwrap_err();
- assert!(error.to_string().contains("partially removed"));
-
- assert!(fs.read("/tree/deleted.txt", 0, 0).await.is_err());
- assert_eq!(
- fs.read("/tree/survivor.txt", 0, 0).await.unwrap(),
- b"survivor"
- );
- assert_eq!(fs.read("/unrelated.txt", 0, 0).await.unwrap(), b"unrelated");
- let entries = fs.read_dir("/tree").await.unwrap();
- assert_eq!(entries.len(), 1);
- assert_eq!(entries[0].name, "survivor.txt");
- assert_eq!(probe.read_count(), 5);
- assert_eq!(probe.read_dir_count(), 2);
-}
-
-#[tokio::test]
-async fn shared_provider_generation_bump_invalidates_other_wrappers() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/tree", 0o755).await.unwrap();
- backend
- .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create)
- .await
- .unwrap();
- let direct = backend.clone();
- let probe = backend.clone();
- let runtime = CacheRuntime::memory();
-
- let first = CachedFileSystem::with_runtime(
- Box::new(backend.clone()),
- runtime.clone(),
- CacheNamespace::new("shared"),
- CachePolicy::default(),
- );
- let second = CachedFileSystem::with_runtime(
- Box::new(backend),
- runtime,
- CacheNamespace::new("shared"),
- CachePolicy::default(),
- );
-
- assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old");
- second.remove_all("/tree").await.unwrap();
-
- direct.mkdir("/tree", 0o755).await.unwrap();
- direct
- .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create)
- .await
- .unwrap();
-
- assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"new");
- assert_eq!(probe.read_count(), 2);
-}
-
-#[tokio::test]
-async fn provider_generation_eviction_after_restart_cannot_revive_old_descendants() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/tree", 0o755).await.unwrap();
- backend
- .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create)
- .await
- .unwrap();
- let direct = backend.clone();
- let probe = backend.clone();
- let provider = Arc::new(MemoryMockProvider::new());
- let first_runtime = CacheRuntime::memory_with_provider(provider.clone());
-
- let first = CachedFileSystem::with_runtime(
- Box::new(backend.clone()),
- first_runtime.clone(),
- CacheNamespace::new("restart"),
- CachePolicy::default(),
- );
- assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old");
- first.remove_all("/tree").await.unwrap();
-
- direct.mkdir("/tree", 0o755).await.unwrap();
- direct
- .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create)
- .await
- .unwrap();
-
- for key in provider.keys().await {
- if key.contains(":subtree:") {
- first_runtime.del(&[key]).await.unwrap();
- }
- }
- drop(first);
- drop(first_runtime);
-
- let restarted = CachedFileSystem::with_runtime(
- Box::new(backend),
- CacheRuntime::memory_with_provider(provider),
- CacheNamespace::new("restart"),
- CachePolicy::default(),
- );
- assert_eq!(
- restarted.read("/tree/leaf.txt", 0, 0).await.unwrap(),
- b"new"
- );
- assert_eq!(probe.read_count(), 2);
-}
-
-#[tokio::test]
-async fn directory_rename_invalidates_old_and_historical_new_subtrees() {
- let backend = CountingFileSystem::new();
- backend.mkdir("/old", 0o755).await.unwrap();
- backend
- .write("/old/leaf.txt", b"moved", 0, WriteFlag::Create)
- .await
- .unwrap();
- let direct = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- assert_eq!(fs.read("/old/leaf.txt", 0, 0).await.unwrap(), b"moved");
- fs.rename("/old", "/new").await.unwrap();
- assert!(fs.read("/old/leaf.txt", 0, 0).await.is_err());
- assert_eq!(fs.read("/new/leaf.txt", 0, 0).await.unwrap(), b"moved");
-
- fs.remove_all("/new").await.unwrap();
- direct.mkdir("/new", 0o755).await.unwrap();
- direct
- .write("/new/leaf.txt", b"replacement", 0, WriteFlag::Create)
- .await
- .unwrap();
- assert_eq!(
- fs.read("/new/leaf.txt", 0, 0).await.unwrap(),
- b"replacement"
- );
-}
-
-#[tokio::test]
-async fn concurrent_misses_share_one_backend_read() {
- let backend = CountingFileSystem::new().with_read_delay(Duration::from_millis(30));
- backend
- .write("/hot.md", b"hot", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- let mut tasks = Vec::new();
- for _ in 0..12 {
- let fs = fs.clone();
- tasks.push(tokio::spawn(async move {
- fs.read("/hot.md", 0, 0).await.unwrap()
- }));
- }
- for task in tasks {
- assert_eq!(task.await.unwrap(), b"hot");
- }
-
- assert_eq!(probe.read_count(), 1);
- let metrics = fs.metrics().snapshot();
- assert_eq!(metrics.inflight_leaders, 1);
- assert_eq!(metrics.inflight_followers, 11);
- assert_eq!(metrics.inflight_backend_saved, 11);
-}
-
-#[tokio::test]
-async fn cache_policy_bypasses_lock_and_control_files() {
- let backend = CountingFileSystem::new();
- backend
- .write("/state.lock", b"first", 0, WriteFlag::Create)
- .await
- .unwrap();
- backend.mkdir("/queue", 0o755).await.unwrap();
- backend
- .write("/queue/peek", b"live", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let (fs, _) = cached_fs(backend);
-
- fs.read("/state.lock", 0, 0).await.unwrap();
- fs.read("/state.lock", 0, 0).await.unwrap();
- fs.read("/queue/peek", 0, 0).await.unwrap();
- fs.read("/queue/peek", 0, 0).await.unwrap();
-
- assert_eq!(probe.read_count(), 4);
- assert_eq!(fs.metrics().snapshot().policy_bypasses, 4);
-}
-
-#[tokio::test]
-async fn failed_invalidation_bypasses_cache_instead_of_serving_stale_data() {
- let backend = CountingFileSystem::new();
- backend
- .write("/value.txt", b"old", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let provider = Arc::new(MemoryMockProvider::new());
- provider.set_delete_failure(true);
- let fs = CachedFileSystem::with_runtime(
- Box::new(backend),
- CacheRuntime::memory_with_provider(provider),
- CacheNamespace::new("delete-failure"),
- CachePolicy::default(),
- );
-
- assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"old");
- fs.write("/value.txt", b"new", 0, WriteFlag::Truncate)
- .await
- .unwrap();
- assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"new");
- assert_eq!(probe.read_count(), 2);
-
- let metrics = fs.metrics().snapshot();
- assert!(metrics.errors >= 1);
- assert!(metrics.policy_bypasses >= 1);
-}
-
-#[tokio::test]
-async fn unavailable_provider_falls_back_to_backend_and_enters_bypass() {
- let backend = CountingFileSystem::new();
- backend
- .write("/available.txt", b"backend", 0, WriteFlag::Create)
- .await
- .unwrap();
- let probe = backend.clone();
- let provider = Arc::new(MemoryMockProvider::new());
- provider.set_unavailable(true);
- let fs = CachedFileSystem::with_runtime(
- Box::new(backend),
- CacheRuntime::memory_with_provider(provider),
- CacheNamespace::new("unavailable"),
- CachePolicy::default(),
- );
-
- assert_eq!(fs.read("/available.txt", 0, 0).await.unwrap(), b"backend");
- assert_eq!(fs.read("/available.txt", 0, 0).await.unwrap(), b"backend");
- assert_eq!(probe.read_count(), 2);
-
- let metrics = fs.metrics().snapshot();
- assert!(metrics.errors >= 2);
- assert!(metrics.policy_bypasses >= 1);
-}
-
-#[tokio::test]
-async fn generation_backfill_is_bounded_and_continues_after_one_set_failure() {
- let backend = CountingFileSystem::new();
- let mut current = String::new();
- for component in ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l"] {
- current.push('/');
- current.push_str(component);
- backend.mkdir(&current, 0o755).await.unwrap();
- }
- let path = "/a/b/c/d/e/f/g/h/i/j/k/l/file.md";
- backend
- .write(path, b"generation", 0, WriteFlag::Create)
- .await
- .unwrap();
-
- let provider = Arc::new(MemoryMockProvider::new().with_set_delay(Duration::from_millis(20)));
- provider.fail_next_set_matching(":subtree:");
- let fs = CachedFileSystem::with_runtime(
- Box::new(backend),
- CacheRuntime::memory_with_provider(Arc::clone(&provider)),
- CacheNamespace::new("generation-set-failure"),
- CachePolicy::default(),
- );
-
- assert_eq!(fs.read(path, 0, 0).await.unwrap(), b"generation");
-
- let keys = provider.keys().await;
- assert_eq!(
- keys.iter().filter(|key| key.contains(":subtree:")).count(),
- 13,
- "one of the fourteen ancestor generation writes should fail"
- );
- assert_eq!(
- keys.iter().filter(|key| key.contains(":file:")).count(),
- 1,
- "a generation write failure must not prevent the file cache fill"
- );
- assert_eq!(fs.metrics().snapshot().errors, 1);
- assert!(
- (2..=8).contains(&provider.max_concurrent_sets()),
- "generation writes should run concurrently with a fixed upper bound"
- );
-}
-
-#[tokio::test]
-async fn metrics_cover_operations_bytes_latency_and_errors() {
- let backend = CountingFileSystem::new();
- backend
- .write("/metrics.txt", b"metrics", 0, WriteFlag::Create)
- .await
- .unwrap();
- let (fs, _) = cached_fs(backend);
-
- assert_eq!(fs.read("/metrics.txt", 0, 0).await.unwrap(), b"metrics");
- assert_eq!(fs.read("/metrics.txt", 0, 0).await.unwrap(), b"metrics");
- fs.write("/metrics.txt", b"updated", 0, WriteFlag::Truncate)
- .await
- .unwrap();
-
- let metrics = fs.metrics().snapshot();
- assert_eq!(metrics.file_hits, 1);
- assert_eq!(metrics.file_misses, 1);
- assert_eq!(metrics.backend_fallbacks, 1);
- assert!(metrics.puts >= 1);
- assert!(metrics.deletes >= 1);
- assert!(metrics.invalidations >= 1);
- assert_eq!(metrics.backend_bytes, 7);
- assert_eq!(metrics.cache_bytes, 7);
- assert!(metrics.get_latency_ns > 0);
- assert!(metrics.put_latency_ns > 0);
- assert!(metrics.delete_latency_ns > 0);
- assert_eq!(metrics.errors, 0);
-}
diff --git a/docs/en/api/01-overview.md b/docs/en/api/01-overview.md
index 79c406d96..f4603bf3b 100644
--- a/docs/en/api/01-overview.md
+++ b/docs/en/api/01-overview.md
@@ -359,10 +359,6 @@ This catalog follows the routes actually mounted by the server. Each group headi
| GET | `/api/v1/system/status` | System status |
| POST | `/api/v1/system/wait` | Wait for background processing |
| POST | `/api/v1/system/consistency` | Check filesystem and vector-index consistency |
-| POST | `/api/v1/system/backend/sync-status` | Query backend synchronization status |
-| POST | `/api/v1/system/backend/sync-retry` | Retry backend synchronization |
-| GET | `/api/v1/system/sync/{sync_path}` | Path-form compatibility endpoint for synchronization status |
-| POST | `/api/v1/system/sync/{sync_path}/retry` | Path-form compatibility endpoint for synchronization retry |
### [Resources](02-resources.md) and [Filesystem](03-filesystem.md)
diff --git a/docs/en/api/07-system.md b/docs/en/api/07-system.md
index 78a175fcc..d87aefa60 100644
--- a/docs/en/api/07-system.md
+++ b/docs/en/api/07-system.md
@@ -1,6 +1,8 @@
# System Status
-The OpenViking System API provides health, readiness, consistency, and multi-write backend synchronization status. Component observers and Prometheus metrics are documented separately.
+The OpenViking System API provides health, readiness, consistency, and
+background-processing status. Component observers and Prometheus metrics are
+documented separately.
## API Reference
@@ -417,99 +419,6 @@ ov system wait --timeout 60
---
-### backend_sync_status()
-
-Return multi-write backend synchronization status for a Viking URI subtree. This endpoint requires ROOT or ADMIN permission.
-
-**HTTP API**
-
-```http
-POST /api/v1/system/backend/sync-status
-Content-Type: application/json
-```
-
-```bash
-curl -X POST http://localhost:1933/api/v1/system/backend/sync-status \
- -H "Content-Type: application/json" \
- -H "X-API-Key: your-admin-key" \
- -d '{"uri":"viking://resources"}'
-```
-
-The URI-path form is also available:
-
-```http
-GET /api/v1/system/sync/{sync_path}
-```
-
-**CLI**
-
-```bash
-ov system backend sync-status viking://resources
-```
-
-**Response example**
-
-```json
-{
- "status": "ok",
- "result": {
- "path": "viking://resources",
- "entry_count": 12
- }
-}
-```
-
-`result` is supplied by the active filesystem backend. `path` identifies the queried scope and `entry_count` is the number of sync records in that scope. A backend may add diagnostics such as pending or failed records.
-
-### backend_sync_retry()
-
-Retry incomplete multi-write backend synchronization work for a URI subtree. This endpoint requires ROOT or ADMIN permission.
-
-**HTTP API**
-
-```http
-POST /api/v1/system/backend/sync-retry
-Content-Type: application/json
-```
-
-```bash
-curl -X POST http://localhost:1933/api/v1/system/backend/sync-retry \
- -H "Content-Type: application/json" \
- -H "X-API-Key: your-admin-key" \
- -d '{"uri":"viking://resources"}'
-```
-
-The URI-path form is:
-
-```http
-POST /api/v1/system/sync/{sync_path}/retry
-```
-
-**CLI**
-
-```bash
-ov system backend sync-retry viking://resources
-```
-
-**Response example**
-
-```json
-{
- "status": "ok",
- "result": {
- "path": "viking://resources",
- "retried": 2,
- "failed": 0
- }
-}
-```
-
-`retried` is the number of records rescheduled by this request, and `failed` is the number that could not be scheduled. A backend may include additional diagnostic fields.
-
-The public Python, TypeScript, and Go SDKs do not currently expose multi-write backend synchronization methods, so the sections above show only HTTP and CLI tabs.
-
----
-
<a id="reindex"></a><a id="observer-api"></a>
## Related Documentation
diff --git a/docs/en/api/20-webdav.md b/docs/en/api/20-webdav.md
index 161ba6d19..9526ff464 100644
--- a/docs/en/api/20-webdav.md
+++ b/docs/en/api/20-webdav.md
@@ -17,7 +17,7 @@ The adapter supports the following scope:
- Resources only. Memories, skills, sessions, and other namespaces are not exposed.
- Text-first writes. `PUT` currently accepts UTF-8 text content only.
- WebDAV subset only. `OPTIONS`, `PROPFIND`, `GET`, `HEAD`, `PUT`, `DELETE`, `MKCOL`, and `MOVE` are supported.
-- Semantic sidecars and internal system files stay internal. Derived or internal files such as `.abstract.md`, `.overview.md`, `.path.ovlock`, `.redirect.json`, and `.sync_log.json` are hidden from listings and cannot be accessed directly through WebDAV.
+- Semantic sidecars and internal system files stay internal. Derived or internal files such as `.abstract.md`, `.overview.md`, `.path.ovlock`, and multi-write metadata are hidden from listings and cannot be accessed directly through WebDAV.
Behavior notes:
@@ -25,7 +25,7 @@ Behavior notes:
- Replacing an existing file through WebDAV refreshes related semantics and vectors, same as `write()`.
- `PUT` does not create parent collections automatically. Create missing directories with `MKCOL` first.
- User-created dot-directories and dot-files remain visible unless they match one of the reserved internal filenames above.
-- When multi-write storage is enabled, files redirected to a backup are still exposed through the filesystem APIs as normal files; internal redirect and sync metadata never become visible to callers.
+- With multi-write enabled, WebDAV reads use the primary. Backup replication is asynchronous.
## API Reference
diff --git a/docs/en/concepts/05-storage.md b/docs/en/concepts/05-storage.md
index ccedc225e..b117329df 100644
--- a/docs/en/concepts/05-storage.md
+++ b/docs/en/concepts/05-storage.md
@@ -54,10 +54,10 @@ AGFS provides POSIX-style file operations with multiple backend support.
By default, AGFS uses a single backend for content storage. Once `storage.agfs.backups` is configured, OpenViking enters multi-write mode:
-- Top-level `storage.agfs.backend` is the primary backend and remains the authoritative write target.
-- `storage.agfs.backups.items[]` defines backup backends for replicas, migration, or read acceleration.
+- Top-level `storage.agfs.backend` is the primary backend and the only direct read source.
+- `storage.agfs.backups.items[]` defines asynchronously updated backup backends.
- The Python SDK, HTTP API, and CLI filesystem interfaces stay unchanged.
-- Multi-write uses `.redirect.json` and `.sync_log.json` internally to track redirect mappings and sync progress. These files are not visible to users.
+- V2 metadata is partitioned and stored only on the primary.
For the conceptual model, see [Multi-Write Storage](./14-multi-write-storage.md). For examples, see the [Multi-Write Storage Guide](../guides/13-multi-write-storage.md).
@@ -161,5 +161,5 @@ client.mv(
- [Architecture Overview](./01-architecture.md) - System architecture
- [Context Layers](./03-context-layers.md) - L0/L1/L2 model
- [Viking URI](./04-viking-uri.md) - URI specification
-- [Multi-Write Storage](./14-multi-write-storage.md) - Primary/backup roles, routing, and consistency
+- [Multi-Write Storage](./14-multi-write-storage.md) - V2 primary and backup consistency
- [Retrieval Mechanism](./07-retrieval.md) - Retrieval process details
diff --git a/docs/en/concepts/12-metrics.md b/docs/en/concepts/12-metrics.md
index b6da7c6bb..224b96a9f 100644
--- a/docs/en/concepts/12-metrics.md
+++ b/docs/en/concepts/12-metrics.md
@@ -289,8 +289,12 @@ seconds, for example `123 ns = 0.000000123 seconds`.
| `openviking_ragfs_cache_bytes_total` | Counter | `source` | bytes from backend/cache |
| `openviking_ragfs_cache_operation_duration_seconds_total` | Counter | `operation` | cumulative get/put/delete duration |
| `openviking_ragfs_cache_inflight_events_total` | Counter | `event` | leader/follower/backend_saved counts |
-| `openviking_ragfs_multiwrite_background_tasks` | Gauge | none | background tasks, including the retry loop |
-| `openviking_ragfs_multiwrite_read_routes_total` | Counter | `route` | primary/backup/redirect/miss selections |
+| `openviking_ragfs_multiwrite_background_tasks` | Gauge | none | active flush runtime, `0` or `1` per mount |
+| `openviking_ragfs_multiwrite_pending_events` | Gauge | none | submitted events that are queued, executing, or awaiting retry |
+| `openviking_ragfs_multiwrite_errors_total` | Counter | `worker` | final failures for `flush`, `checkpoint`, `catch_up`, or `gc` |
+| `openviking_ragfs_multiwrite_backend_lag_events` | Gauge | `backend, partition` | sum of `tail - synced_seq` across accounts |
+| `openviking_ragfs_multiwrite_protocol_version` | Gauge | `version` | mount count for the closed label `v2` |
+| `openviking_ragfs_multiwrite_protocol_status` | Gauge | `status` | mount count for closed labels `migrating` and `stable` |
These metrics have no `mount` or `account_id` label. Cache and multi-backend
families are absent when those capabilities are not enabled. `status` is
@@ -298,6 +302,10 @@ families are absent when those capabilities are not enabled. `status` is
Histogram buckets cover all outcomes, with finite bounds from 0.0001 to 10 seconds.
Python `get_stats()` retains its microsecond fields.
+The protocol family always contains both closed-label series. One unreadable
+or invalid protocol, account, or segment manifest rejects the
+whole scrape snapshot instead of publishing partial zero lag.
+
### Tasks and Task Tracker
| Metric Family | Type | Common Labels | Meaning |
diff --git a/docs/en/concepts/14-multi-write-storage.md b/docs/en/concepts/14-multi-write-storage.md
index b5ffd735b..356cab111 100644
--- a/docs/en/concepts/14-multi-write-storage.md
+++ b/docs/en/concepts/14-multi-write-storage.md
@@ -1,134 +1,51 @@
# Multi-Write Storage
-Multi-write storage lets OpenViking use one primary storage backend together with multiple backup backends under a unified filesystem abstraction. It supports replicas (including cross-region replicas), configurable read routing, and storage migration; it does not automatically promote a backup to primary.
+V2 multi-write keeps one primary backend as the business source of truth and
+replicates its files to configured backups.
-From the API user's point of view, interfaces such as `read()`, `write()`, `ls()`, and `stat()` do not change. Multi-write logic lives inside RAGFS, so callers do not need to care which underlying backend ultimately stores a file.
+## Read And Write Model
-## Core Model
+- All direct reads use the primary. Backups never serve application reads.
+- A successful primary mutation returns immediately.
+- Metadata persistence and backup application continue asynchronously.
+- Backup lag does not change the result already returned for the primary.
+- Multi-write metadata is stored only on the primary.
-Multi-write storage consists of one primary backend and multiple backup backends:
+This model is eventually consistent. A process crash between primary success
+and event persistence can lose that replication event. Rebuild affected
+backups from the primary after such a crash.
-| Role | Configuration | Description |
-| --- | --- | --- |
-| primary | `storage.agfs.backend` | Authoritative write target and final read fallback |
-| backup | `storage.agfs.backups.items[]` | Receives replicated writes and may optionally participate in reads |
+## Metadata Model
-If `backups` is not configured, OpenViking continues to use the original single-backend mode.
+Each account has a fixed set of metadata partitions. Events are routed to one
+partition, appended to a segment log, and consumed independently by each
+backup. Checkpoints compact older history, and garbage collection removes
+metadata that is no longer required.
-## Write Path
+`initial_partitions` controls the partition count when an account's metadata is
+created. Existing accounts keep their persisted partition count and routes when
+the configuration changes. Partition expansion and shrinking are not supported
+in this release.
-By default, writes land on the primary backend first and are then replicated to write-enabled backup backends.
+## First Enable Import
-```text
-Client
- -> OpenViking API
- -> RAGFS MultiWrite
- -> primary
- -> backup1 / backup2 / ...
-```
+When `.multiwrite.json` is absent, startup creates it as `V2/migrating`,
+starts foreground V2 event submission, and imports the current primary files
+in the background. Successful import changes the status to `stable`. Extra
+old files on backups are not removed.
-If a backup does not define `operations`, it participates in writes by default. This keeps cold-backup setup simple.
+Old multi-write protocols and old migration fields are rejected. They are not
+converted or interpreted.
-## Sync Modes
+## Operations
-Multi-write supports two consistency modes.
-
-| Mode | Config value | Behavior | Suitable for |
-| --- | --- | --- | --- |
-| Async multi-write | `async` | Return as soon as the primary write succeeds; backups sync in the background | Low-latency writes, eventual consistency |
-| Sync multi-write | `sync` | Wait for backup acknowledgements after the primary write succeeds | Stronger write confirmation when extra latency is acceptable |
-
-In async mode, backups may lag behind the primary; recovery depends on backend availability and retry outcomes. In sync mode, `write_ack_count` and `write_ack_timeout_ms` control how many backup acknowledgements are required and how long the system waits.
-
-If sync mode fails to reach the required acknowledgement count, the call returns an error even though the primary may already have committed; that write is not automatically rolled back. Backups that did not acknowledge are still retried in the background; to inspect sync state, see the [Multi-Write Storage Guide](../guides/13-multi-write-storage.md).
-
-## Read Path
-
-Reads do not hit every backup by default. Only backups that explicitly declare the `read` operation join the read route.
-
-Read order:
-
-```text
-1. Read-enabled backups in ascending priority order
-2. Fallback to the primary backend
-3. If the file is redirected, access the redirect target
-4. Return NotFound if the file is still missing
-```
-
-This avoids having cold-backup nodes participate in reads by default and reduces the risk of serving stale data.
-
-## Redirect
-
-Redirect means "certain files are not written to the primary backend and are written to a specified backup instead."
-
-Common cases:
-
-- Large files go to object storage.
-- Specific file extensions go to a dedicated backend.
-- The primary backend keeps standard content, while special files live elsewhere.
-
-Redirect policies are configured on the primary backend. When a file matches a policy, OpenViking records the mapping in internal metadata. Calls such as `ls()`, `stat()`, and `read()` still expose a normal filesystem view.
-
-## Exclude
-
-Exclude means "a specific backup backend does not receive matching files."
-
-Common cases:
-
-- A memory or cache backend should not keep large files.
-- One backup only stores text resources.
-- A lower-cost backend excludes temporary or oversized files.
-
-Exclude policies are configured on each backup and affect only whether that backup receives a write.
-
-## Internal Metadata
-
-Multi-write uses two internal metadata files:
-
-| File | Purpose |
-| --- | --- |
-| `.redirect.json` | Records which backend stores each redirected file |
-| `.sync_log.json` | Records per-file sync version and backup acknowledgement progress |
-
-These files are hidden from normal users, do not appear in standard directory listings, and should not be read or written through public APIs.
-
-If the primary backend enables at-rest encryption, these internal metadata files follow the same encryption policy.
-
-## Encryption Relationship
-
-Multi-write does not change OpenViking's transparent encryption model.
-
-Rules:
-
-- The Python layer and public APIs stay unaware of encryption details.
-- The primary backend must be encrypted when global encryption is enabled.
-- Each backup may decide independently whether to enable encryption.
-- Internal metadata must go through the primary backend's encryption path.
-
-That means enabling multi-write does not change how clients call the system; encryption remains a configuration concern per backend.
-
-## Relationship with OVPack
-
-Multi-write only handles new writes after it is enabled. It does not automatically sync historical files that already exist in the primary backend.
-
-Recommended migration flow:
-
-1. Pause writes or arrange a verifiable incremental catch-up window, then migrate existing data with OVPack or another controlled process.
-2. Validate the target backend data.
-3. Enable the multi-write configuration.
-4. Catch up changes made during migration and verify sync state before resuming writes; multi-write replicates subsequent additions and updates.
-
-## Limitations
-
-- In async mode, backups may lag; reads from them cannot assume the latest write is visible.
-- Historical files that existed before multi-write was enabled need a separate migration or backfill process.
-- Redirected files rely on internal metadata to reconstruct the directory view.
-- Concurrent writes from multiple processes to the same primary backend still need future distributed metadata locking.
-- Hot directories may update internal metadata frequently and can introduce additional write amplification.
+V2 does not expose synchronization status or manual retry APIs or CLI
+commands. Use the multi-write Prometheus metrics to observe queue depth,
+worker failures, backup lag, protocol version, and protocol status.
## Related Documents
- [Storage Architecture](./05-storage.md)
+- [Metrics](./12-metrics.md)
- [Configuration Guide](../guides/01-configuration.md)
- [Multi-Write Storage Guide](../guides/13-multi-write-storage.md)
-- [OVPack Import and Export](../guides/09-ovpack.md)
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index 0d47386b3..70ac61e23 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -1300,7 +1300,6 @@ Storage configuration for context data, including file storage (RAGFS) and vecto
| `backend` | str | `"local"`, `"s3"`, or `"memory"` | `"local"` |
| `timeout` | float | Request timeout in seconds | `10.0` |
| `backups` | object | Multi-write storage configuration. When set, the top-level `backend` acts as the primary backend and `backups.items[]` defines backup backends | `null` |
-| `redirects` | array | File redirect policies for multi-write storage. Matching files are written to the specified backup instead of the primary backend | `[]` |
| `queuefs` | object | QueueFS configuration. Controls the namespace mode, backend, and runtime options for `/queue` | `{ "mode": "shared", "backend": "sqlite", "recover_stale_sec": 0, "busy_timeout_ms": 5000 }` |
| `queue_db_path` | str (optional) | Legacy compatibility field for QueueFS sqlite DB path. Superseded by `storage.agfs.queuefs.db_path`. Defaults to `{storage.workspace}/_system/queue/queue.db` when not set. Useful when the workspace volume does not support sqlite (e.g. some network filesystems) | `null` |
| `s3` | object | S3 backend configuration (when backend is 's3') | - |
@@ -1314,7 +1313,8 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro
##### Multi-Write Storage Configuration
-`storage.agfs.backups` enables multi-write storage. If it is not configured, OpenViking stays in single-backend mode.
+`storage.agfs.backups` enables asynchronous V2 multi-write storage. The
+top-level backend remains the only direct read source.
```json
{
@@ -1322,26 +1322,16 @@ RAGFS uses Rust binding mode by default, directly accessing the file system thro
"workspace": "./data",
"agfs": {
"backend": "local",
- "redirects": [
- {
- "type": "FileExtensionPolicy",
- "extensions": ["(pdf|ppt|zip)"],
- "target": ["s3-backup"]
- }
- ],
"backups": {
- "sync_type": "async",
+ "initial_partitions": 16,
+ "checkpoint_interval_secs": 86400,
+ "provider": "filesystem",
"items": [
{
- "name": "s3-backup",
- "backend": "s3",
- "s3": {
- "bucket": "openviking-backup",
- "region": "cn-beijing",
- "endpoint": "https://tos-s3-cn-beijing.volces.com",
- "access_key": "your-ak",
- "secret_key": "your-sk",
- "prefix": "multi-write"
+ "name": "local-backup",
+ "backend": "local",
+ "params": {
+ "workspace": "./data/backup"
}
}
]
@@ -1355,36 +1345,14 @@ Common `backups` fields:
| Parameter | Type | Description | Default |
|-----------|------|-------------|---------|
-| `sync_type` | str | Multi-write sync mode. Supports `"async"` or `"sync"` | `"async"` |
-| `write_ack_count` | int | Number of backup acknowledgements required before a `sync` write returns | all backups |
-| `write_ack_timeout_ms` | int | Timeout in milliseconds while waiting for backup acknowledgements in `sync` mode | `null` |
-| `write_concurrency` | int | Maximum async backup write concurrency | `null` |
-| `items` | array | Backup backend list. Each item reuses normal backend configuration and adds fields such as `name`, `operations`, `excludes`, and `encryption` | `[]` |
-
-Common `redirects` fields:
-
-| Parameter | Type | Description | Default |
-|-----------|------|-------------|---------|
-| `type` | str | Policy type. Supports `"FileExtensionPolicy"` or `"FileOverSizePolicy"` | required |
-| `extensions` | array | Extension regex list used by `FileExtensionPolicy`, for example `["(pdf\\|ppt)"]` | `[]` |
-| `max_size_mb` | int | File size threshold in MB used by `FileOverSizePolicy` | `null` |
-| `target` | array | Backup `name` list that receives matched files | required |
-
-File-size redirect example:
-
-```json
-{
- "type": "FileOverSizePolicy",
- "max_size_mb": 100,
- "target": ["s3-backup"]
-}
-```
-
-Notes:
+| `initial_partitions` | int | Initial metadata partitions; changes do not affect existing accounts | `16` |
+| `checkpoint_interval_secs` | int | Checkpoint discovery interval; minimum `60` | `86400` |
+| `provider` | str | Metadata provider: `filesystem` or top-level CacheRuntime-backed `cache` | `"filesystem"` |
+| `items` | array | Backups with `name`, `backend`, `params`, and optional `encryption` | `[]` |
-- `redirects` is configured at top-level `storage.agfs` and defines redirect policies for the primary backend.
-- `target` must reference an existing backup `name` from `backups.items[]`.
-- Files matched by redirect still appear as normal readable and listable files through the filesystem APIs.
+The Python configuration layer still accepts legacy sync, retry, operations,
+excludes, and redirects fields. It warns and removes them before calling Rust;
+the fields have no effect.
See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more examples.
@@ -1452,9 +1420,9 @@ The top-level `cache` section is a sibling of `storage`. Its public shape is Pro
}
```
-The canonical configuration has no global `cache.enabled`. CacheRuntime is initialized when CacheFS or QueueFS selects `backend=cache`, or when PathLock selects `provider=cache`. Cache-backed PathLock currently requires `cache.provider=redis`; DynamicProvider is not supported for PathLock. When all modules use local providers, `cache.params` is not parsed and no Provider connection is opened.
+The canonical configuration has no global `cache.enabled`. CacheRuntime is initialized when CacheFS or QueueFS selects `backend=cache`, PathLock selects `provider=cache`, or multi-write selects `backups.provider=cache`. Cache-backed PathLock currently requires `cache.provider=redis`; DynamicProvider is not supported for PathLock. When all modules use local providers, `cache.params` is not parsed and no Provider connection is opened.
-This is a breaking configuration change. `storage.agfs.cache`, `storage.agfs.queuefs.backend="redis"`, and `storage.agfs.queuefs.redis` are rejected. Move Provider settings to top-level `cache.provider/cache.params`, select `cachefs.backend="cache"` or `queuefs.backend="cache"`, use Redis `mode="standalone"` instead of `singleton`, and use `rediss://` instead of `tls_enabled`.
+This is a breaking configuration change. `storage.agfs.cache`, `storage.agfs.queuefs.backend="redis"`, `storage.agfs.queuefs.redis`, and `storage.agfs.backups.provider="redis"` are rejected. Move Provider settings to top-level `cache.provider/cache.params`, select `cachefs.backend="cache"`, `queuefs.backend="cache"`, or `backups.provider="cache"`, use Redis `mode="standalone"` instead of `singleton`, and use `rediss://` instead of `tls_enabled`.
##### QueueFS Configuration
diff --git a/docs/en/guides/08-encryption.md b/docs/en/guides/08-encryption.md
index c10ad8926..ea971ce81 100644
--- a/docs/en/guides/08-encryption.md
+++ b/docs/en/guides/08-encryption.md
@@ -21,7 +21,7 @@ Rules:
- When global `encryption.enabled=true`, the primary backend must be encrypted.
- Each backup backend may control its own encryption through `encryption.enabled`.
-- Multi-write internal metadata such as `.redirect.json` and `.sync_log.json` follows the primary backend's encryption policy.
+- V2 multi-write metadata is stored on the raw primary and bypasses `EncryptionWrappedFS`.
- OpenViking does not expose and does not need public encryption APIs for operating on these internal files.
See the [Multi-Write Storage Guide](./13-multi-write-storage.md) for more multi-write configuration details.
diff --git a/docs/en/guides/09-ovpack.md b/docs/en/guides/09-ovpack.md
index eab78f7a7..8b117348f 100644
--- a/docs/en/guides/09-ovpack.md
+++ b/docs/en/guides/09-ovpack.md
@@ -38,26 +38,13 @@ control files, and `.relations.json` are outside the OVPack migration scope.
## Working with Multi-Write Storage
-Multi-write storage only replicates writes that happen after it is enabled. It
-does not automatically copy historical files that already existed before
-`storage.agfs.backups` was turned on.
+V2 startup imports current V1 primary files into partitioned metadata and
+removes V1 sidecars. It does not remove extra old files from backups. Pause
+all instances and take a complete backup before the first V2 startup.
-To seed the primary and replicas together, enable multi-write on the **empty target environment before restoring** the package:
-
-1. Pause application writes and export or back up each source account.
-2. Configure the target primary and backup backends, including their write policies, then start the target server and provision its restore identity.
-3. Restore or import through that server so historical content passes through the configured write fanout.
-4. Check synchronization status for every restored scope and verify the expected files in each replica before switching traffic. Asynchronous replication can still be pending after restore returns.
-5. Resume writes after validating content and index integrity. Keep the source and backup until verification is complete.
-
-For example, using the target account's admin key:
-
-```bash
-ov system backend sync-status viking://resources
-ov system backend sync-status viking://user
-```
-
-Restoring first and enabling backups afterwards only replicates subsequent writes; it does not seed historical content into replicas.
+Once import starts, in-place rollback is unsupported. Restore the complete
+pre-import backup to return to V1. After startup, verify primary content with
+normal file APIs and monitor backup lag through Prometheus metrics.
For more details, see the [Multi-Write Storage Guide](./13-multi-write-storage.md).
diff --git a/docs/en/guides/13-multi-write-storage.md b/docs/en/guides/13-multi-write-storage.md
index 2c6c8f318..4a30c18ab 100644
--- a/docs/en/guides/13-multi-write-storage.md
+++ b/docs/en/guides/13-multi-write-storage.md
@@ -1,20 +1,10 @@
# Multi-Write Storage Guide
-This guide explains how to configure multi-write storage in OpenViking. Multi-write storage lets one primary backend replicate writes to multiple backup backends for replica storage (including cross-region replicas), optional read routing, and migration. It does not automatically promote a backup when the primary fails.
-
-Multi-write lives inside RAGFS. The Python SDK, HTTP API, and CLI usage remain unchanged.
-
-## Prerequisites
-
-- You already have a working `ov.conf`.
-- The primary backend has been verified to read and write correctly.
-- If you plan to use S3-compatible storage, prepare the bucket, endpoint, and access credentials first.
-- For existing data, follow the [OVPack multi-write migration workflow](./09-ovpack.md#working-with-multi-write-storage); enabling backups alone does not copy historical files.
+V2 multi-write returns after the primary succeeds and updates metadata and
+backups asynchronously. The primary is the only direct read source.
## Minimal Configuration
-Merge these excerpts into your existing model/server configuration. The following example uses a local directory as the primary backend and replicates writes to another local directory.
-
```json
{
"storage": {
@@ -22,13 +12,15 @@ Merge these excerpts into your existing model/server configuration. The followin
"agfs": {
"backend": "local",
"backups": {
- "sync_type": "async",
+ "initial_partitions": 16,
+ "checkpoint_interval_secs": 86400,
+ "provider": "filesystem",
"items": [
{
"name": "local-backup",
"backend": "local",
- "local": {
- "workspace": "./data/backup"
+ "params": {
+ "workspace": "./data/backup"
}
}
]
@@ -38,385 +30,62 @@ Merge these excerpts into your existing model/server configuration. The followin
}
```
-Notes:
-
-- The top-level `backend` is the primary backend.
-- `backups.items[]` is the backup backend list.
-- `name` is the stable identity of a backup; later sync metadata refers to it.
-- A backup with `backend = "local"` uses `local.workspace` to point to its local directory.
-- If `sync_type` is omitted, treat it as async by default.
-
-## Configuring Multiple Backups
+The three top-level V2 settings are:
-You can configure more than one backup. The following example writes to both a local replica and S3-compatible object storage.
-
-```json
-{
- "storage": {
- "workspace": "./data",
- "agfs": {
- "backend": "local",
- "backups": {
- "sync_type": "async",
- "items": [
- {
- "name": "local-az2",
- "backend": "local",
- "local": {
- "workspace": "./data/local-az2"
- }
- },
- {
- "name": "object-store",
- "backend": "s3",
- "s3": {
- "bucket": "openviking-backup",
- "region": "us-east-1",
- "endpoint": "https://s3.example.com",
- "access_key": "your-access-key",
- "secret_key": "your-secret-key",
- "prefix": "openviking",
- "directory_marker_mode": "none"
- }
- }
- ]
- }
- }
- }
-}
-```
-
-Recommendations:
-
-- Do not use unstable hostnames or temporary IDs for `name`.
-- The backup path or bucket should not point to the same physical location as the primary backend.
-- Changing a backup `name` affects historical sync metadata recognition, so treat that as a production change.
-
-### S3-Compatible Storage Notes
-
-Match these `s3` fields to the endpoint and object-store configuration:
-
-| Field | Required? | Description |
+| Field | Default | Meaning |
| --- | --- | --- |
-| `use_path_style` | Optional; default `true` | Use path-style URLs (`http://host/bucket/key`); select the style your endpoint supports. |
-| `directory_marker_mode` | Yes for backup items | `none` creates no marker, `empty` creates a zero-byte marker, and `nonempty` creates a marker with content. Set it explicitly in each S3 backup item; if omitted, startup fails with `AGFSConfigError: configuration error: invalid directory_marker_mode: null (valid: none, empty, nonempty)`. |
-| `use_ssl` | Optional | Set to `false` for HTTP endpoints (e.g. `http://localhost:9000`). |
-
-**Minimal S3-compatible example (RustFS/MinIO):**
-
-```json
-{
- "name": "s3-backup",
- "backend": "s3",
- "s3": {
- "bucket": "my-bucket",
- "endpoint": "http://localhost:9000",
- "access_key": "your-access-key",
- "secret_key": "your-secret-key",
- "prefix": "openviking",
- "use_ssl": false,
- "use_path_style": true,
- "directory_marker_mode": "none"
- }
-}
-```
-
-The example uses `none`, which suits S3-compatible services that do not use directory markers (RustFS, MinIO, Ceph, etc.). Backup items do not get a default for this field. The top-level `storage` S3 configuration defaults to `empty`.
-
-### Docker Networking for S3 Backup
-
-When running OpenViking in Docker with an S3 backup on the same host:
-
-- **Linux Docker**: Use `--network host` or the host's LAN IP. Docker bridge network can reach the host's LAN via gateway IP (e.g. `172.17.0.1:9000`).
-- **macOS/Windows Docker Desktop**: Use `host.docker.internal` to reach the host from the container. Host networking is available as an opt-in feature in Docker Desktop 4.34 and later; see [Docker host networking](https://docs.docker.com/engine/network/drivers/host/).
-
-If the server crashes silently on startup with S3 backup enabled, check the Docker networking first. The RAGFS Rust binding will produce `dispatch failure` if the S3 endpoint is unreachable from inside the container.
-
-## Choosing a Sync Mode
-
-### Async Mode
-
-Async mode fits most deployments.
-
-```json
-{
- "backups": {
- "sync_type": "async",
- "items": []
- }
-}
-```
-
-Characteristics:
-
-- Returns immediately after the primary write succeeds.
-- Backup writes run in the background.
-- Low write latency.
-- Lag has no fixed bound while a backend is unavailable; monitor synchronization and recovery.
-
-Suitable for:
-
-- Write throughput first.
-- Backups mainly used for disaster recovery.
-- Eventual consistency is acceptable.
-
-### Sync Mode
-
-Sync mode waits for backup acknowledgements.
-
-```json
-{
- "backups": {
- "sync_type": "sync",
- "write_ack_count": 1,
- "write_ack_timeout_ms": 5000,
- "items": [{"name": "local-backup", "backend": "local", "local": {"workspace": "./backup-data"}}]
- }
-}
-```
-
-Parameters:
-
-| Parameter | Description |
-| --- | --- |
-| `write_ack_count` | Minimum number of backup acknowledgements required before the write returns |
-| `write_ack_timeout_ms` | Timeout in milliseconds while waiting for backup acknowledgements |
-
-Characteristics:
-
-- Stronger write confirmation.
-- Write latency depends on backup responsiveness.
-- Unconfirmed backups continue to be retried in the background.
-- The client may still see an error after the primary write has already succeeded when the required backup acknowledgements are not met.
-
-Suitable for:
-
-- Narrowing the confirmation window between the primary and backups.
-- Backup latency is predictable.
-- The caller can accept the extra latency of synchronous writes.
-
-## Configuring Read Acceleration
-
-Backups do not participate in reads by default. To let a backup serve reads, explicitly configure `operations`.
-
-```json
-{
- "name": "cache-backend",
- "backend": "memfs",
- "operations": [
- {"operation": "write", "priority": 0},
- {
- "operation": "read",
- "priority": 10
- }
- ]
-}
-```
-
-Read priority rules:
-
-- Lower `priority` values are tried first.
-- Only backups with `read` configured join the read route.
-- The primary backend always remains the final fallback.
-- Cold-backup nodes usually should not be read-enabled.
-
-If a backup defines only `read` but not `write`, it does not receive normal multi-write replication. Use that only when you explicitly control how the backend gets its data.
+| `initial_partitions` | `16` | Metadata partitions created for an account |
+| `checkpoint_interval_secs` | `86400` | Checkpoint discovery interval; minimum `60` |
+| `provider` | `"filesystem"` | Metadata segment provider: `filesystem` or `cache` |
-## Redirect Configuration
+Each `items[]` entry needs `name`, `backend`, and backend-specific `params`.
+`encryption` is optional. Backup names must be unique and cannot be
+`primary`.
-Redirect sends matching files to a specified backup instead of the primary backend.
+When `provider` is `cache`, multi-write reuses the top-level CacheRuntime.
+The top-level `cache.provider` selects the implementation; multi-write has no
+separate Cache Provider connection.
-Redirect by file extension:
+The Python configuration layer still accepts legacy V1 sync, acknowledgement,
+retry, operation, exclude, and redirect fields. It warns and removes them
+before calling Rust; the fields have no effect.
-```json
-{
- "storage": {
- "agfs": {
- "backend": "local",
- "redirects": [
- {
- "type": "FileExtensionPolicy",
- "extensions": ["(pdf|ppt|zip)"],
- "target": ["object-store"]
- }
- ],
- "backups": {
- "items": [
- {
- "name": "object-store",
- "backend": "s3",
- "s3": {
- "bucket": "openviking-large-files",
- "endpoint": "https://s3.example.com"
- }
- }
- ]
- }
- }
- }
-}
-```
-
-Redirect by file size:
-
-```json
-{
- "type": "FileOverSizePolicy",
- "max_size_mb": 100,
- "target": ["object-store"]
-}
-```
-
-Notes:
-
-- `target` must reference an existing backup `name`.
-- Redirected files still appear as normal readable, listable, and queryable files through the public APIs.
-- Redirect mappings are stored in internal metadata on the primary backend.
-
-## Exclude Configuration
-
-Exclude makes one backup skip matching files.
-
-```json
-{
- "name": "cache-backend",
- "backend": "memfs",
- "excludes": [
- {
- "type": "FileOverSizePolicy",
- "max_size_mb": 50
- },
- {
- "type": "FileExtensionPolicy",
- "extensions": ["(mp4|zip)"]
- }
- ]
-}
-```
-
-Common uses:
-
-- Exclude large files from cache backends.
-- Exclude file types that do not need to be preserved on a lower-cost backup.
-- Keep one backup focused on text or configuration resources only.
-
-If the target backup of a redirect also excludes the same file, the configuration is self-contradictory. Fix the configuration instead of expecting the system to guess another target.
-
-## Encryption Configuration
-
-Multi-write storage reuses OpenViking's transparent at-rest encryption.
-
-Example with global encryption enabled:
-
-```json
-{
- "encryption": {
- "enabled": true,
- "provider": "local",
- "local": {
- "key_file": "~/.openviking/master.key"
- }
- },
- "storage": {
- "workspace": "./data",
- "agfs": {
- "backend": "local",
- "backups": {
- "items": [
- {
- "name": "plain-cache",
- "backend": "memfs",
- "encryption": {
- "enabled": false
- }
- },
- {
- "name": "encrypted-backup",
- "backend": "local",
- "local": {
- "workspace": "./data/encrypted-backup"
- },
- "encryption": {
- "enabled": true
- }
- }
- ]
- }
- }
- }
-}
-```
-
-Rules:
-
-- When global `encryption.enabled=true`, the primary backend must be encrypted.
-- Each backup may independently control encryption through `encryption.enabled`.
-- The Python SDK, HTTP API, and CLI do not need to handle encryption or decryption.
-- Internal metadata such as `.redirect.json` and `.sync_log.json` follows the primary backend's encryption policy.
+## Runtime Behavior
-## Migrating Existing Data
-
-Multi-write only replicates writes that happen after it is enabled. It does not automatically copy historical files.
-
-For OVPack migration, follow [Working with Multi-Write Storage](./09-ovpack.md#working-with-multi-write-storage): configure multi-write on the clean target before restoring through that server, then verify each replica before resuming writes. That guide also covers target account initialization and restore conflict handling.
-
-## Verifying the Configuration
-
-Before startup, it is recommended to run:
-
-```bash
-openviking-server doctor
-```
-
-After startup, verify with ordinary file APIs:
-
-```bash
-ov write viking://resources/multiwrite-check.txt \
- --content "multi-write check"
-
-ov read viking://resources/multiwrite-check.txt
-```
-
-If you use a local backup, you can also inspect the backup directory directly. Also run `ov system backend sync-status viking://resources` and check every intended replica. A successful ordinary read may come only from the primary and does not prove replication.
-
-## FAQ
-
-### Why is a backup not serving reads?
-
-Backups are write-only by default. To make a backup serve reads, configure:
-
-```json
-{
- "operations": [
- {"operation": "write", "priority": 0},
- {
- "operation": "read",
- "priority": 10
- }
- ]
-}
-```
+- The primary is the only direct read source and business source of truth.
+- Primary success returns without waiting for metadata or backups.
+- Multi-write metadata is stored only on the primary.
+- Backups converge in the background.
+- There is no synchronization status or manual retry API or CLI command.
-### Why do historical files not appear in the backup after enabling multi-write?
+A crash between primary success and event persistence can lose one replication
+event. Rebuild an affected backup from the primary.
-Multi-write only handles new writes after it is enabled. For historical data, follow the [OVPack migration workflow](./09-ovpack.md#working-with-multi-write-storage); enabling backups does not backfill it.
+## Partition Count
-### Can async mode guarantee that the newest data is immediately readable from a backup?
+`initial_partitions` applies only when an account's partition manifest is first
+created. Existing accounts continue to use their persisted partition count and
+routes after configuration changes. Partition expansion and shrinking are not
+supported in this release.
-No. Background retries need a functioning backend to catch up. A stale but successful backup read does not fall back to the primary; route reads to the primary when current data is required.
+## First Enable And Rollback
-### Will internal metadata files appear in normal user listings?
+When `.multiwrite.json` is absent, the mount starts foreground V2 event
+submission and imports current primary files in the background. Extra old
+files on backups are not removed.
-No. `.redirect.json` and `.sync_log.json` are internal files and are hidden from ordinary directory listings.
+Old multi-write protocols and old migration fields are rejected. Take a
+complete backup before enabling multi-write. Restore that backup before
+rolling back the binary.
-### If sync mode returns a failure, does that mean the primary backend definitely did not write the data?
+## Verification
-No. The primary write may already have succeeded while the required backup acknowledgements were not met. In that case the client can see a failure even though the data already exists on the primary backend, and lagging backups will continue to be repaired in the background.
+Use normal file APIs to verify primary reads and writes. Use the metrics in
+[Metrics](../concepts/12-metrics.md) to monitor queue depth, worker failures,
+backup lag, protocol version, and protocol status.
## Related Documents
- [Multi-Write Storage](../concepts/14-multi-write-storage.md)
-- [Storage Architecture](../concepts/05-storage.md)
- [Configuration Guide](./01-configuration.md)
-- [Encryption Guide](./08-encryption.md)
- [OVPack Import and Export](./09-ovpack.md)
diff --git a/docs/scripts/check-api-reference.mjs b/docs/scripts/check-api-reference.mjs
index 6a1a4432e..d83dcb212 100644
--- a/docs/scripts/check-api-reference.mjs
+++ b/docs/scripts/check-api-reference.mjs
@@ -178,11 +178,9 @@ function findRoute(method, documentedPath) {
const candidateMethod = candidate.slice(0, separator)
const candidatePath = candidate.slice(separator + 1)
if (candidateMethod !== method) continue
- const placeholderPattern =
- candidatePath.startsWith('/webdav/resources/{}') ||
- candidatePath.startsWith('/api/v1/system/sync/{}')
- ? '.+'
- : '[^/]+'
+ const placeholderPattern = candidatePath.startsWith('/webdav/resources/{}')
+ ? '.+'
+ : '[^/]+'
const pattern = candidatePath
.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
.replace(/\\\{\\\}/g, placeholderPattern)
@@ -351,10 +349,7 @@ function routeIsCovered(route, coverage) {
.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
.replace(
/\\\{\\\}/g,
- routePath.startsWith('/webdav/resources/{}') ||
- routePath.startsWith('/api/v1/system/sync/{}')
- ? '.+'
- : '[^/]+'
+ routePath.startsWith('/webdav/resources/{}') ? '.+' : '[^/]+'
)
for (const reference of coverage) {
const referenceSeparator = reference.indexOf(' ')
diff --git a/docs/zh/api/01-overview.md b/docs/zh/api/01-overview.md
index 39d6406b7..8e278afd1 100644
--- a/docs/zh/api/01-overview.md
+++ b/docs/zh/api/01-overview.md
@@ -354,10 +354,6 @@ JSON 输出 - 错误:
| GET | `/api/v1/system/status` | 系统状态 |
| POST | `/api/v1/system/wait` | 等待后台处理完成 |
| POST | `/api/v1/system/consistency` | 文件系统与向量索引一致性检查 |
-| POST | `/api/v1/system/backend/sync-status` | 查询后端同步状态 |
-| POST | `/api/v1/system/backend/sync-retry` | 重试后端同步 |
-| GET | `/api/v1/system/sync/{sync_path}` | 路径形式的同步状态兼容接口 |
-| POST | `/api/v1/system/sync/{sync_path}/retry` | 路径形式的同步重试兼容接口 |
### [资源](02-resources.md)与[文件系统](03-filesystem.md)
diff --git a/docs/zh/api/07-system.md b/docs/zh/api/07-system.md
index 9281b3fb1..420fac892 100644
--- a/docs/zh/api/07-system.md
+++ b/docs/zh/api/07-system.md
@@ -1,6 +1,7 @@
# 系统状态
-OpenViking 系统 API 提供健康检查、就绪检查、一致性检查和主备后端同步状态。组件级观测和 Prometheus 指标分别提供独立文档。
+OpenViking 系统 API 提供健康检查、就绪检查、一致性检查和后台处理状态。
+组件级观测和 Prometheus 指标分别提供独立文档。
## API 参考
@@ -409,99 +410,6 @@ ov system wait --timeout 60
---
-### backend_sync_status()
-
-查询指定 Viking URI 子树在主备存储后端之间的同步状态。该接口要求 ROOT 或 ADMIN 权限。
-
-**HTTP API**
-
-```http
-POST /api/v1/system/backend/sync-status
-Content-Type: application/json
-```
-
-```bash
-curl -X POST http://localhost:1933/api/v1/system/backend/sync-status \
- -H "Content-Type: application/json" \
- -H "X-API-Key: your-admin-key" \
- -d '{"uri":"viking://resources"}'
-```
-
-也可以使用 URI 路径形式:
-
-```http
-GET /api/v1/system/sync/{sync_path}
-```
-
-**CLI**
-
-```bash
-ov system backend sync-status viking://resources
-```
-
-**响应示例**
-
-```json
-{
- "status": "ok",
- "result": {
- "path": "viking://resources",
- "entry_count": 12
- }
-}
-```
-
-`result` 由当前文件系统后端返回;`path` 标识查询范围,`entry_count` 表示该范围内的同步记录数。具体后端可能附加待同步、失败记录等诊断字段。
-
-### backend_sync_retry()
-
-重试指定 URI 子树中尚未完成的主备后端同步工作。该接口要求 ROOT 或 ADMIN 权限。
-
-**HTTP API**
-
-```http
-POST /api/v1/system/backend/sync-retry
-Content-Type: application/json
-```
-
-```bash
-curl -X POST http://localhost:1933/api/v1/system/backend/sync-retry \
- -H "Content-Type: application/json" \
- -H "X-API-Key: your-admin-key" \
- -d '{"uri":"viking://resources"}'
-```
-
-URI 路径形式为:
-
-```http
-POST /api/v1/system/sync/{sync_path}/retry
-```
-
-**CLI**
-
-```bash
-ov system backend sync-retry viking://resources
-```
-
-**响应示例**
-
-```json
-{
- "status": "ok",
- "result": {
- "path": "viking://resources",
- "retried": 2,
- "failed": 0
- }
-}
-```
-
-`retried` 是本次重新调度的记录数,`failed` 是重试调度失败的记录数;具体后端可能附加额外诊断字段。
-
-公共 Python、TypeScript 和 Go SDK 当前没有主备后端同步方法,因此以上小节只展示 HTTP 和 CLI Tab。
-
----
-
<a id="reindex"></a><a id="observer-api"></a>
## 相关文档
diff --git a/docs/zh/api/20-webdav.md b/docs/zh/api/20-webdav.md
index 215d1b65c..d6cf424ba 100644
--- a/docs/zh/api/20-webdav.md
+++ b/docs/zh/api/20-webdav.md
@@ -17,7 +17,7 @@ OpenViking Server 也提供了一个面向资源文件的精简 WebDAV 适配层
- 仅开放 `resources` 命名空间,不暴露 memories、skills、sessions 等其他空间。
- 以文本写入为主,当前 `PUT` 只接受 UTF-8 文本内容。
- 只实现一小部分 WebDAV 方法:`OPTIONS`、`PROPFIND`、`GET`、`HEAD`、`PUT`、`DELETE`、`MKCOL`、`MOVE`。
-- 语义侧边文件和系统内部文件保持内部可见。`.abstract.md`、`.overview.md`、`.path.ovlock`、`.redirect.json`、`.sync_log.json` 这些派生或内部文件不会出现在 WebDAV 列表中,也不能被直接访问。
+- 语义侧边文件和系统内部文件保持内部可见。`.abstract.md`、`.overview.md`、`.path.ovlock` 和多写 metadata 不会出现在 WebDAV 列表中,也不能被直接访问。
行为说明:
@@ -25,7 +25,7 @@ OpenViking Server 也提供了一个面向资源文件的精简 WebDAV 适配层
- 通过 WebDAV 覆盖已有文件时,会像 `write()` 一样刷新相关语义和向量。
- `PUT` 不会自动创建父目录。缺失的目录需要先用 `MKCOL` 创建。
- 用户自己创建的点目录或点文件仍然可见,只有上面列出的保留内部文件名会被隐藏。
-- 启用主备存储时,被 redirect 到 backup 的文件仍会通过文件系统 API 呈现为普通文件;内部 redirect 和同步元数据不会暴露给调用方。
+- 启用多写时,WebDAV 读取只访问 primary,backup 在后台异步复制。
## API 参考
diff --git a/docs/zh/concepts/05-storage.md b/docs/zh/concepts/05-storage.md
index 710c964e1..1ab4eeae4 100644
--- a/docs/zh/concepts/05-storage.md
+++ b/docs/zh/concepts/05-storage.md
@@ -54,10 +54,10 @@ AGFS 提供 POSIX 风格的文件操作,支持多种后端。
默认情况下,AGFS 使用一个后端作为内容存储。配置 `storage.agfs.backups` 后,OpenViking 会启用主备模式:
-- 顶层 `storage.agfs.backend` 是 primary,作为权威写入目标。
-- `storage.agfs.backups.items[]` 是 backup,用于副本、迁移或读加速。
+- 顶层 `storage.agfs.backend` 是 primary,也是唯一直接读取来源。
+- `storage.agfs.backups.items[]` 是后台异步更新的 backup。
- Python SDK、HTTP API 和 CLI 的文件系统接口保持不变。
-- 主备存储内部使用 `.redirect.json` 和 `.sync_log.json` 维护 redirect 映射与同步进度,这些文件对用户不可见。
+- V2 metadata 分区存储,并且只存储在 primary。
更多概念说明见 [主备存储](./14-multi-write-storage.md),配置示例见 [主备存储指南](../guides/13-multi-write-storage.md)。
@@ -158,5 +158,5 @@ client.mv(
- [架构概述](./01-architecture.md) - 系统整体架构
- [上下文层级](./03-context-layers.md) - L0/L1/L2 模型
- [Viking URI](./04-viking-uri.md) - URI 规范
-- [主备存储](./14-multi-write-storage.md) - primary/backup、主备路由与一致性
+- [多写存储](./14-multi-write-storage.md) - V2 primary 与 backup 一致性
- [检索机制](./07-retrieval.md) - 检索流程详解
diff --git a/docs/zh/concepts/12-metrics.md b/docs/zh/concepts/12-metrics.md
index 0ccd621fa..29c021a12 100644
--- a/docs/zh/concepts/12-metrics.md
+++ b/docs/zh/concepts/12-metrics.md
@@ -288,8 +288,12 @@ RAGFS 通过一次原生 `metrics()` 调用读取文件系统、Cache、multi-ba
| `openviking_ragfs_cache_bytes_total` | Counter | `source` | 来自后端和缓存的字节数 |
| `openviking_ragfs_cache_operation_duration_seconds_total` | Counter | `operation` | get/put/delete 累计耗时 |
| `openviking_ragfs_cache_inflight_events_total` | Counter | `event` | leader/follower/backend_saved 次数 |
-| `openviking_ragfs_multiwrite_background_tasks` | Gauge | 无 | 后台任务数,包含重试循环 |
-| `openviking_ragfs_multiwrite_read_routes_total` | Counter | `route` | primary/backup/redirect/miss 路由选择次数 |
+| `openviking_ragfs_multiwrite_background_tasks` | Gauge | 无 | 每个 mount 的活跃 flush runtime,值为 `0` 或 `1` |
+| `openviking_ragfs_multiwrite_pending_events` | Gauge | 无 | 已提交且正在排队、执行中或重试中的事件数 |
+| `openviking_ragfs_multiwrite_errors_total` | Counter | `worker` | `flush`、`checkpoint`、`catch_up`、`gc` 的最终失败次数 |
+| `openviking_ragfs_multiwrite_backend_lag_events` | Gauge | `backend, partition` | 所有 account 的 `tail - synced_seq` 之和 |
+| `openviking_ragfs_multiwrite_protocol_version` | Gauge | `version` | 闭集标签 `v2` 对应的 mount 数 |
+| `openviking_ragfs_multiwrite_protocol_status` | Gauge | `status` | 闭集标签 `migrating`、`stable` 对应的 mount 数 |
这些指标不带 `mount` 和 `account_id` 标签。未启用 Cache 或 multi-backend
时,不生成对应指标族。`status` 为 `success` 或 `error`。
@@ -297,6 +301,9 @@ RAGFS 通过一次原生 `metrics()` 调用读取文件系统、Cache、multi-ba
Histogram 包含全部操作结果,有限桶边界范围为 0.0001 至 10 秒。
Python `get_stats()` 保留原有微秒字段。
+protocol 指标始终包含闭集中的两条 series。任何 protocol、account 或 segment
+manifest 无法读取或校验时,本次完整 snapshot 失败,不会发布部分 lag=0。
+
### 任务与 Task Tracker
| 指标族 | 类型 | 常见标签 | 含义 |
diff --git a/docs/zh/concepts/14-multi-write-storage.md b/docs/zh/concepts/14-multi-write-storage.md
index 5464b5bb1..d7d0dbdc3 100644
--- a/docs/zh/concepts/14-multi-write-storage.md
+++ b/docs/zh/concepts/14-multi-write-storage.md
@@ -1,134 +1,45 @@
# 主备存储
-主备存储(multi-write)让 OpenViking 在一个统一的文件系统抽象下,同时使用一个主存储和多个备份存储。它可用于保存副本(包括跨地域副本)、配置读路由和迁移存储;不会自动把 backup 提升为 primary。
+V2 多写以一个 primary backend 作为业务真相,并把文件复制到配置的 backup。
-从 API 使用者视角看,`read()`、`write()`、`ls()`、`stat()` 等接口不变。主备存储逻辑位于 RAGFS 内部,调用方不需要关心文件最终落在哪个底层后端。
+## 读写模型
-## 核心模型
+- 所有直接读取只访问 primary。backup 不参与业务读取。
+- primary 变更成功后立即返回。
+- metadata 持久化和 backup 应用在后台异步执行。
+- backup 落后不会改变 primary 已返回的结果。
+- 多写 metadata 只存储在 primary。
-主备存储由一个 primary 和多个 backup 组成:
+该模型是最终一致。进程可能在 primary 成功和事件持久化之间崩溃,导致该次
+复制事件丢失。发生此情况后,需要从 primary 重建受影响的 backup。
-| 角色 | 配置位置 | 说明 |
-| --- | --- | --- |
-| primary | `storage.agfs.backend` | 权威写入目标,也是读取兜底 |
-| backup | `storage.agfs.backups.items[]` | 接收复制写入,可选参与读取 |
+## Metadata 模型
-没有配置 `backups` 时,OpenViking 继续使用原有单后端模式。
+每个 account 使用固定数量的 metadata partition。事件路由到一个 partition,
+追加到 segment log,再由每个 backup 独立消费。Checkpoint 压缩旧历史,GC
+清理不再需要的 metadata。
-## 写入路径
+`initial_partitions` 仅在 account metadata 首次创建时决定 partition 数量。
+已有 account 始终使用持久化的 partition 数量和 routes,修改配置不会改变
+现有布局。本期不支持扩容或缩容。
-默认情况下,写入先落到 primary,再复制到 write-enabled backup。
+## 首次启用导入
-```text
-Client
- -> OpenViking API
- -> RAGFS MultiWrite
- -> primary
- -> backup1 / backup2 / ...
-```
+`.multiwrite.json` 不存在时,启动流程创建 `V2/migrating` 状态,先启用前台
+V2 事件提交,再在后台导入 primary 当前文件。导入成功后切换为 `stable`。
+导入不会删除 backup 中多余的旧文件。
-backup 未配置 `operations` 时默认参与写入。这样可以用最少配置得到冷备能力。
+旧多写协议和旧迁移字段直接拒绝,不执行转换或解释。
-## 同步模式
+## 运维
-主备存储支持两种一致性模式。
-
-| 模式 | 配置值 | 行为 | 适用场景 |
-| --- | --- | --- | --- |
-| 异步复制 | `async` | primary 写成功后立即返回,backup 后台同步 | 低延迟写入、最终一致 |
-| 同步复制 | `sync` | primary 写成功后等待 backup 确认 | 更强写入确认、可接受额外延迟 |
-
-异步模式下,backup 可能落后于 primary,恢复时间取决于后端可用性和重试结果。同步模式下,可以通过 `write_ack_count` 和 `write_ack_timeout_ms` 控制需要等待多少 backup 确认,以及等待多久。
-
-同步模式未达到所需确认数时会返回错误,但 primary 可能已经写入成功,不会因此自动回滚。未确认的 backup 仍会在后台重试;同步状态的检查方法见[主备存储指南](../guides/13-multi-write-storage.md)。
-
-## 读取路径
-
-读取不会默认访问所有 backup。只有显式声明 `read` 操作的 backup 才会进入读路由。
-
-读取顺序如下:
-
-```text
-1. 按 priority 升序访问 read-enabled backup
-2. 回退到 primary
-3. 如果文件被 redirect,则访问 redirect target
-4. 仍未命中则返回 NotFound
-```
-
-这种设计避免冷备节点默认参与读取,降低读到旧数据的风险。
-
-## Redirect
-
-Redirect 表示“某些文件不写入 primary,而是写入指定 backup”。
-
-常见用途:
-
-- 大文件进入对象存储。
-- 特定后缀文件进入专门 backend。
-- 主存储只保存常规内容,特殊文件由其他 backend 保存。
-
-Redirect 策略配置在 primary 上。命中策略后,OpenViking 会把映射记录到内部元数据中。用户执行 `ls()`、`stat()`、`read()` 时仍能看到正常的文件系统视图。
-
-## Exclude
-
-Exclude 表示“某个 backup 不接收匹配的文件”。
-
-常见用途:
-
-- 内存或缓存 backend 不保存大文件。
-- 某个 backup 只保存文本类资源。
-- 某个低成本 backend 排除临时或超大文件。
-
-Exclude 策略配置在 backup 上,只影响该 backup 是否接收写入。
-
-## 内部元数据
-
-主备存储使用两个内部元数据文件:
-
-| 文件 | 作用 |
-| --- | --- |
-| `.redirect.json` | 记录 redirect 文件对应的目标 backend |
-| `.sync_log.json` | 记录每个文件的同步版本和 backup 确认进度 |
-
-这些文件对普通用户不可见,不会出现在常规列表结果中,也不应通过公开 API 直接读写。
-
-如果 primary 开启静态数据加密,这些内部元数据也会跟随 primary 加密策略写入。
-
-## 加密关系
-
-主备存储不会改变 OpenViking 的透明加密模型。
-
-规则如下:
-
-- Python 层和公共 API 不感知加密实现。
-- primary 在全局加密开启时必须加密。
-- backup 可以独立决定是否加密。
-- 内部元数据必须走 primary 的加密入口。
-
-这意味着启用主备存储后,调用方式仍然不变;只需要通过配置决定每个 backend 的加密策略。
-
-## 与 OVPack 的关系
-
-主备存储只负责“启用之后的新写入”。它不会自动同步启用之前已经存在于 primary 中的历史文件。
-
-如果要迁移存量数据,推荐流程是:
-
-1. 暂停写入或安排可核对的增量补齐窗口,再使用 OVPack 或其他受控方式迁移存量。
-2. 校验目标 backend 数据。
-3. 启用主备存储配置。
-4. 补齐迁移期间的变更并验证同步状态后恢复写入;后续新增和修改由主备存储复制。
-
-## 限制
-
-- 异步模式下 backup 可能落后;读取 backup 时不能假设已经看到最新写入。
-- 启用主备存储前的历史文件需要单独迁移或回填。
-- redirect 文件依赖内部元数据恢复目录视图。
-- 多进程同时写同一 primary 时,需要未来的分布式元数据锁能力。
-- 热点目录会频繁更新内部元数据,可能带来额外写放大。
+V2 不提供同步状态或手工重试 API 和 CLI 命令。请使用多写 Prometheus 指标
+观察队列深度、worker 失败、backup lag、协议版本和协议状态。
## 相关文档
- [存储架构](./05-storage.md)
+- [指标](./12-metrics.md)
- [配置指南](../guides/01-configuration.md)
-- [主备存储指南](../guides/13-multi-write-storage.md)
+- [多写存储指南](../guides/13-multi-write-storage.md)
- [OVPack 导入导出](../guides/09-ovpack.md)
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index 12dd465be..66df58985 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -1290,8 +1290,7 @@ Glob 引擎配置,用于路径模式匹配。这些设置为服务端配置,
|------|------|------|--------|
| `backend` | str | `"local"`、`"s3"` 或 `"memory"` | `"local"` |
| `timeout` | float | 请求超时时间(秒) | `10.0` |
-| `backups` | object | 主备存储配置。配置后顶层 `backend` 作为 primary,`backups.items[]` 作为 backup | `null` |
-| `redirects` | array | 主备存储的文件重定向策略。命中后文件写入指定 backup,而不是 primary | `[]` |
+| `backups` | object | 多写存储配置。配置后顶层 `backend` 作为 primary,`backups.items[]` 作为 backup | `null` |
| `queuefs` | object | QueueFS 配置。控制 `/queue` 的命名空间模式、后端和运行时参数 | `{ "mode": "shared", "backend": "sqlite", "recover_stale_sec": 0, "busy_timeout_ms": 5000 }` |
| `queue_db_path` | str(可选)| 旧版兼容字段,用于覆盖 QueueFS 的 sqlite 数据库文件路径。已被 `storage.agfs.queuefs.db_path` 取代。未设置时默认为 `{storage.workspace}/_system/queue/queue.db`。适用于 workspace 卷不支持 sqlite 的场景(例如某些网络文件系统) | `null` |
| `s3` | object | S3 后端配置(`backend=s3` 时使用) | - |
@@ -1306,7 +1305,7 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
##### 主备存储配置
-`storage.agfs.backups` 用于启用主备存储。未配置时,OpenViking 保持单 backend 模式。
+`storage.agfs.backups` 启用异步 V2 多写。顶层 backend 始终是唯一直接读取来源。
```json
{
@@ -1314,26 +1313,16 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
"workspace": "./data",
"agfs": {
"backend": "local",
- "redirects": [
- {
- "type": "FileExtensionPolicy",
- "extensions": ["(pdf|ppt|zip)"],
- "target": ["s3-backup"]
- }
- ],
"backups": {
- "sync_type": "async",
+ "initial_partitions": 16,
+ "checkpoint_interval_secs": 86400,
+ "provider": "filesystem",
"items": [
{
- "name": "s3-backup",
- "backend": "s3",
- "s3": {
- "bucket": "openviking-backup",
- "region": "cn-beijing",
- "endpoint": "https://tos-s3-cn-beijing.volces.com",
- "access_key": "your-ak",
- "secret_key": "your-sk",
- "prefix": "multi-write"
+ "name": "local-backup",
+ "backend": "local",
+ "params": {
+ "workspace": "./data/backup"
}
}
]
@@ -1347,36 +1336,13 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
| 参数 | 类型 | 说明 | 默认值 |
|------|------|------|--------|
-| `sync_type` | str | 主备同步模式,支持 `"async"` 或 `"sync"` | `"async"` |
-| `write_ack_count` | int | `sync` 模式下返回前需要的 backup 确认数 | 全部 backup |
-| `write_ack_timeout_ms` | int | `sync` 模式下等待 backup 确认的超时时间,单位毫秒 | `null` |
-| `write_concurrency` | int | 异步 backup 写入并发上限 | `null` |
-| `items` | array | backup backend 列表,每个 item 复用普通 backend 配置并增加 `name`、`operations`、`excludes`、`encryption` 等字段 | `[]` |
-
-`redirects` 常用字段:
-
-| 参数 | 类型 | 说明 | 默认值 |
-|------|------|------|--------|
-| `type` | str | 策略类型,支持 `"FileExtensionPolicy"` 或 `"FileOverSizePolicy"` | 必填 |
-| `extensions` | array | `FileExtensionPolicy` 使用的扩展名正则列表,例如 `["(pdf\|ppt)"]` | `[]` |
-| `max_size_mb` | int | `FileOverSizePolicy` 使用的文件大小阈值,单位 MB | `null` |
-| `target` | array | 命中策略后写入的 backup `name` 列表 | 必填 |
-
-按文件大小重定向示例:
-
-```json
-{
- "type": "FileOverSizePolicy",
- "max_size_mb": 100,
- "target": ["s3-backup"]
-}
-```
-
-注意:
+| `initial_partitions` | int | account 首次初始化的 metadata partition 数;修改不影响已有 account | `16` |
+| `checkpoint_interval_secs` | int | Checkpoint 扫描间隔,最小值为 `60` | `86400` |
+| `provider` | str | Metadata provider:`filesystem` 或复用顶层 CacheRuntime 的 `cache` | `"filesystem"` |
+| `items` | array | backup 列表,包含 `name`、`backend`、`params` 和可选 `encryption` | `[]` |
-- `redirects` 配置在顶层 `storage.agfs`,表示 primary 的重定向策略。
-- `target` 必须引用 `backups.items[]` 中已经定义的 backup `name`。
-- 命中 redirect 的文件仍会通过普通文件系统 API 呈现为可读、可列举的文件。
+Python 配置层仍接受旧 sync、retry、operations、excludes 和 redirects
+字段,输出 warning 后会在调用 Rust 前删除,字段不生效。
更多配置示例见 [主备存储指南](./13-multi-write-storage.md)。
@@ -1444,9 +1410,9 @@ RAGFS 默认使用 Rust binding 模式,通过 Rust 实现直接访问文件系
}
```
-标准配置没有全局 `cache.enabled`。当 CacheFS 或 QueueFS 选择 `backend=cache`,或 PathLock 选择 `provider=cache` 时初始化 CacheRuntime。Cache PathLock 当前只支持 `cache.provider=redis`,不支持 DynamicProvider。全部模块使用本地 Provider 时不解析 `cache.params`,也不连接 Provider。
+标准配置没有全局 `cache.enabled`。当 CacheFS 或 QueueFS 选择 `backend=cache`、PathLock 选择 `provider=cache`,或多写选择 `backups.provider=cache` 时初始化 CacheRuntime。Cache PathLock 当前只支持 `cache.provider=redis`,不支持 DynamicProvider。全部模块使用本地 Provider 时不解析 `cache.params`,也不连接 Provider。
-这是一次配置破坏性变更:`storage.agfs.cache`、`storage.agfs.queuefs.backend="redis"` 和 `storage.agfs.queuefs.redis` 已删除并会被拒绝。请把 Provider 参数迁移到顶层 `cache.provider/cache.params`,业务模块改为 `cachefs.backend="cache"` 或 `queuefs.backend="cache"`;Redis 的 `singleton` 改为 `standalone`,`tls_enabled` 改为使用 `rediss://` endpoint。
+这是一次配置破坏性变更:`storage.agfs.cache`、`storage.agfs.queuefs.backend="redis"`、`storage.agfs.queuefs.redis` 和 `storage.agfs.backups.provider="redis"` 已删除并会被拒绝。请把 Provider 参数迁移到顶层 `cache.provider/cache.params`,业务模块改为 `cachefs.backend="cache"`、`queuefs.backend="cache"` 或 `backups.provider="cache"`;Redis 的 `singleton` 改为 `standalone`,`tls_enabled` 改为使用 `rediss://` endpoint。
##### QueueFS 配置
diff --git a/docs/zh/guides/08-encryption.md b/docs/zh/guides/08-encryption.md
index 674571b08..4c1efffc2 100644
--- a/docs/zh/guides/08-encryption.md
+++ b/docs/zh/guides/08-encryption.md
@@ -21,7 +21,7 @@ OpenViking 可以按账户加密存储中新写入的文件。加解密由存储
- 全局 `encryption.enabled=true` 时,primary backend 必须加密。
- backup backend 可以通过自己的 `encryption.enabled` 控制是否加密。
-- `.redirect.json` 和 `.sync_log.json` 等主备存储内部元数据跟随 primary 加密策略。
+- V2 多写 metadata 存储在 raw primary,不经过 `EncryptionWrappedFS`。
- OpenViking 不提供也不需要公开的加解密 API 来操作这些内部文件。
更多主备存储配置见 [主备存储指南](./13-multi-write-storage.md)。
diff --git a/docs/zh/guides/09-ovpack.md b/docs/zh/guides/09-ovpack.md
index a611267db..9533748a8 100644
--- a/docs/zh/guides/09-ovpack.md
+++ b/docs/zh/guides/09-ovpack.md
@@ -33,22 +33,12 @@ Session 通过 user 命名空间一起迁移,路径为
## 与主备存储配合
-主备存储只复制启用之后的新写入,不会自动同步启用之前已经存在的历史文件。如需同时为 primary 和副本写入存量,应在**空目标环境恢复之前**启用主备存储:
+V2 启动会把当前 V1 primary 文件导入分区 metadata,并删除 V1 sidecar。
+导入不会删除 backup 中多余的旧文件。首次 V2 启动前,应停服所有实例并创建
+完整备份。
-1. 暂停业务写入,按源 account 分别导出或备份。
-2. 配置目标 primary、backup backend 及其写策略,启动目标服务并创建恢复身份。
-3. 通过该服务恢复或导入,使存量内容经过已配置的主备复制分发。
-4. 对每个恢复的 scope 检查同步状态,并在切流前验证各副本中的文件。恢复返回时,异步复制仍可能未完成。
-5. 验证内容和索引完整性后再恢复业务写入;验证完成前保留源数据和备份。
-
-例如,使用目标 account 的 admin key 检查:
-
-```bash
-ov system backend sync-status viking://resources
-ov system backend sync-status viking://user
-```
-
-先恢复再启用 backups 只会复制后续写入,不能把历史内容补到副本。
+导入开始后不支持原地回滚。要恢复 V1,必须恢复导入前的完整备份。启动后使用
+普通文件 API 验证 primary 内容,并通过 Prometheus 指标观察 backup lag。
更多说明见 [主备存储指南](./13-multi-write-storage.md)。
diff --git a/docs/zh/guides/13-multi-write-storage.md b/docs/zh/guides/13-multi-write-storage.md
index 14d5e2dc1..3211082db 100644
--- a/docs/zh/guides/13-multi-write-storage.md
+++ b/docs/zh/guides/13-multi-write-storage.md
@@ -1,20 +1,10 @@
-# 主备存储指南
+# 多写存储指南
-本指南介绍如何配置 OpenViking 的主备存储能力。主备存储允许一个 primary 后端同时复制写入多个 backup 后端,用于保存副本(包括跨地域副本)、配置读路由和存储迁移。primary 故障时不会自动提升 backup。
-
-主备存储能力位于 RAGFS 内部。OpenViking 的 Python SDK、HTTP API 和 CLI 使用方式保持不变。
-
-## 前置条件
-
-- 已有可用的 `ov.conf`。
-- 已确认 primary backend 可以正常读写。
-- 如果要接入 S3 兼容存储,已准备好 bucket、endpoint 和访问凭据。
-- 如需迁移已有数据,请遵循 [OVPack 主备存储迁移流程](./09-ovpack.md#与主备存储配合);仅启用 backups 不会复制历史文件。
+V2 多写在 primary 成功后返回,metadata 和 backup 在后台异步更新。primary
+是唯一直接读取来源。
## 最小配置
-将下列片段合并到现有模型和服务端配置。下面示例使用本地目录作为 primary,并把写入复制到另一个本地目录。
-
```json
{
"storage": {
@@ -22,13 +12,15 @@
"agfs": {
"backend": "local",
"backups": {
- "sync_type": "async",
+ "initial_partitions": 16,
+ "checkpoint_interval_secs": 86400,
+ "provider": "filesystem",
"items": [
{
"name": "local-backup",
"backend": "local",
- "local": {
- "workspace": "./data/backup"
+ "params": {
+ "workspace": "./data/backup"
}
}
]
@@ -38,385 +30,55 @@
}
```
-说明:
-
-- 顶层 `backend` 是 primary。
-- `backups.items[]` 是 backup 列表。
-- `name` 是 backup 的稳定身份,后续同步元数据会引用它。
-- `backend = "local"` 的 backup 使用 `local.workspace` 指定本地目录。
-- `sync_type` 不配置时默认按异步模式理解。
-
-## 多 Backup 配置
+V2 的三个顶层配置是:
-可以配置多个 backup。下面示例同时写入本地副本和 S3 兼容对象存储。
-
-```json
-{
- "storage": {
- "workspace": "./data",
- "agfs": {
- "backend": "local",
- "backups": {
- "sync_type": "async",
- "items": [
- {
- "name": "local-az2",
- "backend": "local",
- "local": {
- "workspace": "./data/local-az2"
- }
- },
- {
- "name": "object-store",
- "backend": "s3",
- "s3": {
- "bucket": "openviking-backup",
- "region": "us-east-1",
- "endpoint": "https://s3.example.com",
- "access_key": "your-access-key",
- "secret_key": "your-secret-key",
- "prefix": "openviking",
- "directory_marker_mode": "none"
- }
- }
- ]
- }
- }
- }
-}
-```
-
-建议:
-
-- `name` 不要使用会频繁变化的机器名或临时编号。
-- backup 的底层路径或 bucket 应避免与 primary 指向同一物理位置。
-- 修改 backup `name` 会影响历史同步元数据的识别,生产环境应谨慎变更。
-
-### S3 兼容存储注意事项
-
-按实际 endpoint 和对象存储配置选择以下 `s3` 字段:
-
-| 字段 | 是否必填 | 说明 |
+| 字段 | 默认值 | 含义 |
| --- | --- | --- |
-| `use_path_style` | 可选,默认 `true` | 使用路径风格 URL(`http://host/bucket/key`),按 endpoint 支持的方式选择。 |
-| `directory_marker_mode` | backup 项必填 | `none` 不创建标记,`empty` 创建零字节标记,`nonempty` 创建带内容的标记。每个 S3 backup 项都要显式设置;省略时启动会报 `AGFSConfigError: configuration error: invalid directory_marker_mode: null (valid: none, empty, nonempty)`。 |
-| `use_ssl` | 可选 | HTTP 端点(如 `http://localhost:9000`)需要设置为 `false`。 |
-
-**S3 兼容存储最小示例(RustFS/MinIO):**
-
-```json
-{
- "name": "s3-backup",
- "backend": "s3",
- "s3": {
- "bucket": "my-bucket",
- "endpoint": "http://localhost:9000",
- "access_key": "your-access-key",
- "secret_key": "your-secret-key",
- "prefix": "openviking",
- "use_ssl": false,
- "use_path_style": true,
- "directory_marker_mode": "none"
- }
-}
-```
-
-上面的示例使用 `none`,适用于不使用目录标记的 S3 兼容服务(RustFS、MinIO、Ceph 等)。backup 项的这个字段没有默认值;顶层 `storage` 的 S3 配置默认使用 `empty`。
-
-### Docker 网络配置
-
-在 Docker 中运行 OpenViking 并配置同主机的 S3 备份时,需要注意:
-
-- **Linux Docker**:使用 `--network host` 或宿主机局域网 IP。Docker bridge 网络可通过网关 IP(如 `172.17.0.1:9000`)访问宿主机局域网。
-- **macOS/Windows Docker Desktop**:从容器访问宿主机时可使用 `host.docker.internal`。Docker Desktop 4.34 及之后的版本也提供可选的 host networking,启用方式见 [Docker 官方说明](https://docs.docker.com/engine/network/drivers/host/)。
-
-如果启用 S3 备份后服务静默崩溃,请优先排查 Docker 网络。RAGFS Rust binding 在容器内无法访问 S3 端点时会报 `dispatch failure` 错误。
-
-## 同步模式选择
-
-### 异步模式
-
-异步模式适合大多数场景。
-
-```json
-{
- "backups": {
- "sync_type": "async",
- "items": []
- }
-}
-```
-
-特点:
-
-- primary 写入成功后立即返回。
-- backup 写入在后台执行。
-- 写入延迟低。
-- 后端不可用期间,落后时间没有固定上限,需观察同步和恢复状态。
-
-适合:
-
-- 写入吞吐优先。
-- backup 主要用于灾备。
-- 可以接受最终一致性。
-
-### 同步模式
-
-同步模式会等待 backup 确认。
-
-```json
-{
- "backups": {
- "sync_type": "sync",
- "write_ack_count": 1,
- "write_ack_timeout_ms": 5000,
- "items": [{"name": "local-backup", "backend": "local", "local": {"workspace": "./backup-data"}}]
- }
-}
-```
-
-参数说明:
-
-| 参数 | 说明 |
-| --- | --- |
-| `write_ack_count` | 写入返回前至少需要多少个 backup 确认 |
-| `write_ack_timeout_ms` | 等待 backup 确认的超时时间,单位毫秒 |
-
-特点:
-
-- 写入确认更强。
-- 写入延迟受 backup 影响。
-- 未确认的 backup 会继续由后台重试修复。
-- primary 已写成功但 backup 未达确认数时,客户端可能收到错误;此时 primary 中可能已经存在数据。
-
-适合:
-
-- 希望尽量减少 primary 与 backup 的确认窗口。
-- backup 延迟可控。
-- 调用方能接受同步写入带来的额外延迟。
-
-## 配置读加速
-
-backup 默认不参与读取。要让 backup 服务读取,需要显式配置 `operations`。
-
-```json
-{
- "name": "cache-backend",
- "backend": "memfs",
- "operations": [
- {"operation": "write", "priority": 0},
- {
- "operation": "read",
- "priority": 10
- }
- ]
-}
-```
-
-读取优先级规则:
-
-- `priority` 越小越优先。
-- 只有声明 `read` 的 backup 才参与读取。
-- primary 始终作为最终兜底。
-- 冷备 backup 不建议配置读能力。
-
-如果一个 backup 只配置了 `read`,没有配置 `write`,它不会接收普通复制写入。只有在你明确知道该 backend 的数据来源时,才应使用这种配置。
+| `initial_partitions` | `16` | 为 account 创建的 metadata partition 数量 |
+| `checkpoint_interval_secs` | `86400` | Checkpoint 扫描间隔,最小值为 `60` |
+| `provider` | `"filesystem"` | Metadata segment provider:`filesystem` 或 `cache` |
-## Redirect 配置
+每个 `items[]` 需要 `name`、`backend` 和 backend 专用 `params`。
+`encryption` 可选。backup 名称必须唯一,且不能为 `primary`。
-Redirect 用于把匹配的文件写入指定 backup,而不是写入 primary。
+`provider` 为 `cache` 时,多写复用顶层 CacheRuntime。实际实现由顶层
+`cache.provider` 选择;多写没有独立 Cache Provider 连接配置。
-按扩展名重定向:
+Python 配置层仍接受 V1 的 sync、ack、retry、operations、excludes 和
+redirects 字段,输出 warning 后会在调用 Rust 前删除,字段不生效。
-```json
-{
- "storage": {
- "agfs": {
- "backend": "local",
- "redirects": [
- {
- "type": "FileExtensionPolicy",
- "extensions": ["(pdf|ppt|zip)"],
- "target": ["object-store"]
- }
- ],
- "backups": {
- "items": [
- {
- "name": "object-store",
- "backend": "s3",
- "s3": {
- "bucket": "openviking-large-files",
- "endpoint": "https://s3.example.com"
- }
- }
- ]
- }
- }
- }
-}
-```
-
-按大小重定向:
-
-```json
-{
- "type": "FileOverSizePolicy",
- "max_size_mb": 100,
- "target": ["object-store"]
-}
-```
-
-注意:
-
-- `target` 必须引用已有 backup 的 `name`。
-- redirect 文件仍会通过普通 API 呈现为可读、可列举、可查询状态。
-- redirect 映射保存在 primary 的内部元数据中。
-
-## Exclude 配置
-
-Exclude 用于让某个 backup 跳过匹配文件。
-
-```json
-{
- "name": "cache-backend",
- "backend": "memfs",
- "excludes": [
- {
- "type": "FileOverSizePolicy",
- "max_size_mb": 50
- },
- {
- "type": "FileExtensionPolicy",
- "extensions": ["(mp4|zip)"]
- }
- ]
-}
-```
-
-常见用法:
-
-- 缓存 backend 排除大文件。
-- 低成本备份排除无需保存的文件类型。
-- 某个 backup 只保存文本或配置类资源。
-
-如果 redirect 的目标 backup 同时 exclude 了该文件,说明配置互相冲突。请优先修正配置,不要依赖系统自动猜测其他目标。
-
-## 加密配置
-
-主备存储复用 OpenViking 的透明静态加密能力。
-
-全局加密开启示例:
-
-```json
-{
- "encryption": {
- "enabled": true,
- "provider": "local",
- "local": {
- "key_file": "~/.openviking/master.key"
- }
- },
- "storage": {
- "workspace": "./data",
- "agfs": {
- "backend": "local",
- "backups": {
- "items": [
- {
- "name": "plain-cache",
- "backend": "memfs",
- "encryption": {
- "enabled": false
- }
- },
- {
- "name": "encrypted-backup",
- "backend": "local",
- "local": {
- "workspace": "./data/encrypted-backup"
- },
- "encryption": {
- "enabled": true
- }
- }
- ]
- }
- }
- }
-}
-```
-
-规则:
-
-- 全局 `encryption.enabled=true` 时,primary 必须加密。
-- backup 可以通过 `encryption.enabled` 单独控制是否加密。
-- Python SDK、HTTP API 和 CLI 不需要处理加解密。
-- `.redirect.json` 和 `.sync_log.json` 等内部元数据会跟随 primary 加密策略。
+## 运行行为
-## 存量数据迁移
-
-主备存储只复制启用之后的新写入,不会自动复制历史文件。
-
-使用 OVPack 迁移时,请遵循 [与主备存储配合](./09-ovpack.md#与主备存储配合):先在干净目标配置主备存储,再通过该服务恢复,验证各副本后恢复业务写入。目标 account 初始化和恢复冲突处理也在该流程中说明。
-
-## 验证配置
-
-启动前建议运行:
-
-```bash
-openviking-server doctor
-```
-
-启动后可以用普通文件 API 验证:
-
-```bash
-ov write viking://resources/multiwrite-check.txt \
- --content "multi-write check"
-
-ov read viking://resources/multiwrite-check.txt
-```
-
-如果使用本地 backup,可以直接检查 backup 目录中是否出现对应文件。还应运行 `ov system backend sync-status viking://resources`,逐个核对预期副本。普通读取成功可能只读到了 primary,不能证明复制完成。
-
-## 常见问题
-
-### 为什么 backup 没有参与读取?
-
-backup 默认只参与写入,不参与读取。需要在 backup 上显式配置:
-
-```json
-{
- "operations": [
- {"operation": "write", "priority": 0},
- {
- "operation": "read",
- "priority": 10
- }
- ]
-}
-```
+- primary 是唯一直接读取来源和业务真相。
+- primary 成功后立即返回,不等待 metadata 或 backup。
+- 多写 metadata 只存储在 primary。
+- backup 在后台最终收敛。
+- 不提供同步状态或手工重试 API 和 CLI 命令。
-### 为什么启用主备存储后历史文件没有出现在 backup?
+进程可能在 primary 成功和事件持久化之间崩溃,导致一次复制事件丢失。此时
+需要从 primary 重建受影响的 backup。
-主备存储只处理启用后的新写入。历史文件请按 [OVPack 迁移流程](./09-ovpack.md#与主备存储配合) 处理;启用 backups 不会自动补齐。
+## Partition 数量
-### 异步模式下能否保证立即读到 backup 的最新数据?
+`initial_partitions` 仅在 account 的 partition manifest 首次创建时生效。
+已有 account 在配置变化后继续使用持久化的 partition 数量和 routes。
+本期不支持扩容或缩容。
-不能。后台重试需要后端恢复可用才能追上进度。backup 返回旧数据但读取成功时,不会自动回退 primary;需要最新数据时,应让读路由使用 primary。
+## 首次启用与回滚
-### 内部元数据文件会出现在用户列表里吗?
+`.multiwrite.json` 不存在时,mount 先启用前台 V2 事件提交,再在后台导入
+primary 当前文件。导入不会删除 backup 中多余的旧文件。
-不会。`.redirect.json` 和 `.sync_log.json` 是内部文件,会被普通目录列表隐藏。
+旧多写协议和旧迁移字段直接拒绝。启用多写前必须创建完整备份;回滚二进制
+前必须先恢复该备份。
-### sync 模式返回失败是否表示 primary 一定没写入?
+## 验证
-不是。primary 写成功但 backup 未达到确认数时,客户端可能收到失败。此时 primary 数据可能已经存在,落后的 backup 会由后台重试修复。
+使用普通文件 API 验证 primary 读写。使用[指标](../concepts/12-metrics.md)
+监控队列深度、worker 失败、backup lag、协议版本和协议状态。
## 相关文档
-- [主备存储](../concepts/14-multi-write-storage.md)
-- [存储架构](../concepts/05-storage.md)
+- [多写存储](../concepts/14-multi-write-storage.md)
- [配置指南](./01-configuration.md)
-- [加密指南](./08-encryption.md)
- [OVPack 导入导出](./09-ovpack.md)
diff --git a/examples/claude-code-memory-plugin/.claude-plugin/plugin.json b/examples/claude-code-memory-plugin/.claude-plugin/plugin.json
index 627d750e5..c839bd3b2 100644
--- a/examples/claude-code-memory-plugin/.claude-plugin/plugin.json
+++ b/examples/claude-code-memory-plugin/.claude-plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "openviking-memory",
- "version": "0.7.10",
+ "version": "0.7.11",
"description": "Long-term semantic memory for Claude Code, powered by OpenViking. Auto-recall relevant memories at session start and capture important information during conversations.",
"author": {
"name": "OpenViking",
@@ -17,4 +17,4 @@
],
"mcpServers": "./.mcp.json",
"types": "./mods/usage/types.d.ts"
-}
+}
\ No newline at end of file
diff --git a/examples/claude-code-memory-plugin/package.json b/examples/claude-code-memory-plugin/package.json
index 71eb2f10f..52205300a 100644
--- a/examples/claude-code-memory-plugin/package.json
+++ b/examples/claude-code-memory-plugin/package.json
@@ -1,6 +1,6 @@
{
"name": "claude-code-openviking-memory",
- "version": "0.7.10",
+ "version": "0.7.11",
"description": "OpenViking memory plugin for Claude Code — semantic long-term memory",
"type": "module",
"scripts": {
diff --git a/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md b/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md
index 684d8172a..04c78ecc9 100644
--- a/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md
+++ b/examples/claude-code-memory-plugin/skills/ov-memory-doctor/reference.md
@@ -159,7 +159,7 @@ port is bound only after initialization finishes, so "connection refused"
during startup is normal; `/health` 200 says nothing about embedding or VLM.
`GET /ready` (no auth, 200 or 503):
-`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…,"multiwrite_sync":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`.
+`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`.
`ok`, `not_configured` and `not_supported` count as healthy; `embedding` is a
real embed call with a 10s cap. `503 {"status":"not_ready","reason":"initializing"}`
while booting; 404 on servers that predate the endpoint. The official docker
diff --git a/examples/codex-memory-plugin/.codex-plugin/plugin.json b/examples/codex-memory-plugin/.codex-plugin/plugin.json
index 2a3f34ac2..4870aa30a 100644
--- a/examples/codex-memory-plugin/.codex-plugin/plugin.json
+++ b/examples/codex-memory-plugin/.codex-plugin/plugin.json
@@ -1,6 +1,6 @@
{
"name": "openviking-memory",
- "version": "0.11.9",
+ "version": "0.11.10",
"description": "Long-term semantic memory for Codex, powered by OpenViking. Recall on UserPromptSubmit, incremental add_message on Stop (per turn), commit on SessionEnd (Codex >= 0.145) and on PreCompact, with an idle/ended sweep on SessionStart as the fallback for signals, crashes and older Codex builds. Includes a local stdio MCP proxy that reads OpenViking credentials from env/ovcli.conf.",
"author": {
"name": "OpenViking",
diff --git a/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md b/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md
index 63b6a73c8..6e36ee300 100644
--- a/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md
+++ b/examples/codex-memory-plugin/skills/ov-memory-doctor/reference.md
@@ -152,7 +152,7 @@ port is bound only after initialization finishes, so "connection refused"
during startup is normal; `/health` 200 says nothing about embedding or VLM.
`GET /ready` (no auth, 200 or 503):
-`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…,"multiwrite_sync":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`.
+`{"status":"ready"|"not_ready","checks":{"agfs":{"status":…,"checks":{"filesystem":…}},"vectordb":…,"api_key_manager":…,"embedding":…,"ollama":…}}`.
`ok`, `not_configured` and `not_supported` count as healthy; `embedding` is a
real embed call with a 10s cap. `503 {"status":"not_ready","reason":"initializing"}`
while booting; 404 on servers that predate the endpoint. The official docker
diff --git a/examples/memory-plugin-shared/doctor-core.test.mjs b/examples/memory-plugin-shared/doctor-core.test.mjs
index 8ff2b844c..86d8bf55b 100644
--- a/examples/memory-plugin-shared/doctor-core.test.mjs
+++ b/examples/memory-plugin-shared/doctor-core.test.mjs
@@ -237,7 +237,7 @@ test("lintServerConf covers every harness, not just the two with a legacy ov.con
});
test("assessReady interprets the readiness checks", () => {
- const ok = { ok: true, status: 200, json: { status: "ready", checks: { agfs: { status: "ok", checks: { filesystem: "ok", multiwrite_sync: "not_supported" } }, vectordb: "ok", api_key_manager: "not_configured", embedding: "ok", ollama: "not_configured" } } };
+ const ok = { ok: true, status: 200, json: { status: "ready", checks: { agfs: { status: "ok", checks: { filesystem: "ok" } }, vectordb: "ok", api_key_manager: "not_configured", embedding: "ok", ollama: "not_configured" } } };
let report = createReport();
assert.equal(assessReady(report, ok).ready, true);
assert.equal(report.exitCode(), 0);
diff --git a/examples/ov.conf.example b/examples/ov.conf.example
index 5b81cb297..8650d0b0c 100644
--- a/examples/ov.conf.example
+++ b/examples/ov.conf.example
@@ -77,6 +77,17 @@
"agfs": {
"backend": "local",
"timeout": 10,
+ "backups_example": {
+ "_comment": "Rename this key to backups to enable V2 multi-write.",
+ "initial_partitions": 16,
+ "checkpoint_interval_secs": 86400,
+ "provider": "cache",
+ "items": [{
+ "name": "local-backup",
+ "backend": "local",
+ "params": {"workspace": "./data/backup"}
+ }]
+ },
"cachefs": {
"backend": "local",
"namespace": "openviking",
diff --git a/openviking/pyagfs/async_client.py b/openviking/pyagfs/async_client.py
index ebfa1d563..4c6657e8c 100644
--- a/openviking/pyagfs/async_client.py
+++ b/openviking/pyagfs/async_client.py
@@ -349,18 +349,6 @@ class AsyncAGFSClient:
ctx=_fs_ctx_or_default(path, fs_ctx),
)
- async def system_sync_status(
- self, path: str, *, fs_ctx: Dict[str, str] | None = None
- ) -> Dict[str, Any]:
- """Return multi-write sync status for a file or directory path."""
- return await self.run("system_sync_status", path, ctx=_fs_ctx_or_default(path, fs_ctx))
-
- async def system_sync_retry(
- self, path: str, *, fs_ctx: Dict[str, str] | None = None
- ) -> Dict[str, Any]:
- """Retry pending multi-write sync work for a file or directory path."""
- return await self.run("system_sync_retry", path, ctx=_fs_ctx_or_default(path, fs_ctx))
-
# -- pathlock async wrappers ------------------------------------------------
async def pathlock_acquire_exact(
diff --git a/openviking/pyagfs/protocols.py b/openviking/pyagfs/protocols.py
index 020151cb8..5c2a484b2 100644
--- a/openviking/pyagfs/protocols.py
+++ b/openviking/pyagfs/protocols.py
@@ -118,9 +118,3 @@ class AGFSSyncClientProtocol(Protocol):
continuation_token: str | None = None,
) -> Dict[str, Any]:
"""Return one page of flat glob matches for the given AGFS directory."""
-
- def system_sync_status(self, path: str) -> Dict[str, Any]:
- """Return multi-write sync status for a file or directory path."""
-
- def system_sync_retry(self, path: str) -> Dict[str, Any]:
- """Retry pending multi-write sync work for a file or directory path."""
diff --git a/openviking/server/auth/plugins/api_key.py b/openviking/server/auth/plugins/api_key.py
index 686444728..75d44bb92 100644
--- a/openviking/server/auth/plugins/api_key.py
+++ b/openviking/server/auth/plugins/api_key.py
@@ -24,8 +24,6 @@ _API_KEY_ROOT_ALLOWED_PREFIXES = (
"/api/v1/observer",
"/api/v1/console",
"/api/v1/tasks",
- "/api/v1/system/backend",
- "/api/v1/system/sync",
)
diff --git a/openviking/server/routers/system.py b/openviking/server/routers/system.py
index 0ef245f79..78b62f4d6 100644
--- a/openviking/server/routers/system.py
+++ b/openviking/server/routers/system.py
@@ -38,19 +38,13 @@ def _is_ready_check_ok(value) -> bool:
async def _probe_agfs_readiness() -> dict[str, object]:
- """Return structured AGFS readiness, including multi-write sync health when available."""
+ """Return structured AGFS filesystem readiness."""
viking_fs = get_viking_fs()
checks: dict[str, object] = {}
await viking_fs.ls("viking://", ctx=None)
checks["filesystem"] = "ok"
- try:
- await viking_fs.system_sync_status("viking://", ctx=None)
- checks["multiwrite_sync"] = "ok"
- except (AGFSInvalidOperationError, AGFSNotSupportedError):
- checks["multiwrite_sync"] = "not_supported"
-
return {"status": "ok", "checks": checks}
@@ -126,7 +120,7 @@ async def readiness_check(request: Request):
checks = {}
- # 1. AGFS: probe filesystem access and multi-write sync health
+ # 1. AGFS: probe filesystem access
try:
checks["agfs"] = await _probe_agfs_readiness()
except Exception as e:
@@ -220,12 +214,6 @@ class ConsistencyRequest(BaseModel):
uri: str
-class BackendSyncRequest(BaseModel):
- """Request model for backend sync status and retry operations."""
-
- uri: str
-
-
@router.post("/api/v1/system/wait", tags=["system"])
async def wait_processed(
request: WaitRequest,
@@ -250,51 +238,3 @@ async def check_consistency(
ctx=ctx,
)
return Response(status="ok", result=result)
-
-
[email protected]("/api/v1/system/backend/sync-status", tags=["system"])
-async def backend_sync_status(
- request: BackendSyncRequest,
- ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),
-):
- """Return multi-write backend sync status for a Viking URI subtree."""
- service = get_service()
- uri = validate_request_viking_uri(resolve_path_variables(request.uri), ctx)
- result = await service.fs.system_sync_status(uri, ctx=ctx)
- return Response(status="ok", result=result)
-
-
[email protected]("/api/v1/system/backend/sync-retry", tags=["system"])
-async def backend_sync_retry(
- request: BackendSyncRequest,
- ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),
-):
- """Retry pending multi-write backend sync work for a Viking URI subtree."""
- service = get_service()
- uri = validate_request_viking_uri(resolve_path_variables(request.uri), ctx)
- result = await service.fs.system_sync_retry(uri, ctx=ctx)
- return Response(status="ok", result=result)
-
-
[email protected]("/api/v1/system/sync/{sync_path:path}", tags=["system"])
-async def admin_sync_status(
- sync_path: str,
- ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),
-):
- """Return multi-write backend sync status for one URI subtree through the admin API."""
- service = get_service()
- uri = validate_request_viking_uri(resolve_path_variables(sync_path), ctx)
- result = await service.fs.system_sync_status(uri, ctx=ctx)
- return Response(status="ok", result=result)
-
-
[email protected]("/api/v1/system/sync/{sync_path:path}/retry", tags=["system"])
-async def admin_sync_retry(
- sync_path: str,
- ctx: RequestContext = require_role(Role.ROOT, Role.ADMIN),
-):
- """Retry pending multi-write backend sync work for one URI subtree through the admin API."""
- service = get_service()
- uri = validate_request_viking_uri(resolve_path_variables(sync_path), ctx)
- result = await service.fs.system_sync_retry(uri, ctx=ctx)
- return Response(status="ok", result=result)
diff --git a/openviking/service/fs_service.py b/openviking/service/fs_service.py
index 163e75e66..ee32386a7 100644
--- a/openviking/service/fs_service.py
+++ b/openviking/service/fs_service.py
@@ -1062,16 +1062,6 @@ class FSService:
viking_fs = self._ensure_initialized()
await viking_fs._ensure_access(uri, ctx, action=AclAction.WRITE)
- async def system_sync_status(self, uri: str, ctx: RequestContext) -> Dict[str, Any]:
- """Return multi-write sync status for one Viking URI subtree."""
- viking_fs = self._ensure_initialized()
- return await viking_fs.system_sync_status(uri, ctx=ctx)
-
- async def system_sync_retry(self, uri: str, ctx: RequestContext) -> Dict[str, Any]:
- """Retry multi-write sync work for one Viking URI subtree."""
- viking_fs = self._ensure_initialized()
- return await viking_fs.system_sync_retry(uri, ctx=ctx)
-
async def read(self, uri: str, ctx: RequestContext, offset: int = 0, limit: int = -1) -> str:
"""Read file content. Accepts a Viking URI or a 32-char hex vector record id."""
viking_fs = self._ensure_initialized()
diff --git a/openviking/storage/viking_fs/_snapshot.py b/openviking/storage/viking_fs/_snapshot.py
index 92dc4df79..c5f561947 100644
--- a/openviking/storage/viking_fs/_snapshot.py
+++ b/openviking/storage/viking_fs/_snapshot.py
@@ -121,30 +121,6 @@ class _SnapshotMixin:
def _restore_reindex_context(ctx: RequestContext) -> RequestContext:
return replace(ctx, bypass_acl=True)
- async def system_sync_status(
- self, uri: str, ctx: Optional[RequestContext] = None
- ) -> Dict[str, Any]:
- """Return multi-write sync status for one Viking URI subtree."""
- await self._ensure_access(uri, ctx)
- real_ctx = self._ctx_or_default(ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- return await self._async_agfs.system_sync_status(
- path,
- fs_ctx={"account_id": real_ctx.account_id},
- )
-
- async def system_sync_retry(
- self, uri: str, ctx: Optional[RequestContext] = None
- ) -> Dict[str, Any]:
- """Retry multi-write sync for one Viking URI subtree."""
- await self._ensure_access(uri, ctx, action=AclAction.WRITE)
- real_ctx = self._ctx_or_default(ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- return await self._async_agfs.system_sync_retry(
- path,
- fs_ctx={"account_id": real_ctx.account_id},
- )
-
async def get_gitignore(self, ctx: Optional[RequestContext] = None) -> str:
"""Return the account-level .ovgitignore content, or an empty string if absent.
diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.py
index 6f6fc7b83..6acc259e1 100644
--- a/openviking/utils/agfs_utils.py
+++ b/openviking/utils/agfs_utils.py
@@ -41,7 +41,13 @@ class RagfsBindingConfig:
cachefs_backend = getattr(cachefs, "backend", "local")
pathlock = self.agfs.pathlock.model_dump(mode="json", exclude_none=True)
pathlock_uses_cache = pathlock.get("provider") == "cache"
- uses_runtime = cachefs_backend == "cache" or queue_backend == "cache" or pathlock_uses_cache
+ backups = getattr(self.agfs, "backups", None)
+ uses_runtime = (
+ cachefs_backend == "cache"
+ or queue_backend == "cache"
+ or pathlock_uses_cache
+ or _get_config_value(backups, "provider", "filesystem") == "cache"
+ )
if (
pathlock_uses_cache
and self.cache is not None
@@ -60,8 +66,8 @@ class RagfsBindingConfig:
else:
if uses_runtime:
raise ValueError(
- "top-level cache config is required when CacheFS, QueueFS, or cache-backed PathLock "
- "uses the shared Runtime"
+ "top-level cache config is required when CacheFS, QueueFS, cache-backed PathLock, "
+ "or cache-backed multi-write uses the shared Runtime"
)
cache_config = _disabled_cache_config(cachefs)
binding_config: Dict[str, Any] = {
@@ -301,10 +307,8 @@ def _generate_plugin_config(
# Check for multi-write configuration
backups_config = getattr(agfs_config, "backups", None)
redirects_config = getattr(agfs_config, "redirects", None)
- if redirects_config is not None and backups_config is None:
- raise ValueError(
- "redirects requires backups; single-backend mode does not support redirects"
- )
+ if redirects_config is not None:
+ logger.warning("storage.agfs.redirects is deprecated and ignored")
# Build primary backend plugin config
primary_plugin_config: Dict[str, Any] = {}
@@ -326,12 +330,6 @@ def _generate_plugin_config(
mount_config["server_encryption_enabled"] = server_encryption_enabled
mount_config["primary_encryption_enabled"] = server_encryption_enabled
- # Serialize redirect policies
- if redirects_config is not None:
- mount_config["primary_redirects"] = [
- _serialize_redirect_policy(p) for p in redirects_config
- ]
-
# Determine the plugin type name for the primary backend
if backend == "local":
plugin_name = "localfs"
@@ -370,6 +368,9 @@ def _get_config_value(config: Any, key: str, default: Any = None) -> Any:
def _get_backend_specific_params(item: Any) -> Any:
"""Return backend-specific nested params from one backup item using the backend name key."""
+ params = _get_config_value(item, "params")
+ if params is not None:
+ return params
backend_params = _get_config_value(item, "backend_params")
if backend_params is not None:
return backend_params
@@ -482,21 +483,42 @@ def _normalize_backup_item(item: Any, data_path: Path) -> Dict[str, Any]:
def _serialize_backups_config(backups_config: Any, data_path: Path) -> Dict[str, Any]:
- """Serialize raw multi-write backups config with top-level passthrough and normalized items."""
+ """Serialize V2 backups while warning about and stripping legacy fields."""
result = _dump_config_object(backups_config)
- result.setdefault("sync_type", "async")
- result["items"] = [
- _normalize_backup_item(item, data_path)
- for item in _get_config_value(backups_config, "items", [])
- ]
+ deprecated_fields = []
+ for field_name in (
+ "sync_type",
+ "write_ack_count",
+ "write_ack_timeout_ms",
+ "write_concurrency",
+ "retry_interval_ms",
+ "retry_backoff_base_ms",
+ "retry_max_retries_per_round",
+ "retry_quarantine_after_failures",
+ "read_probe_cache_ttl_ms",
+ ):
+ if field_name in result:
+ deprecated_fields.append(field_name)
+ result.pop(field_name)
+
+ items = []
+ for item in _get_config_value(backups_config, "items", []):
+ normalized = _normalize_backup_item(item, data_path)
+ for field_name in ("timeout", "operations", "excludes"):
+ if field_name in normalized:
+ deprecated_fields.append(f"items.{field_name}")
+ normalized.pop(field_name)
+ items.append(normalized)
+ result["items"] = items
+
+ if deprecated_fields:
+ logger.warning(
+ "Deprecated backups configuration fields are ignored: %s",
+ ", ".join(dict.fromkeys(deprecated_fields)),
+ )
return result
-def _serialize_redirect_policy(policy: Any) -> Dict[str, Any]:
- """Serialize one raw redirect/exclude policy object to a dict."""
- return _dump_config_object(policy)
-
-
def _build_git_config_dict(git_config: Any, storage_path: Path) -> Dict[str, Any]:
"""Build an in-memory git config dict consumed by the ragfs binding.
diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.py
index a9afb07aa..fb8de031b 100644
--- a/openviking_cli/utils/config/agfs_config.py
+++ b/openviking_cli/utils/config/agfs_config.py
@@ -446,7 +446,7 @@ class AGFSConfig(BaseModel):
default=None, description="Multi-write backups configuration. None = single backend mode."
)
redirects: Optional[List[dict[str, Any]]] = Field(
- default=None, description="Primary redirect policies."
+ default=None, description="[Deprecated] Primary redirect policies. Ignored by V2."
)
@model_validator(mode="after")
@@ -494,9 +494,38 @@ class AGFSConfig(BaseModel):
"db_path/queue_db_path will be ignored."
)
- if self.redirects is not None and self.backups is None:
- raise ValueError(
- "redirects requires backups; single-backend mode does not support redirects"
- )
+ if self.backups is not None:
+ initial_partitions = self.backups.get("initial_partitions", 16)
+ if (
+ not isinstance(initial_partitions, int)
+ or isinstance(initial_partitions, bool)
+ or not 1 <= initial_partitions <= 1024
+ ):
+ raise ValueError("backups initial_partitions must be between 1 and 1024")
+
+ checkpoint_interval_secs = self.backups.get("checkpoint_interval_secs", 86400)
+ if (
+ not isinstance(checkpoint_interval_secs, int)
+ or isinstance(checkpoint_interval_secs, bool)
+ or checkpoint_interval_secs < 60
+ ):
+ raise ValueError("backups checkpoint_interval_secs must be at least 60")
+
+ if self.backups.get("provider", "filesystem") not in {"filesystem", "cache"}:
+ raise ValueError("backups provider must be 'filesystem' or 'cache'")
+
+ names = set()
+ for item in self.backups.get("items", []):
+ if not isinstance(item, dict):
+ raise ValueError("backups items must be objects")
+ name = item.get("name")
+ if not isinstance(name, str) or not name.strip():
+ raise ValueError("backup name must not be empty")
+ name = name.strip()
+ if name == "primary":
+ raise ValueError("backup name 'primary' is reserved")
+ if name in names:
+ raise ValueError("backup names must be unique")
+ names.add(name)
return self
diff --git a/tests/agfs/test_async_client.py b/tests/agfs/test_async_client.py
index 9c8b025d4..cb0897771 100644
--- a/tests/agfs/test_async_client.py
+++ b/tests/agfs/test_async_client.py
@@ -6,10 +6,17 @@ import threading
import pytest
-from openviking.pyagfs import AsyncAGFSClient
+from openviking.pyagfs import AsyncAGFSClient, RAGFSBindingClient
from openviking.storage.viking_vector_index_backend import _AsyncVectorAdapter
+def test_removed_sync_methods_are_not_exposed():
+ """Removed sync methods must stay absent from Python and native clients."""
+ for client_type in (AsyncAGFSClient, RAGFSBindingClient):
+ assert not hasattr(client_type, "system_sync_status")
+ assert not hasattr(client_type, "system_sync_retry")
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("backend", ["agfs", "vector"])
async def test_async_agfs_client_hides_threadpool(backend):
diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.py
index 76a73cbb7..a632835c8 100644
--- a/tests/misc/test_config_validation.py
+++ b/tests/misc/test_config_validation.py
@@ -440,59 +440,62 @@ def test_generate_plugin_config_forwards_queuefs_runtime_options():
assert plugins["queuefs"]["config"]["busy_timeout_ms"] == 1234
-def test_agfs_redirects_require_backups():
- """Single-backend mode must reject redirect policies during config validation."""
- with pytest.raises(ValueError, match="redirects requires backups"):
- AGFSConfig(
- path="/tmp/ov-test",
- backend="local",
- redirects=[
- {
- "type": "FileExtensionPolicy",
- "extensions": ["(md)"],
- "target": ["s3-backup"],
- }
- ],
- )
+def test_agfs_redirects_without_backups_are_parse_only():
+ """Legacy redirects must remain constructible without enabling multi-write."""
+ redirects = [
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": ["(md)"],
+ "target": ["missing-backup"],
+ }
+ ]
+
+ config = AGFSConfig(
+ path="/tmp/ov-test",
+ backend="local",
+ redirects=redirects,
+ )
+ assert config.redirects == redirects
-def test_generate_plugin_config_rejects_redirects_without_backups(tmp_path):
- """Runtime plugin config generation must also reject redirect-only configs."""
- config = type(
- "RedirectOnlyConfig",
- (),
+
+def test_generate_plugin_config_ignores_redirects_without_backups(tmp_path):
+ """Legacy redirects must not alter a single-backend mount config."""
+ redirects = [
{
- "backend": "local",
- "s3": None,
- "backups": None,
- "redirects": [
- type(
- "RedirectPolicy",
- (),
- {
- "type": "FileExtensionPolicy",
- "extensions": ["(md)"],
- "target": ["s3-backup"],
- },
- )()
- ],
- "queuefs": type(
- "QueueConfig",
- (),
- {
- "mode": "shared",
- "backend": "sqlite",
- "db_path": None,
- "recover_stale_sec": 0,
- "busy_timeout_ms": 5000,
- },
- )(),
- "queue_db_path": None,
- },
- )()
+ "type": "FileExtensionPolicy",
+ "extensions": ["(md)"],
+ "target": ["missing-backup"],
+ }
+ ]
+ config = AGFSConfig(path=str(tmp_path), backend="local", redirects=redirects)
+ plain_config = AGFSConfig(path=str(tmp_path), backend="local")
+
+ mount_config = _generate_plugin_config(config, tmp_path)["localfs"]["config"]
+ plain_mount_config = _generate_plugin_config(plain_config, tmp_path)["localfs"]["config"]
+
+ assert mount_config == plain_mount_config
+
+
+def test_generate_plugin_config_ignores_redirects_with_backups(tmp_path):
+ """Legacy redirects must remain accepted without reaching Rust."""
+ redirects = [
+ {
+ "type": "FileExtensionPolicy",
+ "extensions": ["(md)"],
+ "target": ["missing-backup"],
+ }
+ ]
+ config = AGFSConfig(
+ path=str(tmp_path),
+ backend="local",
+ backups={"items": [{"name": "mem-backup", "backend": "memory"}]},
+ redirects=redirects,
+ )
+
+ mount_config = _generate_plugin_config(config, tmp_path)["localfs"]["config"]
- with pytest.raises(ValueError, match="redirects requires backups"):
- _generate_plugin_config(config, tmp_path)
+ assert "primary_redirects" not in mount_config
def test_generate_plugin_config_passes_multiwrite_encryption_flag(tmp_path):
@@ -517,20 +520,85 @@ def test_generate_plugin_config_passes_multiwrite_encryption_flag(tmp_path):
assert mount_config["primary_encryption_enabled"] is True
-def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path):
- """Plugin config generation should normalize backup params while preserving top-level multi-write fields."""
+def test_generate_plugin_config_preserves_v2_backups_without_sync_type(tmp_path):
+ """V2 backup fields must pass through without injecting a V1 sync mode."""
+ config = AGFSConfig(
+ path=str(tmp_path),
+ backend="local",
+ backups={
+ "initial_partitions": 32,
+ "checkpoint_interval_secs": 120,
+ "provider": "filesystem",
+ "items": [{"name": "mem-backup", "backend": "memory"}],
+ },
+ )
+
+ backups = _generate_plugin_config(config, tmp_path)["localfs"]["config"]["backups"]
+
+ assert backups["initial_partitions"] == 32
+ assert backups["checkpoint_interval_secs"] == 120
+ assert backups["provider"] == "filesystem"
+ assert "sync_type" not in backups
+
+
[email protected]("provider", ["filesystem", "cache"])
+def test_agfs_config_accepts_supported_v2_backup_providers(provider):
+ """AGFS config must accept each specified metadata provider."""
+ config = AGFSConfig(
+ path="/tmp/ov-test",
+ backend="local",
+ backups={"provider": provider, "items": []},
+ )
+
+ assert config.backups["provider"] == provider
+
+
+ ("backups", "expected"),
+ [
+ ({"initial_partitions": 0, "items": []}, "initial_partitions"),
+ ({"initial_partitions": 1025, "items": []}, "initial_partitions"),
+ ({"checkpoint_interval_secs": 59, "items": []}, "checkpoint_interval_secs"),
+ ({"provider": "memory", "items": []}, "provider"),
+ ({"provider": "redis", "items": []}, "provider"),
+ ({"items": [{"name": " ", "backend": "memory"}]}, "name"),
+ ({"items": [{"name": "primary", "backend": "memory"}]}, "primary"),
+ (
+ {
+ "items": [
+ {"name": "backup-a", "backend": "memory"},
+ {"name": "backup-a", "backend": "memory"},
+ ]
+ },
+ "unique",
+ ),
+ ],
+)
+def test_agfs_config_rejects_invalid_v2_backups(backups, expected):
+ """AGFS config must reject invalid V2 backup fields before Rust startup."""
+ with pytest.raises(ValueError, match=expected):
+ AGFSConfig(path="/tmp/ov-test", backend="local", backups=backups)
+
+
+def test_generate_plugin_config_strips_legacy_multiwrite_fields(tmp_path):
+ """Plugin config generation should strip legacy fields before calling Rust."""
explicit_backup_dir = tmp_path / "backup-local-no-mkdir"
config = AGFSConfig(
path=str(tmp_path),
backend="local",
backups={
- "retry_interval_ms": 1234,
- "retry_backoff_base_ms": 55,
+ "sync_type": "async",
+ "write_ack_count": 1,
+ "write_ack_timeout_ms": 1000,
+ "write_concurrency": 2,
"items": [
{
"name": "local-explicit",
"backend": "local",
- "local": {"workspace": str(explicit_backup_dir)},
+ "params": {"workspace": str(explicit_backup_dir)},
+ "timeout": 10,
+ "operations": [{"operation": "write", "priority": 1}],
+ "excludes": [{"type": "FileExtensionPolicy", "extensions": [".tmp"]}],
},
{
"name": "local-default",
@@ -558,12 +626,17 @@ def test_generate_plugin_config_materializes_multiwrite_backups(tmp_path):
plugins = _generate_plugin_config(config, tmp_path)
mount_backups = plugins["localfs"]["config"]["backups"]
- assert mount_backups["retry_interval_ms"] == 1234
- assert mount_backups["retry_backoff_base_ms"] == 55
+ assert {
+ "sync_type",
+ "write_ack_count",
+ "write_ack_timeout_ms",
+ "write_concurrency",
+ }.isdisjoint(mount_backups)
explicit_local, default_local, s3_backup = mount_backups["items"]
assert explicit_local["backend"] == "localfs"
assert explicit_local["params"]["local_dir"] == str(explicit_backup_dir)
+ assert {"timeout", "operations", "excludes"}.isdisjoint(explicit_local)
assert not explicit_backup_dir.exists()
assert default_local["backend"] == "localfs"
@@ -705,6 +778,23 @@ def test_ragfs_binding_enables_runtime_for_queuefs_cache_backend(tmp_path):
assert binding["cache"]["runtime_enabled"] is True
+def test_ragfs_binding_enables_runtime_for_multiwrite_cache_provider(tmp_path):
+ """Multi-write cache metadata should enable Runtime without CacheFS."""
+ agfs_config = AGFSConfig(
+ path=str(tmp_path),
+ backend="local",
+ backups={"provider": "cache", "items": []},
+ )
+
+ binding = RagfsBindingConfig(
+ agfs=agfs_config,
+ cache=CacheConfig(provider="redis", params={}),
+ ).to_binding_dict()
+
+ assert binding["cache"]["enabled"] is False
+ assert binding["cache"]["runtime_enabled"] is True
+
+
def test_agfs_pathlock_config_validates_provider_and_expiry(tmp_path):
"""PathLock config validates Redis namespace and expiry boundaries."""
config = AGFSConfig(
diff --git a/tests/server/test_admin_api.py b/tests/server/test_admin_api.py
index 739f97899..365a7f6f1 100644
--- a/tests/server/test_admin_api.py
+++ b/tests/server/test_admin_api.py
@@ -34,7 +34,7 @@ from openviking.server.user_config import (
user_config_backup_uri,
)
from openviking.service.core import OpenVikingService
-from openviking.service.deletion import setup_deletion
+from openviking.service.deletion import DeletionService, setup_deletion
from openviking.service.task_store import (
SYSTEM_TASK_ACCOUNT_ID,
SYSTEM_TASK_USER_ID,
@@ -311,6 +311,28 @@ async def _wait_for_task(client: httpx.AsyncClient, task_id: str) -> dict:
# ---- Account CRUD ----
+async def test_delete_account_files_disables_automatic_pathlock(monkeypatch):
+ """Account cleanup must bypass the deleted tree's automatic PathLock."""
+ path = "/local/account-to-delete"
+ agfs = Mock()
+ agfs.rm = AsyncMock()
+ agfs.stat = AsyncMock(side_effect=AGFSNotFoundError(path))
+ service = Mock()
+ service.viking_fs._async_agfs = agfs
+ tracker = Mock()
+ monkeypatch.setattr("openviking.service.deletion.get_task_tracker", lambda: tracker)
+ deletion = DeletionService(
+ service=service,
+ manager=Mock(),
+ service_loop=asyncio.get_running_loop(),
+ )
+
+ await deletion._delete_account_files("account-to-delete")
+
+ agfs.rm.assert_awaited_once_with(path, recursive=True, auto_pathlock=False)
+ tracker.forget_account_tasks.assert_called_once_with("account-to-delete")
+
+
@pytest_asyncio.fixture
async def template_account(lightweight_admin_client):
account_id = _uid()
@@ -324,17 +346,31 @@ async def template_account(lightweight_admin_client):
async def test_studio_file_configuration_permissions_revision_and_overrides(
- lightweight_admin_client, lightweight_admin_app, template_account, tmp_path, monkeypatch,
+ lightweight_admin_client,
+ lightweight_admin_app,
+ template_account,
+ tmp_path,
+ monkeypatch,
):
from openviking.config.scope import ConfigScope
from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton
account_id, admin_headers = template_account
config_path = tmp_path / "ov.conf"
- config_path.write_text(json.dumps({"vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "file-secret"}}))
+ config_path.write_text(
+ json.dumps({"vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "file-secret"}})
+ )
monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file", config_path)
manager = lightweight_admin_app.state.fake_service.runtime_config_manager
- await manager.patch_account(account_id, {"vlm": {"model": "override-model", "credentials": [{"provider": "openai", "api_key": "account-secret"}]}})
+ await manager.patch_account(
+ account_id,
+ {
+ "vlm": {
+ "model": "override-model",
+ "credentials": [{"provider": "openai", "api_key": "account-secret"}],
+ }
+ },
+ )
url = "/api/v1/admin/configuration"
params = {"source": "file", "account_id": account_id}
denied = await lightweight_admin_client.get(url, params=params, headers=admin_headers)
@@ -355,15 +391,28 @@ async def test_studio_file_configuration_permissions_revision_and_overrides(
assert override_response.json()["result"]["overrides"]["cluster"] == ["query_planner"]
content = json.loads(result["content"])
content["rerank"] = {
- "provider": "jev", "api_key": "jev-secret", "mode": "choice", "threshold": 0
+ "provider": "jev",
+ "api_key": "jev-secret",
+ "mode": "choice",
+ "threshold": 0,
}
body = {"revision": result["revision"], "content": json.dumps(content)}
- assert (await lightweight_admin_client.patch(url, params={"source": "file"}, headers=admin_headers, json=body)).status_code == 403
- saved = await lightweight_admin_client.patch(url, params={"source": "file"}, headers=root_headers(), json=body)
+ assert (
+ await lightweight_admin_client.patch(
+ url, params={"source": "file"}, headers=admin_headers, json=body
+ )
+ ).status_code == 403
+ saved = await lightweight_admin_client.patch(
+ url, params={"source": "file"}, headers=root_headers(), json=body
+ )
assert saved.status_code == 200, saved.text
assert saved.json()["result"]["restart_required"]
assert json.loads(config_path.read_text())["rerank"]["provider"] == "jev"
- assert (await lightweight_admin_client.patch(url, params={"source": "file"}, headers=root_headers(), json=body)).status_code == 400
+ assert (
+ await lightweight_admin_client.patch(
+ url, params={"source": "file"}, headers=root_headers(), json=body
+ )
+ ).status_code == 400
settings = await manager.get_settings(ConfigScope.account(account_id))
assert settings["vlm"]["model"] == "override-model"
@@ -987,12 +1036,10 @@ async def test_account_memory_templates_range_expressions_reach_file_body(
fs = lightweight_admin_app.state.fake_service.viking_fs
snapshot = await resolve_account_memory_registry(fs, account_id, MemoryTypeRegistry())
schema = snapshot.get("events")
- context = ExtractContext(
- [
- Message(id="m1", role="user", parts=[TextPart("Selected source message")]),
- Message(id="m2", role="user", parts=[TextPart("Unrelated source message")]),
- ]
- )
+ context = ExtractContext([
+ Message(id="m1", role="user", parts=[TextPart("Selected source message")]),
+ Message(id="m2", role="user", parts=[TextPart("Unrelated source message")]),
+ ])
rendered = MemoryFileUtils.write(
MemoryFile(memory_type="events", extra_fields={"ranges": "0", "summary": "summary"}),
content_template=schema.content_template,
@@ -1315,33 +1362,29 @@ async def test_account_memory_templates_locked_fields_cannot_be_modified(
for key, value in defaults.items()
if key not in editable
]
- invalid.extend(
- [
- {"unknown": "ignored?"},
- {"Description": "wrong key"},
- {"_updated_at": "forged"},
- {"_account_content_template": False},
- {"deployment_content_template": "forged"},
- {"enabled": 1},
- {"fields": [{"name": "renamed_field", "description": "changed"}]},
- {"description": None},
- {"description": " \n"},
- {"fields": [None]},
- ]
- )
+ invalid.extend([
+ {"unknown": "ignored?"},
+ {"Description": "wrong key"},
+ {"_updated_at": "forged"},
+ {"_account_content_template": False},
+ {"deployment_content_template": "forged"},
+ {"enabled": 1},
+ {"fields": [{"name": "renamed_field", "description": "changed"}]},
+ {"description": None},
+ {"description": " \n"},
+ {"fields": [None]},
+ ])
for field in defaults["fields"]:
invalid.extend(
{"fields": [{"name": field["name"], key: None if value is not None else "changed"}]}
for key, value in field.items()
if key not in {"name", "description"}
)
- invalid.extend(
- [
- {"fields": [{"name": field["name"], "unknown": "ignored?"}]},
- {"fields": [{"name": field["name"], "description": ""}]},
- {"fields": [{"name": field["name"]}, {"name": field["name"]}]},
- ]
- )
+ invalid.extend([
+ {"fields": [{"name": field["name"], "unknown": "ignored?"}]},
+ {"fields": [{"name": field["name"], "description": ""}]},
+ {"fields": [{"name": field["name"]}, {"name": field["name"]}]},
+ ])
if memory_type == "events":
invalid.extend(
{"fields": [{"name": name, "description": "changed"}]} for name in ("goal", "ranges")
@@ -2131,6 +2174,33 @@ async def test_list_users_for_default_account_without_users_file_returns_empty_l
assert resp.json()["result"] == []
+async def test_list_accounts_with_watcher_uses_memory(
+ lightweight_admin_client: httpx.AsyncClient,
+ lightweight_admin_app: FastAPI,
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """A running watcher keeps list requests on the in-memory fast path."""
+ loop = asyncio.get_running_loop()
+ watch_task = loop.create_future()
+
+ async def _unexpected_read(_path: str):
+ raise AssertionError("list accounts should not read AGFS while watcher is running")
+
+ manager = lightweight_admin_app.state.api_key_manager
+ manager._store._watch_task = watch_task
+ monkeypatch.setattr(manager._store, "_read_json", _unexpected_read)
+ try:
+ resp = await lightweight_admin_client.get("/api/v1/admin/accounts", headers=root_headers())
+ users_resp = await lightweight_admin_client.get(
+ "/api/v1/admin/accounts/default/users", headers=root_headers()
+ )
+ finally:
+ watch_task.cancel()
+
+ assert resp.status_code == 200, resp.text
+ assert users_resp.status_code == 200, users_resp.text
+
+
async def test_identity_settings_refreshes_a_stale_registry_on_demand(
admin_client: httpx.AsyncClient,
admin_app: FastAPI,
@@ -2308,22 +2378,18 @@ async def test_delete_account(
account_id=SYSTEM_TASK_ACCOUNT_ID,
user_id=SYSTEM_TASK_USER_ID,
)
- await queue.enqueue(
- {
- "task_id": user_cleanup.task_id,
- "account_id": acct,
- "user_id": "alice",
- "target": {"account_id": acct, "user_id": "bob"},
- }
- )
- await queue.enqueue(
- {
- "task_id": late_task_id,
- "account_id": SYSTEM_TASK_ACCOUNT_ID,
- "user_id": SYSTEM_TASK_USER_ID,
- "target": {"account_id": acct, "user_id": "bob"},
- }
- )
+ await queue.enqueue({
+ "task_id": user_cleanup.task_id,
+ "account_id": acct,
+ "user_id": "alice",
+ "target": {"account_id": acct, "user_id": "bob"},
+ })
+ await queue.enqueue({
+ "task_id": late_task_id,
+ "account_id": SYSTEM_TASK_ACCOUNT_ID,
+ "user_id": SYSTEM_TASK_USER_ID,
+ "target": {"account_id": acct, "user_id": "bob"},
+ })
skipped = await _wait_for_task(admin_client, late_task_id)
assert skipped["status"] == "completed"
assert skipped["result"] == {"deleted": not recreated, "stale": True}
@@ -3129,10 +3195,7 @@ async def test_trusted_mode_admin_can_register_user_in_own_account(
headers=trusted_headers(account=acct, user="alice", include_api_key=True),
)
assert users.status_code == 200
- assert all(
- user["api_key_available"] is False
- for user in users.json()["result"]
- )
+ assert all(user["api_key_available"] is False for user in users.json()["result"])
rotated = await trusted_admin_client.post(
f"/api/v1/admin/accounts/{acct}/users/bob/key",
@@ -3449,12 +3512,10 @@ async def test_server_restart_requires_root_and_valid_revision(
server_overrides = {"host": cli_host}
lightweight_admin_app.state.server_config_overrides = server_overrides
path.write_text(
- json.dumps(
- {
- "server": server,
- "vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "test-key"},
- }
- )
+ json.dumps({
+ "server": server,
+ "vlm": {"provider": "openai", "model": "gpt-4o", "api_key": "test-key"},
+ })
)
monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file", path)
monkeypatch.setattr(OpenVikingConfigSingleton, "_config_file_content", path.read_bytes())
diff --git a/tests/server/test_auth.py b/tests/server/test_auth.py
index 06c941a65..dfe92bb2b 100644
--- a/tests/server/test_auth.py
+++ b/tests/server/test_auth.py
@@ -16,6 +16,7 @@ from starlette.requests import Request
from openviking.server.api_keys import APIKeyManager
from openviking.server.app import create_app
from openviking.server.auth import get_request_context, resolve_identity
+from openviking.server.auth.plugins.api_key import _api_key_root_can_access_path
from openviking.server.auth.registry import get_registry
from openviking.server.config import ServerConfig, _is_localhost, validate_server_config
from openviking.server.dependencies import set_service
@@ -81,6 +82,20 @@ def _set_fake_task_tracker():
ROOT_KEY = "root-secret-key-for-testing-only-1234567890abcdef"
+ "path",
+ [
+ "/api/v1/system/backend/sync-status",
+ "/api/v1/system/backend/sync-retry",
+ "/api/v1/system/sync/viking://resources",
+ "/api/v1/system/sync/viking://resources/retry",
+ ],
+)
+def test_root_key_allowlist_rejects_removed_sync_paths(path: str):
+ """Removed sync paths must stay outside the API-key root allowlist."""
+ assert not _api_key_root_can_access_path(path)
+
+
def _make_request(
path: str,
headers: dict[str, str] | None = None,
@@ -390,46 +405,6 @@ async def test_auth_on_multiple_endpoints(auth_client: httpx.AsyncClient):
assert tenant_resp.json()["error"]["code"] == "PERMISSION_DENIED"
-async def test_admin_sync_route_accepts_root_key(auth_client: httpx.AsyncClient, auth_service):
- """ROOT keys should be allowed to call the system sync admin route."""
- calls: list[str] = []
-
- async def _fake_system_sync_status(uri: str, ctx):
- calls.append(uri)
- return {"path": uri, "entry_count": 1}
-
- auth_service.fs.system_sync_status = _fake_system_sync_status
-
- resp = await auth_client.get(
- "/api/v1/system/sync/viking://resources",
- headers={"X-API-Key": ROOT_KEY},
- )
- assert resp.status_code == 200
- assert resp.json()["result"] == {"path": "viking://resources", "entry_count": 1}
- assert calls == ["viking://resources"]
-
-
-async def test_admin_sync_route_rejects_user_key(auth_client: httpx.AsyncClient, user_key: str):
- """Regular user keys must not access the system sync admin route."""
- resp = await auth_client.get(
- "/api/v1/system/sync/viking://resources",
- headers={"X-API-Key": user_key},
- )
- assert resp.status_code == 403
- assert resp.json()["error"]["code"] == "PERMISSION_DENIED"
-
- tenant_resp = await auth_client.get(
- "/api/v1/fs/ls?uri=viking://",
- headers={
- "X-API-Key": ROOT_KEY,
- "X-OpenViking-Account": "default",
- "X-OpenViking-User": "default",
- },
- )
- assert tenant_resp.status_code == 403
- assert tenant_resp.json()["error"]["code"] == "PERMISSION_DENIED"
-
-
async def test_task_endpoints_require_auth():
"""Task endpoints must reject unauthenticated callers before lookup/filtering."""
set_task_tracker(None)
@@ -1113,6 +1088,45 @@ async def test_invalid_trusted_identity_does_not_block_later_registration(
await plugin.shutdown()
+async def test_trusted_identity_registration_skips_identity_loaded_from_registry(auth_service):
+ """A restarted plugin must not queue an identity already loaded from storage."""
+ from openviking.server.auth.plugins import TrustedAuthPlugin
+
+ config = ServerConfig(auth_mode="trusted", root_api_key=ROOT_KEY)
+ app = FastAPI()
+ app.state.config = config
+ first_plugin = TrustedAuthPlugin()
+ await first_plugin.initialize(app, auth_service, config)
+ try:
+ await first_plugin._api_key_manager.ensure_trusted_identities({"acme": {"alice"}})
+ finally:
+ await first_plugin.shutdown()
+
+ restarted_app = FastAPI()
+ restarted_app.state.config = config
+ restarted_plugin = TrustedAuthPlugin()
+ await restarted_plugin.initialize(restarted_app, auth_service, config)
+ try:
+ request = _make_request(
+ "/api/v1/resources",
+ headers={
+ "X-OpenViking-Account": "acme",
+ "X-OpenViking-User": "alice",
+ },
+ auth_enabled=False,
+ auth_mode="trusted",
+ )
+ request.app.state.config = config
+
+ await restarted_plugin.resolve_identity(
+ request, x_openviking_account="acme", x_openviking_user="alice"
+ )
+
+ assert restarted_plugin._pending == {}
+ finally:
+ await restarted_plugin.shutdown()
+
+
@pytest.mark.parametrize(
"interval,path",
[(0, "/api/v1/resources"), (0, "/api/v1/admin/accounts"), (300, "/api/v1/admin/accounts")],
@@ -1210,7 +1224,14 @@ async def test_trusted_identity_registration_retries_failed_batch_without_exceed
await plugin.shutdown()
-def test_trusted_identity_registration_config_rejects_invalid_limits():
+def test_trusted_identity_registration_config_allows_disabling():
+ """A zero interval disables registration; negative values remain invalid."""
+ assert (
+ ServerConfig(
+ trusted_identity_flush_interval_seconds=0
+ ).trusted_identity_flush_interval_seconds
+ == 0
+ )
with pytest.raises(ValueError):
ServerConfig(trusted_identity_flush_interval_seconds=-1)
with pytest.raises(ValueError):
diff --git a/tests/server/test_server_health.py b/tests/server/test_server_health.py
index d0e78b960..d02d4a7c2 100644
--- a/tests/server/test_server_health.py
+++ b/tests/server/test_server_health.py
@@ -150,82 +150,24 @@ async def test_system_status(client: httpx.AsyncClient):
assert body["result"]["initialized"] is True
-async def test_backend_sync_status_endpoint(client: httpx.AsyncClient, service):
- calls: list[str] = []
-
- async def _fake_system_sync_status(uri: str, ctx):
- calls.append(uri)
- assert ctx is not None
- return {"path": uri, "entry_count": 1}
-
- service.fs.system_sync_status = _fake_system_sync_status
-
- resp = await client.post(
- "/api/v1/system/backend/sync-status",
- json={"uri": "viking://resources"},
- )
- assert resp.status_code == 200
- body = resp.json()
- assert body["status"] == "ok"
- assert body["result"] == {"path": "viking://resources", "entry_count": 1}
- assert calls == ["viking://resources"]
-
-
-async def test_backend_sync_retry_endpoint(client: httpx.AsyncClient, service):
- calls: list[str] = []
-
- async def _fake_system_sync_retry(uri: str, ctx):
- calls.append(uri)
- assert ctx is not None
- return {"path": uri, "retried": 2, "failed": 0}
-
- service.fs.system_sync_retry = _fake_system_sync_retry
-
- resp = await client.post(
- "/api/v1/system/backend/sync-retry",
- json={"uri": "viking://resources"},
- )
- assert resp.status_code == 200
- body = resp.json()
- assert body["status"] == "ok"
- assert body["result"] == {"path": "viking://resources", "retried": 2, "failed": 0}
- assert calls == ["viking://resources"]
-
-
-async def test_admin_sync_status_route(client: httpx.AsyncClient, service):
- calls: list[str] = []
-
- async def _fake_system_sync_status(uri: str, ctx):
- calls.append(uri)
- assert ctx is not None
- return {"path": uri, "entry_count": 3}
-
- service.fs.system_sync_status = _fake_system_sync_status
-
- resp = await client.get("/api/v1/system/sync/viking://resources")
- assert resp.status_code == 200
- body = resp.json()
- assert body["status"] == "ok"
- assert body["result"] == {"path": "viking://resources", "entry_count": 3}
- assert calls == ["viking://resources"]
-
-
-async def test_admin_sync_retry_route(client: httpx.AsyncClient, service):
- calls: list[str] = []
-
- async def _fake_system_sync_retry(uri: str, ctx):
- calls.append(uri)
- assert ctx is not None
- return {"path": uri, "retried": 4, "failed": 1}
-
- service.fs.system_sync_retry = _fake_system_sync_retry
-
- resp = await client.post("/api/v1/system/sync/viking://resources/retry")
- assert resp.status_code == 200
- body = resp.json()
- assert body["status"] == "ok"
- assert body["result"] == {"path": "viking://resources", "retried": 4, "failed": 1}
- assert calls == ["viking://resources"]
+ ("method", "path", "json"),
+ [
+ ("POST", "/api/v1/system/backend/sync-status", {"uri": "viking://resources"}),
+ ("POST", "/api/v1/system/backend/sync-retry", {"uri": "viking://resources"}),
+ ("GET", "/api/v1/system/sync/viking://resources", None),
+ ("POST", "/api/v1/system/sync/viking://resources/retry", None),
+ ],
+)
+async def test_removed_sync_routes_return_404(
+ client: httpx.AsyncClient,
+ method: str,
+ path: str,
+ json: dict | None,
+):
+ """Removed V1 sync routes must stay unregistered."""
+ resp = await client.request(method, path, json=json)
+ assert resp.status_code == 404
async def test_process_time_header(client: httpx.AsyncClient):
@@ -253,15 +195,16 @@ async def test_404_for_unknown_route(client: httpx.AsyncClient):
async def test_lifespan_shutdown_ignores_cancelled_service_close(monkeypatch):
class _Service:
+ def __init__(self):
+ self.viking_fs = object()
+ self._queue_manager = None
+
async def initialize(self):
pass
async def apply_agent_evolution_config(self):
pass
- viking_fs = None
- _queue_manager = None
-
async def close(self):
raise asyncio.CancelledError("shutdown")
@@ -323,9 +266,6 @@ async def test_ready_returns_200_after_initialized(monkeypatch):
async def ls(self, path, ctx=None):
return []
- async def system_sync_status(self, uri, ctx=None):
- return {"path": uri, "entry_count": 0}
-
def _get_vector_store(self):
class MockVectorStore:
async def health_check(self):
@@ -344,6 +284,10 @@ async def test_ready_returns_200_after_initialized(monkeypatch):
"openviking_cli.utils.ollama.detect_ollama_in_config",
lambda config: (False, None, None),
)
+ monkeypatch.setattr(
+ "openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance",
+ lambda: SimpleNamespace(embedding=SimpleNamespace(get_embedder=lambda: None)),
+ )
app = create_app(config=ServerConfig(), service=service)
transport = httpx.ASGITransport(app=app)
@@ -354,7 +298,7 @@ async def test_ready_returns_200_after_initialized(monkeypatch):
assert body["status"] == "ready"
assert body["checks"]["agfs"]["status"] == "ok"
assert body["checks"]["agfs"]["checks"]["filesystem"] == "ok"
- assert body["checks"]["agfs"]["checks"]["multiwrite_sync"] == "ok"
+ assert body["checks"]["agfs"]["checks"] == {"filesystem": "ok"}
async def test_slow_init_does_not_block_health(monkeypatch):
@@ -386,6 +330,7 @@ async def test_initialize_runtime_state_loads_api_key_manager(monkeypatch):
def __init__(self):
self._initialized = False
self.viking_fs = object()
+ self._queue_manager = None
async def initialize(self):
self._initialized = True
diff --git a/tests/test_config_loader.py b/tests/test_config_loader.py
index 7e8484014..08444234c 100644
--- a/tests/test_config_loader.py
+++ b/tests/test_config_loader.py
@@ -256,6 +256,14 @@ def test_generic_code_hosting_domains_include_supported_platforms():
]
+def test_example_config_loads_as_runtime_config():
+ example_path = Path(__file__).resolve().parents[1] / "examples" / "ov.conf.example"
+
+ config = OpenVikingConfig.from_dict(load_json_config(example_path))
+
+ assert config.storage.agfs.backend == "local"
+
+
def test_example_code_hosting_domains_match_runtime_defaults():
example_path = Path(__file__).resolve().parents[1] / "examples" / "ov.conf.example"
example_text = example_path.read_text(encoding="utf-8")