Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(ragfs): add CachedFileSystem and Redis/Mooncake/Yuanrong cache providers (#2520)
tuofang2d56bd71f476
diff --git a/Cargo.lock b/Cargo.lockindex d79c3a35b..8af03acc9 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1067,6 +1067,20 @@ dependencies = ["windows-sys 0.59.0",]+[[package]]+name = "combine"+version = "4.6.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"+dependencies = [+ "bytes",+ "futures-core",+ "memchr",+ "pin-project-lite",+ "tokio",+ "tokio-util",+]+[[package]]name = "compact_str"version = "0.8.1"@@ -3730,12 +3744,26 @@ dependencies = ["uuid",]+[[package]]+name = "ragfs-cache-redis"+version = "0.1.0"+dependencies = [+ "async-trait",+ "bytes",+ "ragfs",+ "redis",+ "thiserror 1.0.69",+ "tokio",+ "url",+]+[[package]]name = "ragfs-python"version = "0.1.0"dependencies = ["pyo3","ragfs",+ "ragfs-cache-redis","serde_json","tokio",]@@ -3890,6 +3918,28 @@ dependencies = ["crossbeam-utils",]+[[package]]+name = "redis"+version = "0.32.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "014cc767fefab6a3e798ca45112bccad9c6e0e218fbd49720042716c73cfef44"+dependencies = [+ "bytes",+ "cfg-if",+ "combine",+ "futures-util",+ "itoa",+ "num-bigint",+ "percent-encoding",+ "pin-project-lite",+ "ryu",+ "sha1_smol",+ "socket2 0.6.3",+ "tokio",+ "tokio-util",+ "url",+]+[[package]]name = "redox_syscall"version = "0.5.18"@@ -4365,6 +4415,12 @@ dependencies = ["digest",]+[[package]]+name = "sha1_smol"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d"+[[package]]name = "sha2"version = "0.10.9"diff --git a/Cargo.toml b/Cargo.tomlindex 9aa6944d8..2831b663d 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,5 +1,16 @@[workspace]-members = ["crates/ov_cli", "crates/ragfs", "crates/ragfs-python"]+members = [+ "crates/ov_cli",+ "crates/ragfs",+ "crates/ragfs-cache-redis",+ "crates/ragfs-python",+]+exclude = [+ "crates/ragfs-cache-mooncake",+ "crates/ragfs-cache-yuanrong",+ "crates/ragfs-cache-yuanrong-sys",+ "crates/ragfs-python-native",+]resolver = "2"[profile.release]diff --git a/MANIFEST.in b/MANIFEST.inindex d93d175a5..673316ab0 100644--- a/MANIFEST.in+++ b/MANIFEST.in@@ -10,6 +10,7 @@ include setup.pyinclude Cargo.tomlinclude Cargo.lockgraft crates/ragfs+graft crates/ragfs-cache-redisgraft crates/ragfs-pythonrecursive-include openviking *.yamldiff --git a/crates/ragfs-cache-mooncake/Cargo.toml b/crates/ragfs-cache-mooncake/Cargo.tomlnew file mode 100644index 000000000..2f90001b8--- /dev/null+++ b/crates/ragfs-cache-mooncake/Cargo.toml@@ -0,0 +1,20 @@+[package]+name = "ragfs-cache-mooncake"+version = "0.1.0"+edition = "2021"+rust-version = "1.91.1"+license = "Apache-2.0"+description = "Mooncake cache provider adapter for RAGFS"++[dependencies]+async-trait = "0.1"+bytes = "1.5"+futures = "0.3"+mooncake_store = { git = "https://github.com/kvcache-ai/Mooncake.git", rev = "1352bbec43081e461356aaecf6c70cddd826b455", optional = true }+ragfs = { path = "../ragfs", features = ["cache"] }+thiserror = "1.0"+tokio = { version = "1.38", features = ["rt-multi-thread", "sync", "time", "macros"] }++[features]+default = []+mooncake-native = ["dep:mooncake_store"]diff --git a/crates/ragfs-cache-mooncake/README.md b/crates/ragfs-cache-mooncake/README.mdnew file mode 100644index 000000000..0695fd727--- /dev/null+++ b/crates/ragfs-cache-mooncake/README.md@@ -0,0 +1,82 @@+# RAGFS Mooncake Cache Provider++This crate maps the RAGFS `CacheProvider` contract to Mooncake Store without+changing `CachedFileSystem`.++The optional official binding is pinned to Mooncake commit:++```text+1352bbec43081e461356aaecf6c70cddd826b455+```++Default tests use the same synchronous object boundary as the native binding,+so they run without Mooncake services or C++ libraries:++```bash+cargo test -p ragfs-cache-mooncake+```++## Native Build++Build the matching Mooncake checkout with Store Rust support:++```bash+cmake -S /path/to/Mooncake -B /path/to/Mooncake/build \+ -DWITH_STORE=ON \+ -DWITH_STORE_RUST=ON+cmake --build /path/to/Mooncake/build \+ --target build_mooncake_store_rust -j+```++Before building OpenViking, provide the paths expected by the official crate:++```bash+export MOONCAKE_BUILD_DIR=/path/to/Mooncake/build+export MOONCAKE_STORE_LIB_DIR=$MOONCAKE_BUILD_DIR/mooncake-store/src+export MOONCAKE_STORE_INCLUDE_DIR=/path/to/Mooncake/mooncake-store/include+export LD_LIBRARY_PATH="$MOONCAKE_STORE_LIB_DIR:$LD_LIBRARY_PATH"+```++Compile the adapter against the official binding:++```bash+cargo check -p ragfs-cache-mooncake --features mooncake-native+```++## TCP Smoke Test++With Mooncake Master and its metadata service running:++```bash+OPENVIKING_RUN_MOONCAKE_INTEGRATION=true \+MOONCAKE_LOCAL_HOSTNAME=127.0.0.1 \+MOONCAKE_METADATA_SERVER=http://127.0.0.1:8080/metadata \+MOONCAKE_MASTER_SERVER_ADDR=127.0.0.1:50051 \+MOONCAKE_PROTOCOL=tcp \+cargo test -p ragfs-cache-mooncake \+ --features mooncake-native \+ --test native_smoke -- --nocapture+```++The test writes one complete object, reads it back, removes it, verifies the+miss result, and closes the provider.++## Docker Smoke Test++The repository includes a Linux ARM64 image that builds the pinned Mooncake+commit, starts its HTTP metadata service and Master, runs Mooncake's official+Rust smoke test, and then runs the OpenViking provider smoke test:++```bash+docker build \+ -t openviking-mooncake-test:arm64 \+ docker/mooncake-test++docker run --rm --shm-size=4g \+ -v "$PWD:/workspace/OpenViking" \+ -v openviking-cargo-registry:/root/.cargo/registry \+ -v openviking-cargo-git:/root/.cargo/git \+ -v openviking-cargo-target:/workspace/OpenViking/target-docker \+ -e CARGO_TARGET_DIR=/workspace/OpenViking/target-docker \+ openviking-mooncake-test:arm64+```diff --git a/crates/ragfs-cache-mooncake/src/client.rs b/crates/ragfs-cache-mooncake/src/client.rsnew file mode 100644index 000000000..1893c8720--- /dev/null+++ b/crates/ragfs-cache-mooncake/src/client.rs@@ -0,0 +1,136 @@+use crate::{MooncakeObjectStore, MooncakeReplicateConfig, MooncakeStoreError};+use ragfs::cache::{CacheError, CacheResult};+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::Arc;+use std::time::Duration;+use tokio::sync::{RwLock, Semaphore};++pub(crate) struct MooncakeClient {+ store: RwLock<Option<Arc<dyn MooncakeObjectStore>>>,+ concurrency: Arc<Semaphore>,+ concurrency_limit: u32,+ timeout: Duration,+ closed: AtomicBool,+}++impl MooncakeClient {+ pub(crate) fn new(+ store: Arc<dyn MooncakeObjectStore>,+ concurrency_limit: usize,+ timeout: Duration,+ ) -> Self {+ Self {+ store: RwLock::new(Some(store)),+ concurrency: Arc::new(Semaphore::new(concurrency_limit)),+ concurrency_limit: concurrency_limit as u32,+ timeout,+ closed: AtomicBool::new(false),+ }+ }++ async fn execute<T, F>(&self, operation: &'static str, call: F) -> CacheResult<T>+ where+ T: Send + 'static,+ F: FnOnce(Arc<dyn MooncakeObjectStore>) -> Result<T, MooncakeStoreError> + Send + 'static,+ {+ if self.closed.load(Ordering::Acquire) {+ return Err(CacheError::Unavailable(+ "Mooncake provider is closed".into(),+ ));+ }++ let work = async {+ let permit = Arc::clone(&self.concurrency)+ .acquire_owned()+ .await+ .map_err(|_| CacheError::Unavailable("Mooncake client is closing".into()))?;+ if self.closed.load(Ordering::Acquire) {+ return Err(CacheError::Unavailable(+ "Mooncake provider is closed".into(),+ ));+ }+ let store = self.store.read().await.clone().ok_or_else(|| {+ CacheError::Unavailable("Mooncake store has been released".into())+ })?;+ tokio::task::spawn_blocking(move || {+ let _permit = permit;+ call(store)+ })+ .await+ .map_err(|error| {+ CacheError::Internal(format!(+ "Mooncake {operation} blocking task failed: {error}"+ ))+ })?+ .map_err(map_store_error)+ };++ tokio::time::timeout(self.timeout, work)+ .await+ .map_err(|_| {+ CacheError::Timeout(format!(+ "Mooncake {operation} exceeded {} ms",+ self.timeout.as_millis()+ ))+ })?+ }++ pub(crate) async fn health_check(&self) -> CacheResult<()> {+ self.execute("health_check", |store| store.health_check())+ .await+ }++ pub(crate) async fn is_exist(&self, key: &str) -> CacheResult<bool> {+ let key = key.to_owned();+ self.execute("is_exist", move |store| store.is_exist(&key))+ .await+ }++ pub(crate) async fn get(&self, key: &str) -> CacheResult<Vec<u8>> {+ let key = key.to_owned();+ self.execute("get", move |store| store.get(&key)).await+ }++ pub(crate) async fn put(+ &self,+ key: &str,+ value: &[u8],+ replicate: &MooncakeReplicateConfig,+ ) -> CacheResult<()> {+ let key = key.to_owned();+ let value = value.to_vec();+ let replicate = replicate.clone();+ self.execute("put", move |store| store.put(&key, &value, &replicate))+ .await+ }++ pub(crate) async fn remove(&self, key: &str) -> CacheResult<()> {+ let key = key.to_owned();+ self.execute("remove", move |store| store.remove(&key, true))+ .await+ }++ pub(crate) async fn close(&self) -> CacheResult<()> {+ if self.closed.swap(true, Ordering::AcqRel) {+ return Ok(());+ }+ let permits = Arc::clone(&self.concurrency)+ .acquire_many_owned(self.concurrency_limit)+ .await+ .map_err(|_| CacheError::Unavailable("Mooncake client is closing".into()))?;+ self.store.write().await.take();+ drop(permits);+ Ok(())+ }+}++fn map_store_error(error: MooncakeStoreError) -> CacheError {+ match error {+ MooncakeStoreError::NotFound => {+ CacheError::Unavailable("Mooncake object disappeared".into())+ }+ MooncakeStoreError::Unavailable(message) => CacheError::Unavailable(message),+ MooncakeStoreError::InvalidArgument(message) => CacheError::InvalidArgument(message),+ MooncakeStoreError::Internal(message) => CacheError::Internal(message),+ }+}diff --git a/crates/ragfs-cache-mooncake/src/config.rs b/crates/ragfs-cache-mooncake/src/config.rsnew file mode 100644index 000000000..c5dddf006--- /dev/null+++ b/crates/ragfs-cache-mooncake/src/config.rs@@ -0,0 +1,73 @@+use ragfs::cache::{CacheError, CacheResult};++/// Connection and execution settings for a Mooncake provider.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct MooncakeConfig {+ /// Hostname or IP address registered for the local Mooncake client.+ pub local_hostname: String,+ /// Metadata service URL.+ pub metadata_server: String,+ /// Mooncake Master service address.+ pub master_server_addr: String,+ /// Transfer protocol, such as `tcp` or `rdma`.+ pub protocol: String,+ /// Optional transport device name.+ pub device_name: String,+ /// Size of the global Mooncake segment in bytes.+ pub global_segment_size: u64,+ /// Size of the local transfer buffer in bytes.+ pub local_buffer_size: u64,+ /// Number of object replicas requested for writes.+ pub replica_num: usize,+ /// Maximum number of concurrent synchronous SDK operations.+ pub sdk_concurrency: usize,+ /// Timeout applied while waiting for each SDK operation.+ pub operation_timeout_ms: u64,+}++impl MooncakeConfig {+ pub(crate) fn validate(&self) -> CacheResult<()> {+ for (name, value) in [+ ("local_hostname", self.local_hostname.as_str()),+ ("metadata_server", self.metadata_server.as_str()),+ ("master_server_addr", self.master_server_addr.as_str()),+ ("protocol", self.protocol.as_str()),+ ] {+ if value.trim().is_empty() {+ return Err(CacheError::InvalidArgument(format!(+ "Mooncake {name} must not be empty"+ )));+ }+ }+ if !matches!(+ self.protocol.as_str(),+ "tcp" | "rdma" | "ascend" | "cxl" | "nvlink" | "barex"+ ) {+ return Err(CacheError::InvalidArgument(format!(+ "unsupported Mooncake protocol: {}",+ self.protocol+ )));+ }+ if self.global_segment_size == 0 {+ return Err(CacheError::InvalidArgument(+ "Mooncake global_segment_size must be greater than zero".into(),+ ));+ }+ if self.local_buffer_size == 0 {+ return Err(CacheError::InvalidArgument(+ "Mooncake local_buffer_size must be greater than zero".into(),+ ));+ }+ if self.sdk_concurrency == 0 || self.sdk_concurrency > u32::MAX as usize {+ return Err(CacheError::InvalidArgument(+ "Mooncake sdk_concurrency must be between 1 and u32::MAX".into(),+ ));+ }+ if self.operation_timeout_ms == 0 {+ return Err(CacheError::InvalidArgument(+ "Mooncake operation_timeout_ms must be greater than zero".into(),+ ));+ }+ Ok(())+ }+}diff --git a/crates/ragfs-cache-mooncake/src/error.rs b/crates/ragfs-cache-mooncake/src/error.rsnew file mode 100644index 000000000..273a5ed83--- /dev/null+++ b/crates/ragfs-cache-mooncake/src/error.rs@@ -0,0 +1,16 @@+/// Errors returned by the synchronous Mooncake object API.+#[derive(Debug, thiserror::Error)]+pub enum MooncakeStoreError {+ /// The requested object does not exist.+ #[error("object not found")]+ NotFound,+ /// Mooncake or one of its required services is unavailable.+ #[error("Mooncake unavailable: {0}")]+ Unavailable(String),+ /// Mooncake rejected an argument.+ #[error("invalid Mooncake argument: {0}")]+ InvalidArgument(String),+ /// Mooncake returned an unspecified operation failure.+ #[error("Mooncake operation failed: {0}")]+ Internal(String),+}diff --git a/crates/ragfs-cache-mooncake/src/lib.rs b/crates/ragfs-cache-mooncake/src/lib.rsnew file mode 100644index 000000000..7ba5a1c1f--- /dev/null+++ b/crates/ragfs-cache-mooncake/src/lib.rs@@ -0,0 +1,14 @@+//! Mooncake cache provider adapter for RAGFS.++mod client;+mod config;+mod error;+#[cfg(feature = "mooncake-native")]+mod native;+mod provider;+mod store;++pub use config::MooncakeConfig;+pub use error::MooncakeStoreError;+pub use provider::MooncakeProvider;+pub use store::{MooncakeObjectStore, MooncakeReplicateConfig};diff --git a/crates/ragfs-cache-mooncake/src/native.rs b/crates/ragfs-cache-mooncake/src/native.rsnew file mode 100644index 000000000..48e4b626b--- /dev/null+++ b/crates/ragfs-cache-mooncake/src/native.rs@@ -0,0 +1,121 @@+use crate::{+ MooncakeConfig, MooncakeObjectStore, MooncakeProvider, MooncakeReplicateConfig,+ MooncakeStoreError,+};+use mooncake_store::{MooncakeStore, ReplicateConfig as NativeReplicateConfig, StoreError};+use ragfs::cache::{CacheError, CacheResult};+use std::sync::Arc;+use std::time::Duration;++struct NativeMooncakeStore {+ store: MooncakeStore,+}++impl NativeMooncakeStore {+ fn connect(config: &MooncakeConfig) -> Result<Self, MooncakeStoreError> {+ let store = MooncakeStore::new().map_err(map_initialization_error)?;+ store+ .setup(+ &config.local_hostname,+ &config.metadata_server,+ config.global_segment_size,+ config.local_buffer_size,+ &config.protocol,+ &config.device_name,+ &config.master_server_addr,+ )+ .map_err(map_initialization_error)?;+ Ok(Self { store })+ }+}++impl MooncakeObjectStore for NativeMooncakeStore {+ fn health_check(&self) -> Result<(), MooncakeStoreError> {+ self.store.health_check().map_err(map_initialization_error)+ }++ fn is_exist(&self, key: &str) -> Result<bool, MooncakeStoreError> {+ self.store.is_exist(key).map_err(map_operation_error)+ }++ fn get(&self, key: &str) -> Result<Vec<u8>, MooncakeStoreError> {+ self.store.get(key).map_err(map_operation_error)+ }++ fn put(+ &self,+ key: &str,+ value: &[u8],+ replicate: &MooncakeReplicateConfig,+ ) -> Result<(), MooncakeStoreError> {+ let native = NativeReplicateConfig {+ replica_num: replicate.replica_num,+ with_soft_pin: replicate.with_soft_pin,+ with_hard_pin: replicate.with_hard_pin,+ preferred_segments: replicate.preferred_segments.clone(),+ };+ self.store+ .put(key, value, Some(&native))+ .map_err(map_operation_error)+ }++ fn remove(&self, key: &str, force: bool) -> Result<(), MooncakeStoreError> {+ self.store.remove(key, force).map_err(map_operation_error)+ }+}++impl MooncakeProvider {+ /// Connect to Mooncake through the pinned official Rust binding.+ pub async fn connect(config: MooncakeConfig) -> CacheResult<Self> {+ config.validate()?;+ let setup_config = config.clone();+ let setup_timeout = Duration::from_millis(config.operation_timeout_ms);+ let store = tokio::time::timeout(+ setup_timeout,+ tokio::task::spawn_blocking(move || NativeMooncakeStore::connect(&setup_config)),+ )+ .await+ .map_err(|_| {+ CacheError::Timeout(format!(+ "Mooncake setup exceeded {} ms",+ setup_timeout.as_millis()+ ))+ })?+ .map_err(|error| CacheError::Internal(format!("Mooncake setup task failed: {error}")))?+ .map_err(map_cache_error)?;++ Self::from_store(config, Arc::new(store)).await+ }+}++fn map_initialization_error(error: StoreError) -> MooncakeStoreError {+ match error {+ StoreError::InvalidString(error) => MooncakeStoreError::InvalidArgument(error.to_string()),+ StoreError::InvalidArgument(message) => MooncakeStoreError::InvalidArgument(message),+ StoreError::NullHandle | StoreError::OperationFailed(_) | StoreError::NotFound => {+ MooncakeStoreError::Unavailable(error.to_string())+ }+ }+}++fn map_operation_error(error: StoreError) -> MooncakeStoreError {+ match error {+ StoreError::NotFound => MooncakeStoreError::NotFound,+ StoreError::InvalidString(error) => MooncakeStoreError::InvalidArgument(error.to_string()),+ StoreError::InvalidArgument(message) => MooncakeStoreError::InvalidArgument(message),+ StoreError::NullHandle | StoreError::OperationFailed(_) => {+ MooncakeStoreError::Internal(error.to_string())+ }+ }+}++fn map_cache_error(error: MooncakeStoreError) -> CacheError {+ match error {+ MooncakeStoreError::NotFound => {+ CacheError::Unavailable("Mooncake setup object not found".into())+ }+ MooncakeStoreError::Unavailable(message) => CacheError::Unavailable(message),+ MooncakeStoreError::InvalidArgument(message) => CacheError::InvalidArgument(message),+ MooncakeStoreError::Internal(message) => CacheError::Internal(message),+ }+}diff --git a/crates/ragfs-cache-mooncake/src/provider.rs b/crates/ragfs-cache-mooncake/src/provider.rsnew file mode 100644index 000000000..6ae2f2590--- /dev/null+++ b/crates/ragfs-cache-mooncake/src/provider.rs@@ -0,0 +1,209 @@+use crate::client::MooncakeClient;+use crate::{MooncakeConfig, MooncakeObjectStore, MooncakeReplicateConfig};+use async_trait::async_trait;+use bytes::Bytes;+use futures::stream::{self, StreamExt};+use ragfs::cache::{CacheError, CacheProvider, CacheResult, ProviderCapabilities};+use std::collections::HashSet;+use std::fmt;+use std::sync::{Arc, Mutex, MutexGuard};+use std::time::Duration;++/// Mooncake implementation of the common RAGFS cache provider contract.+pub struct MooncakeProvider {+ client: Arc<MooncakeClient>,+ replicate: MooncakeReplicateConfig,+ batch_concurrency: usize,+ known_keys: Mutex<HashSet<String>>,+}++impl fmt::Debug for MooncakeProvider {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter+ .debug_struct("MooncakeProvider")+ .field("batch_concurrency", &self.batch_concurrency)+ .finish_non_exhaustive()+ }+}++impl MooncakeProvider {+ /// Construct a provider over a Mooncake-compatible object store.+ ///+ /// Construction validates configuration and performs a health check.+ pub async fn from_store(+ config: MooncakeConfig,+ store: Arc<dyn MooncakeObjectStore>,+ ) -> CacheResult<Self> {+ config.validate()?;+ let client = Arc::new(MooncakeClient::new(+ store,+ config.sdk_concurrency,+ Duration::from_millis(config.operation_timeout_ms),+ ));+ client.health_check().await?;+ Ok(Self {+ client,+ replicate: MooncakeReplicateConfig {+ replica_num: config.replica_num,+ ..MooncakeReplicateConfig::default()+ },+ batch_concurrency: config.sdk_concurrency,+ known_keys: Mutex::new(HashSet::new()),+ })+ }++ /// Return a startup error when the official native binding is not compiled.+ #[cfg(not(feature = "mooncake-native"))]+ pub async fn connect(config: MooncakeConfig) -> CacheResult<Self> {+ config.validate()?;+ Err(CacheError::Unavailable(+ "Mooncake support requires the mooncake-native feature".into(),+ ))+ }++ /// Check whether the connected Mooncake services are healthy.+ pub async fn health_check(&self) -> CacheResult<()> {+ self.client.health_check().await+ }++ async fn get_object(&self, key: &str) -> CacheResult<Option<Bytes>> {+ if !self.client.is_exist(key).await? {+ return Ok(None);+ }+ match self.client.get(key).await {+ Ok(value) => Ok(Some(Bytes::from(value))),+ Err(error) => match self.client.is_exist(key).await {+ Ok(false) => Ok(None),+ Ok(true) | Err(_) => Err(error),+ },+ }+ }++ async fn delete_object(&self, key: &str) -> CacheResult<()> {+ if !self.client.is_exist(key).await? {+ lock_known_keys(&self.known_keys)?.remove(key);+ return Ok(());+ }+ match self.client.remove(key).await {+ Ok(()) => {+ lock_known_keys(&self.known_keys)?.remove(key);+ Ok(())+ }+ Err(error) => match self.client.is_exist(key).await {+ Ok(false) => {+ lock_known_keys(&self.known_keys)?.remove(key);+ Ok(())+ }+ Ok(true) | Err(_) => Err(error),+ },+ }+ }+}++fn lock_known_keys(+ known_keys: &Mutex<HashSet<String>>,+) -> CacheResult<MutexGuard<'_, HashSet<String>>> {+ known_keys+ .lock()+ .map_err(|_| CacheError::Internal("Mooncake key tracker is poisoned".into()))+}++#[async_trait]+impl CacheProvider for MooncakeProvider {+ fn name(&self) -> &'static str {+ "mooncake"+ }++ fn capabilities(&self) -> ProviderCapabilities {+ ProviderCapabilities {+ batch_get: true,+ batch_put: true,+ native_ttl: false,+ }+ }++ async fn get(&self, key: &str) -> CacheResult<Option<Bytes>> {+ self.get_object(key).await+ }++ async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> {+ self.client.put(key, &value, &self.replicate).await?;+ lock_known_keys(&self.known_keys)?.insert(key.to_owned());+ Ok(())+ }++ async fn delete(&self, key: &str) -> CacheResult<()> {+ self.delete_object(key).await+ }++ async fn exists(&self, key: &str) -> CacheResult<bool> {+ self.client.is_exist(key).await+ }++ async fn batch_get(&self, keys: &[String]) -> CacheResult<Vec<Option<Bytes>>> {+ let mut values = stream::iter(keys.iter().cloned().enumerate())+ .map(|(index, key)| async move { (index, self.get_object(&key).await) })+ .buffer_unordered(self.batch_concurrency)+ .collect::<Vec<_>>()+ .await;+ values.sort_by_key(|(index, _)| *index);+ values+ .into_iter()+ .map(|(_, value)| value)+ .collect::<CacheResult<Vec<_>>>()+ }++ async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> {+ let results = stream::iter(entries)+ .map(|(key, value)| async move { self.put(&key, value).await })+ .buffer_unordered(self.batch_concurrency)+ .collect::<Vec<_>>()+ .await;+ results.into_iter().collect()+ }++ async fn invalidate(&self, keys: &[String]) -> CacheResult<()> {+ let results = stream::iter(keys.iter().cloned())+ .map(|key| async move { self.delete_object(&key).await })+ .buffer_unordered(self.batch_concurrency)+ .collect::<Vec<_>>()+ .await;+ results.into_iter().collect()+ }++ async fn flush(&self) -> CacheResult<()> {+ let keys = lock_known_keys(&self.known_keys)?+ .iter()+ .cloned()+ .collect::<Vec<_>>();+ self.invalidate(&keys).await+ }++ async fn close(&self) -> CacheResult<()> {+ self.client.close().await+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::panic::{self, AssertUnwindSafe};++ #[test]+ fn lock_known_keys_returns_internal_error_when_poisoned() {+ let known_keys = Mutex::new(HashSet::new());++ let _ = panic::catch_unwind(AssertUnwindSafe(|| {+ let _guard = known_keys.lock().unwrap();+ panic!("poison known_keys");+ }));++ let error = lock_known_keys(&known_keys).unwrap_err();++ assert!(matches!(+ error,+ CacheError::Internal(message)+ if message == "Mooncake key tracker is poisoned"+ ));+ }+}diff --git a/crates/ragfs-cache-mooncake/src/store.rs b/crates/ragfs-cache-mooncake/src/store.rsnew file mode 100644index 000000000..c3fdde08a--- /dev/null+++ b/crates/ragfs-cache-mooncake/src/store.rs@@ -0,0 +1,35 @@+use crate::MooncakeStoreError;++/// Replication options applied to each Mooncake object write.+#[derive(Debug, Clone, Default, PartialEq, Eq)]+pub struct MooncakeReplicateConfig {+ /// Number of replicas to create.+ pub replica_num: usize,+ /// Prefer a replica local to the client.+ pub with_soft_pin: bool,+ /// Prevent eviction of the object.+ pub with_hard_pin: bool,+ /// Preferred Mooncake segment names.+ pub preferred_segments: Vec<String>,+}++/// Synchronous object operations exposed by Mooncake's Rust binding.+///+/// The asynchronous client executes every method on Tokio's blocking pool.+pub trait MooncakeObjectStore: Send + Sync + 'static {+ /// Check that Mooncake services are reachable.+ fn health_check(&self) -> Result<(), MooncakeStoreError>;+ /// Return whether an object exists.+ fn is_exist(&self, key: &str) -> Result<bool, MooncakeStoreError>;+ /// Read a complete object.+ fn get(&self, key: &str) -> Result<Vec<u8>, MooncakeStoreError>;+ /// Store a complete object.+ fn put(+ &self,+ key: &str,+ value: &[u8],+ replicate: &MooncakeReplicateConfig,+ ) -> Result<(), MooncakeStoreError>;+ /// Remove an object.+ fn remove(&self, key: &str, force: bool) -> Result<(), MooncakeStoreError>;+}diff --git a/crates/ragfs-cache-mooncake/tests/cached_filesystem.rs b/crates/ragfs-cache-mooncake/tests/cached_filesystem.rsnew file mode 100644index 000000000..eeb73d6ca--- /dev/null+++ b/crates/ragfs-cache-mooncake/tests/cached_filesystem.rs@@ -0,0 +1,335 @@+use async_trait::async_trait;+use ragfs::cache::{CacheNamespace, CachePolicy, CacheProvider, CachedFileSystem};+use ragfs::core::{GrepResult, TreeEntry};+use ragfs::plugins::MemFileSystem;+use ragfs::{FileInfo, FileSystem, Result, WriteFlag};+use ragfs_cache_mooncake::{+ MooncakeConfig, MooncakeObjectStore, MooncakeProvider, MooncakeReplicateConfig,+ MooncakeStoreError,+};+use std::collections::HashMap;+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::{Arc, Mutex};++#[derive(Default)]+struct SharedStore {+ values: Mutex<HashMap<String, Vec<u8>>>,+ available: AtomicBool,+}++impl SharedStore {+ fn available() -> Self {+ Self {+ available: AtomicBool::new(true),+ ..Self::default()+ }+ }++ fn check(&self) -> std::result::Result<(), MooncakeStoreError> {+ if self.available.load(Ordering::SeqCst) {+ Ok(())+ } else {+ Err(MooncakeStoreError::Unavailable("store unavailable".into()))+ }+ }+}++impl MooncakeObjectStore for SharedStore {+ fn health_check(&self) -> std::result::Result<(), MooncakeStoreError> {+ self.check()+ }++ fn is_exist(&self, key: &str) -> std::result::Result<bool, MooncakeStoreError> {+ self.check()?;+ Ok(self.values.lock().unwrap().contains_key(key))+ }++ fn get(&self, key: &str) -> std::result::Result<Vec<u8>, MooncakeStoreError> {+ self.check()?;+ self.values+ .lock()+ .unwrap()+ .get(key)+ .cloned()+ .ok_or(MooncakeStoreError::NotFound)+ }++ fn put(+ &self,+ key: &str,+ value: &[u8],+ _replicate: &MooncakeReplicateConfig,+ ) -> std::result::Result<(), MooncakeStoreError> {+ self.check()?;+ self.values+ .lock()+ .unwrap()+ .insert(key.to_owned(), value.to_vec());+ Ok(())+ }++ fn remove(&self, key: &str, _force: bool) -> std::result::Result<(), MooncakeStoreError> {+ self.check()?;+ self.values.lock().unwrap().remove(key);+ Ok(())+ }+}++#[derive(Clone)]+struct CountingFileSystem {+ inner: Arc<MemFileSystem>,+ reads: Arc<AtomicU64>,+}++impl CountingFileSystem {+ fn new() -> Self {+ Self {+ inner: Arc::new(MemFileSystem::new()),+ reads: Arc::new(AtomicU64::new(0)),+ }+ }++ fn read_count(&self) -> u64 {+ self.reads.load(Ordering::SeqCst)+ }+}++#[async_trait]+impl FileSystem for CountingFileSystem {+ async fn create(&self, path: &str) -> Result<()> {+ self.inner.create(path).await+ }++ async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {+ self.inner.mkdir(path, mode).await+ }++ async fn remove(&self, path: &str) -> Result<()> {+ self.inner.remove(path).await+ }++ async fn remove_all(&self, path: &str) -> Result<()> {+ self.inner.remove_all(path).await+ }++ async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {+ self.reads.fetch_add(1, Ordering::SeqCst);+ self.inner.read(path, offset, size).await+ }++ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {+ self.inner.write(path, data, offset, flags).await+ }++ async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {+ self.inner.read_dir(path).await+ }++ async fn stat(&self, path: &str) -> Result<FileInfo> {+ self.inner.stat(path).await+ }++ async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {+ self.inner.rename(old_path, new_path).await+ }++ async fn chmod(&self, path: &str, mode: u32) -> Result<()> {+ self.inner.chmod(path, mode).await+ }++ async fn truncate(&self, path: &str, size: u64) -> Result<()> {+ self.inner.truncate(path, size).await+ }++ async fn grep(+ &self,+ path: &str,+ pattern: &str,+ recursive: bool,+ case_insensitive: bool,+ node_limit: Option<usize>,+ exclude_path: Option<&str>,+ level_limit: Option<usize>,+ ) -> Result<GrepResult> {+ self.inner+ .grep(+ path,+ pattern,+ recursive,+ case_insensitive,+ node_limit,+ exclude_path,+ level_limit,+ )+ .await+ }++ async fn tree_directory(+ &self,+ path: &str,+ show_hidden: bool,+ node_limit: Option<usize>,+ level_limit: Option<usize>,+ ) -> Result<Vec<TreeEntry>> {+ self.inner+ .tree_directory(path, show_hidden, node_limit, level_limit)+ .await+ }+}++fn config() -> MooncakeConfig {+ MooncakeConfig {+ local_hostname: "127.0.0.1".into(),+ metadata_server: "http://127.0.0.1:8080/metadata".into(),+ master_server_addr: "127.0.0.1:50051".into(),+ protocol: "tcp".into(),+ device_name: String::new(),+ global_segment_size: 512 << 20,+ local_buffer_size: 128 << 20,+ replica_num: 2,+ sdk_concurrency: 4,+ operation_timeout_ms: 100,+ }+}++async fn cached_fs(+ backend: CountingFileSystem,+ store: Arc<SharedStore>,+ namespace: &str,+) -> CachedFileSystem {+ let provider: Arc<dyn CacheProvider> =+ Arc::new(MooncakeProvider::from_store(config(), store).await.unwrap());+ CachedFileSystem::new(+ Box::new(backend),+ provider,+ CacheNamespace::new(namespace),+ CachePolicy::default(),+ )+}++#[tokio::test]+async fn mooncake_provider_supports_miss_fill_hit_and_write_after_read() {+ let backend = CountingFileSystem::new();+ backend+ .write("/value.md", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let fs = cached_fs(backend, Arc::new(SharedStore::available()), "read-write").await;++ assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old");+ assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old");+ assert_eq!(probe.read_count(), 1);++ fs.write("/value.md", b"new", 0, WriteFlag::Truncate)+ .await+ .unwrap();+ assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"new");+ assert_eq!(probe.read_count(), 1);+}++#[tokio::test]+async fn mooncake_provider_preserves_invalidation_and_subtree_generation_rules() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/tree", 0o755).await.unwrap();+ backend+ .write("/tree/leaf", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let fs = cached_fs(backend, Arc::new(SharedStore::available()), "invalidation").await;++ assert_eq!(fs.read("/tree/leaf", 0, 0).await.unwrap(), b"old");+ fs.rename("/tree/leaf", "/tree/moved").await.unwrap();+ assert!(fs.read("/tree/leaf", 0, 0).await.is_err());+ assert_eq!(fs.read("/tree/moved", 0, 0).await.unwrap(), b"old");+ fs.remove("/tree/moved").await.unwrap();+ assert!(fs.read("/tree/moved", 0, 0).await.is_err());++ direct+ .write("/tree/leaf", b"stale", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(fs.read("/tree/leaf", 0, 0).await.unwrap(), b"stale");+ fs.remove_all("/tree").await.unwrap();+ direct.mkdir("/tree", 0o755).await.unwrap();+ direct+ .write("/tree/leaf", b"fresh", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(fs.read("/tree/leaf", 0, 0).await.unwrap(), b"fresh");+}++#[tokio::test]+async fn mooncake_provider_caches_directory_entries_and_invalidates_directory_renames() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/root", 0o755).await.unwrap();+ backend.mkdir("/root/old", 0o755).await.unwrap();+ backend+ .write("/root/old/leaf", b"value", 0, WriteFlag::Create)+ .await+ .unwrap();+ let fs = cached_fs(backend, Arc::new(SharedStore::available()), "directories").await;++ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1);+ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1);+ fs.mkdir("/root/created", 0o755).await.unwrap();+ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);++ assert_eq!(fs.read("/root/old/leaf", 0, 0).await.unwrap(), b"value");+ fs.rename("/root/old", "/root/moved").await.unwrap();+ assert!(fs.read("/root/old/leaf", 0, 0).await.is_err());+ assert_eq!(fs.read("/root/moved/leaf", 0, 0).await.unwrap(), b"value");+}++#[tokio::test]+async fn unavailable_mooncake_falls_back_without_breaking_backend_reads() {+ let backend = CountingFileSystem::new();+ backend+ .write("/available.md", b"backend", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let store = Arc::new(SharedStore::available());+ let fs = cached_fs(backend, store.clone(), "fallback").await;+ store.available.store(false, Ordering::SeqCst);++ assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend");+ assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend");+ assert_eq!(probe.read_count(), 2);+ assert!(fs.metrics().snapshot().errors >= 1);+}++#[tokio::test]+async fn multiple_wrappers_share_one_mooncake_provider_without_key_collisions() {+ let store = Arc::new(SharedStore::available());+ let provider: Arc<dyn CacheProvider> =+ Arc::new(MooncakeProvider::from_store(config(), store).await.unwrap());+ let first_backend = CountingFileSystem::new();+ first_backend+ .write("/same.md", b"first", 0, WriteFlag::Create)+ .await+ .unwrap();+ let second_backend = CountingFileSystem::new();+ second_backend+ .write("/same.md", b"second", 0, WriteFlag::Create)+ .await+ .unwrap();+ let first = CachedFileSystem::new(+ Box::new(first_backend),+ provider.clone(),+ CacheNamespace::new("mount-one"),+ CachePolicy::default(),+ );+ let second = CachedFileSystem::new(+ Box::new(second_backend),+ provider,+ CacheNamespace::new("mount-two"),+ CachePolicy::default(),+ );++ assert_eq!(first.read("/same.md", 0, 0).await.unwrap(), b"first");+ assert_eq!(second.read("/same.md", 0, 0).await.unwrap(), b"second");+ assert_eq!(first.read("/same.md", 0, 0).await.unwrap(), b"first");+ assert_eq!(second.read("/same.md", 0, 0).await.unwrap(), b"second");+}diff --git a/crates/ragfs-cache-mooncake/tests/native_smoke.rs b/crates/ragfs-cache-mooncake/tests/native_smoke.rsnew file mode 100644index 000000000..6419ab861--- /dev/null+++ b/crates/ragfs-cache-mooncake/tests/native_smoke.rs@@ -0,0 +1,43 @@+#![cfg(feature = "mooncake-native")]++use bytes::Bytes;+use ragfs::cache::CacheProvider;+use ragfs_cache_mooncake::{MooncakeConfig, MooncakeProvider};++fn required_env(name: &str) -> String {+ std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set for native smoke test"))+}++#[tokio::test]+async fn native_mooncake_round_trips_complete_objects_over_tcp() {+ if std::env::var("OPENVIKING_RUN_MOONCAKE_INTEGRATION").as_deref() != Ok("true") {+ return;+ }++ let config = MooncakeConfig {+ local_hostname: required_env("MOONCAKE_LOCAL_HOSTNAME"),+ metadata_server: required_env("MOONCAKE_METADATA_SERVER"),+ master_server_addr: required_env("MOONCAKE_MASTER_SERVER_ADDR"),+ protocol: std::env::var("MOONCAKE_PROTOCOL").unwrap_or_else(|_| "tcp".into()),+ device_name: std::env::var("MOONCAKE_DEVICE_NAME").unwrap_or_default(),+ global_segment_size: 512 << 20,+ local_buffer_size: 128 << 20,+ replica_num: 1,+ sdk_concurrency: 4,+ operation_timeout_ms: 5_000,+ };+ let provider = MooncakeProvider::connect(config).await.unwrap();+ let key = format!("openviking:smoke:{}", std::process::id());++ provider+ .put(&key, Bytes::from_static(b"mooncake-object"))+ .await+ .unwrap();+ assert_eq!(+ provider.get(&key).await.unwrap(),+ Some(Bytes::from_static(b"mooncake-object"))+ );+ provider.delete(&key).await.unwrap();+ assert_eq!(provider.get(&key).await.unwrap(), None);+ provider.close().await.unwrap();+}diff --git a/crates/ragfs-cache-mooncake/tests/provider_contract.rs b/crates/ragfs-cache-mooncake/tests/provider_contract.rsnew file mode 100644index 000000000..9fb346447--- /dev/null+++ b/crates/ragfs-cache-mooncake/tests/provider_contract.rs@@ -0,0 +1,357 @@+use bytes::Bytes;+use ragfs::cache::{CacheError, CacheProvider};+use ragfs_cache_mooncake::{+ MooncakeConfig, MooncakeObjectStore, MooncakeProvider, MooncakeReplicateConfig,+ MooncakeStoreError,+};+use std::collections::HashMap;+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+use std::sync::{Arc, Mutex};+use std::time::{Duration, Instant};++#[derive(Default)]+struct FakeStore {+ values: Mutex<HashMap<String, Vec<u8>>>,+ remove_force_flags: Mutex<Vec<bool>>,+ healthy: AtomicBool,+ fail_get: AtomicBool,+ remove_during_get: AtomicBool,+ delay_ms: AtomicUsize,+ active: AtomicUsize,+ max_active: AtomicUsize,+}++impl FakeStore {+ fn healthy() -> Self {+ Self {+ healthy: AtomicBool::new(true),+ ..Self::default()+ }+ }++ fn enter(&self) -> ActiveGuard<'_> {+ let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;+ self.max_active.fetch_max(active, Ordering::SeqCst);+ let delay = self.delay_ms.load(Ordering::SeqCst);+ if delay > 0 {+ std::thread::sleep(Duration::from_millis(delay as u64));+ }+ ActiveGuard { store: self }+ }+}++struct ActiveGuard<'a> {+ store: &'a FakeStore,+}++impl Drop for ActiveGuard<'_> {+ fn drop(&mut self) {+ self.store.active.fetch_sub(1, Ordering::SeqCst);+ }+}++impl MooncakeObjectStore for FakeStore {+ fn health_check(&self) -> Result<(), MooncakeStoreError> {+ if self.healthy.load(Ordering::SeqCst) {+ Ok(())+ } else {+ Err(MooncakeStoreError::Unavailable("not healthy".into()))+ }+ }++ fn is_exist(&self, key: &str) -> Result<bool, MooncakeStoreError> {+ let _guard = self.enter();+ Ok(self.values.lock().unwrap().contains_key(key))+ }++ fn get(&self, key: &str) -> Result<Vec<u8>, MooncakeStoreError> {+ let _guard = self.enter();+ if self.remove_during_get.load(Ordering::SeqCst) {+ self.values.lock().unwrap().remove(key);+ return Err(MooncakeStoreError::NotFound);+ }+ if self.fail_get.load(Ordering::SeqCst) {+ return Err(MooncakeStoreError::Internal("get failed".into()));+ }+ self.values+ .lock()+ .unwrap()+ .get(key)+ .cloned()+ .ok_or(MooncakeStoreError::NotFound)+ }++ fn put(+ &self,+ key: &str,+ value: &[u8],+ _replicate: &MooncakeReplicateConfig,+ ) -> Result<(), MooncakeStoreError> {+ let _guard = self.enter();+ self.values+ .lock()+ .unwrap()+ .insert(key.to_string(), value.to_vec());+ Ok(())+ }++ fn remove(&self, key: &str, force: bool) -> Result<(), MooncakeStoreError> {+ let _guard = self.enter();+ self.remove_force_flags.lock().unwrap().push(force);+ self.values.lock().unwrap().remove(key);+ Ok(())+ }+}++fn config() -> MooncakeConfig {+ MooncakeConfig {+ local_hostname: "127.0.0.1".into(),+ metadata_server: "http://127.0.0.1:8080/metadata".into(),+ master_server_addr: "127.0.0.1:50051".into(),+ protocol: "tcp".into(),+ device_name: String::new(),+ global_segment_size: 512 << 20,+ local_buffer_size: 128 << 20,+ replica_num: 2,+ sdk_concurrency: 2,+ operation_timeout_ms: 100,+ }+}++async fn provider(store: Arc<FakeStore>) -> MooncakeProvider {+ MooncakeProvider::from_store(config(), store).await.unwrap()+}++#[tokio::test]+async fn initialization_requires_valid_config_and_health_check() {+ let mut invalid = config();+ invalid.sdk_concurrency = 0;+ let error = MooncakeProvider::from_store(invalid, Arc::new(FakeStore::healthy()))+ .await+ .unwrap_err();+ assert!(matches!(error, CacheError::InvalidArgument(_)));++ let mut invalid_protocol = config();+ invalid_protocol.protocol = "unknown".into();+ let error = MooncakeProvider::from_store(invalid_protocol, Arc::new(FakeStore::healthy()))+ .await+ .unwrap_err();+ assert!(matches!(error, CacheError::InvalidArgument(_)));++ let error = MooncakeProvider::from_store(config(), Arc::new(FakeStore::default()))+ .await+ .unwrap_err();+ assert!(matches!(error, CacheError::Unavailable(_)));+}++#[tokio::test]+async fn get_distinguishes_hit_miss_and_internal_error() {+ let store = Arc::new(FakeStore::healthy());+ store+ .values+ .lock()+ .unwrap()+ .insert("hit".into(), b"value".to_vec());+ let provider = provider(store.clone()).await;++ assert_eq!(+ provider.get("hit").await.unwrap(),+ Some(Bytes::from_static(b"value"))+ );+ assert_eq!(provider.get("missing").await.unwrap(), None);++ store.fail_get.store(true, Ordering::SeqCst);+ let error = provider.get("hit").await.unwrap_err();+ assert!(matches!(error, CacheError::Internal(_)));+}++#[tokio::test]+async fn object_removed_between_exist_and_get_is_a_miss() {+ let store = Arc::new(FakeStore::healthy());+ store+ .values+ .lock()+ .unwrap()+ .insert("racy".into(), b"value".to_vec());+ store.remove_during_get.store(true, Ordering::SeqCst);+ let provider = provider(store).await;++ assert_eq!(provider.get("racy").await.unwrap(), None);+}++#[tokio::test]+async fn put_delete_and_batch_operations_preserve_object_semantics() {+ let store = Arc::new(FakeStore::healthy());+ let provider = provider(store.clone()).await;++ provider.put("one", Bytes::from_static(b"1")).await.unwrap();+ provider+ .batch_put(vec![+ ("two".into(), Bytes::from_static(b"2")),+ ("three".into(), Bytes::from_static(b"3")),+ ])+ .await+ .unwrap();+ assert_eq!(+ provider+ .batch_get(&["three".into(), "missing".into(), "one".into()])+ .await+ .unwrap(),+ vec![+ Some(Bytes::from_static(b"3")),+ None,+ Some(Bytes::from_static(b"1"))+ ]+ );++ provider.delete("one").await.unwrap();+ provider.delete("one").await.unwrap();+ assert_eq!(provider.get("one").await.unwrap(), None);++ provider.invalidate(&["two".into()]).await.unwrap();+ assert_eq!(provider.get("two").await.unwrap(), None);+ provider.flush().await.unwrap();+ assert_eq!(provider.get("three").await.unwrap(), None);+ assert!(+ store+ .remove_force_flags+ .lock()+ .unwrap()+ .iter()+ .all(|force| *force),+ "cache invalidation must override active Mooncake leases"+ );++ assert!(provider.capabilities().batch_get);+ assert!(provider.capabilities().batch_put);+ assert!(!provider.capabilities().native_ttl);+}++#[tokio::test]+async fn blocking_calls_are_bounded_and_time_out_without_becoming_misses() {+ let store = Arc::new(FakeStore::healthy());+ store+ .values+ .lock()+ .unwrap()+ .insert("slow".into(), b"value".to_vec());+ store.delay_ms.store(40, Ordering::SeqCst);+ let mut bounded_config = config();+ bounded_config.operation_timeout_ms = 500;+ let provider = Arc::new(+ MooncakeProvider::from_store(bounded_config, store.clone())+ .await+ .unwrap(),+ );++ let mut tasks = Vec::new();+ for _ in 0..8 {+ let provider = provider.clone();+ tasks.push(tokio::spawn(+ async move { provider.get("slow").await.unwrap() },+ ));+ }+ for task in tasks {+ assert_eq!(task.await.unwrap(), Some(Bytes::from_static(b"value")));+ }+ assert!(store.max_active.load(Ordering::SeqCst) <= 2);++ let timeout_store = Arc::new(FakeStore::healthy());+ timeout_store.delay_ms.store(80, Ordering::SeqCst);+ let mut timeout_config = config();+ timeout_config.operation_timeout_ms = 10;+ let timeout_provider = MooncakeProvider::from_store(timeout_config, timeout_store)+ .await+ .unwrap();+ let error = timeout_provider.get("slow").await.unwrap_err();+ assert!(matches!(error, CacheError::Timeout(_)));+}++#[tokio::test]+async fn close_rejects_new_operations() {+ let provider = provider(Arc::new(FakeStore::healthy())).await;+ provider.close().await.unwrap();++ let error = provider.get("key").await.unwrap_err();+ assert!(matches!(error, CacheError::Unavailable(_)));+}++struct SlowDropStore {+ started: Arc<AtomicBool>,+ dropped: Arc<AtomicBool>,+}++impl Drop for SlowDropStore {+ fn drop(&mut self) {+ self.dropped.store(true, Ordering::SeqCst);+ }+}++impl MooncakeObjectStore for SlowDropStore {+ fn health_check(&self) -> Result<(), MooncakeStoreError> {+ Ok(())+ }++ fn is_exist(&self, _key: &str) -> Result<bool, MooncakeStoreError> {+ self.started.store(true, Ordering::SeqCst);+ std::thread::sleep(Duration::from_millis(80));+ Ok(false)+ }++ fn get(&self, _key: &str) -> Result<Vec<u8>, MooncakeStoreError> {+ unreachable!("missing object must not be read")+ }++ fn put(+ &self,+ _key: &str,+ _value: &[u8],+ _replicate: &MooncakeReplicateConfig,+ ) -> Result<(), MooncakeStoreError> {+ Ok(())+ }++ fn remove(&self, _key: &str, _force: bool) -> Result<(), MooncakeStoreError> {+ Ok(())+ }+}++#[tokio::test]+async fn close_waits_for_inflight_calls_before_releasing_store() {+ let started = Arc::new(AtomicBool::new(false));+ let dropped = Arc::new(AtomicBool::new(false));+ let store = Arc::new(SlowDropStore {+ started: started.clone(),+ dropped: dropped.clone(),+ });+ let provider = Arc::new(+ MooncakeProvider::from_store(config(), store.clone())+ .await+ .unwrap(),+ );+ drop(store);++ let reader = {+ let provider = provider.clone();+ tokio::spawn(async move { provider.get("slow").await })+ };+ while !started.load(Ordering::SeqCst) {+ tokio::task::yield_now().await;+ }++ let close_started = Instant::now();+ provider.close().await.unwrap();+ assert!(close_started.elapsed() >= Duration::from_millis(50));+ assert_eq!(reader.await.unwrap().unwrap(), None);+ assert!(dropped.load(Ordering::SeqCst));+}++#[cfg(not(feature = "mooncake-native"))]+#[tokio::test]+async fn connect_without_native_feature_returns_startup_error() {+ let error = MooncakeProvider::connect(config()).await.unwrap_err();+ assert!(matches!(+ error,+ CacheError::Unavailable(message) if message.contains("mooncake-native")+ ));+}diff --git a/crates/ragfs-cache-redis/Cargo.toml b/crates/ragfs-cache-redis/Cargo.tomlnew file mode 100644index 000000000..408cf0a80--- /dev/null+++ b/crates/ragfs-cache-redis/Cargo.toml@@ -0,0 +1,19 @@+[package]+name = "ragfs-cache-redis"+version = "0.1.0"+edition = "2021"+rust-version = "1.91.1"+license = "Apache-2.0"+description = "Redis cache provider adapter for RAGFS"++[dependencies]+async-trait = "0.1"+bytes = "1.5"+ragfs = { path = "../ragfs", features = ["cache"] }+redis = { version = "0.32", features = ["tokio-comp"] }+thiserror = "1.0"+tokio = { version = "1.38", features = ["rt-multi-thread", "sync", "time", "macros"] }+url = "2"++[dev-dependencies]+ragfs = { path = "../ragfs", features = ["cache"] }diff --git a/crates/ragfs-cache-redis/src/client.rs b/crates/ragfs-cache-redis/src/client.rsnew file mode 100644index 000000000..051ff9159--- /dev/null+++ b/crates/ragfs-cache-redis/src/client.rs@@ -0,0 +1,261 @@+use crate::RedisConfig;+use ragfs::cache::{CacheError, CacheResult};+use redis::aio::MultiplexedConnection;+use redis::{AsyncCommands, RedisError};+use std::env;+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::Arc;+use std::time::Duration;+use tokio::sync::{RwLock, Semaphore};+use url::Url;++pub(crate) struct RedisClient {+ connection: RwLock<Option<MultiplexedConnection>>,+ concurrency: Arc<Semaphore>,+ concurrency_limit: u32,+ command_timeout: Duration,+ closed: AtomicBool,+}++impl RedisClient {+ pub(crate) async fn connect(config: &RedisConfig) -> CacheResult<Self> {+ config.validate()?;+ let endpoint = endpoint_url(config)?;+ let redis_client =+ redis::Client::open(endpoint).map_err(|error| map_redis_error("open", error))?;+ let connect = async {+ let connection = redis_client+ .get_multiplexed_async_connection()+ .await+ .map_err(|error| map_redis_error("connect", error))?;+ Ok::<_, CacheError>(connection)+ };+ let connection =+ tokio::time::timeout(Duration::from_millis(config.connect_timeout_ms), connect)+ .await+ .map_err(|_| {+ CacheError::Timeout(format!(+ "Redis connect exceeded {} ms",+ config.connect_timeout_ms+ ))+ })??;+ let client = Self {+ connection: RwLock::new(Some(connection)),+ concurrency: Arc::new(Semaphore::new(config.pool_size)),+ concurrency_limit: config.pool_size as u32,+ command_timeout: Duration::from_millis(config.command_timeout_ms),+ closed: AtomicBool::new(false),+ };+ client.health_check().await?;+ Ok(client)+ }++ async fn execute<T, F, Fut>(&self, operation: &'static str, call: F) -> CacheResult<T>+ where+ T: Send,+ F: FnOnce(MultiplexedConnection) -> Fut + Send,+ Fut: std::future::Future<Output = Result<T, RedisError>> + Send,+ {+ if self.closed.load(Ordering::Acquire) {+ return Err(CacheError::Unavailable("Redis provider is closed".into()));+ }++ let work = async {+ let _permit = Arc::clone(&self.concurrency)+ .acquire_owned()+ .await+ .map_err(|_| CacheError::Unavailable("Redis client is closing".into()))?;+ if self.closed.load(Ordering::Acquire) {+ return Err(CacheError::Unavailable("Redis provider is closed".into()));+ }+ let connection = self.connection.read().await.clone().ok_or_else(|| {+ CacheError::Unavailable("Redis connection has been released".into())+ })?;+ call(connection)+ .await+ .map_err(|error| map_redis_error(operation, error))+ };++ tokio::time::timeout(self.command_timeout, work)+ .await+ .map_err(|_| {+ CacheError::Timeout(format!(+ "Redis {operation} exceeded {} ms",+ self.command_timeout.as_millis()+ ))+ })?+ }++ pub(crate) async fn health_check(&self) -> CacheResult<()> {+ self.execute("PING", |mut connection| async move {+ redis::cmd("PING").query_async(&mut connection).await+ })+ .await+ }++ pub(crate) async fn get(&self, key: String) -> CacheResult<Option<Vec<u8>>> {+ self.execute(+ "GET",+ |mut connection| async move { connection.get(key).await },+ )+ .await+ }++ pub(crate) async fn set(+ &self,+ key: String,+ value: Vec<u8>,+ ttl_ms: Option<u64>,+ ) -> CacheResult<()> {+ self.execute("SET", |mut connection| async move {+ if let Some(ttl_ms) = ttl_ms {+ redis::cmd("SET")+ .arg(key)+ .arg(value)+ .arg("PX")+ .arg(ttl_ms)+ .query_async(&mut connection)+ .await+ } else {+ connection.set(key, value).await+ }+ })+ .await+ }++ pub(crate) async fn delete(&self, key: String) -> CacheResult<()> {+ self.execute("DEL", |mut connection| async move {+ let _: u64 = connection.del(key).await?;+ Ok(())+ })+ .await+ }++ pub(crate) async fn exists(&self, key: String) -> CacheResult<bool> {+ self.execute("EXISTS", |mut connection| async move {+ connection.exists(key).await+ })+ .await+ }++ pub(crate) async fn batch_get(&self, keys: Vec<String>) -> CacheResult<Vec<Option<Vec<u8>>>> {+ if keys.is_empty() {+ return Ok(Vec::new());+ }+ self.execute("MGET", |mut connection| async move {+ connection.get(keys).await+ })+ .await+ }++ pub(crate) async fn batch_set(+ &self,+ entries: Vec<(String, Vec<u8>)>,+ ttl_ms: Option<u64>,+ ) -> CacheResult<()> {+ if entries.is_empty() {+ return Ok(());+ }+ self.execute("pipeline SET", |mut connection| async move {+ let mut pipe = redis::pipe();+ for (key, value) in entries {+ if let Some(ttl_ms) = ttl_ms {+ pipe.cmd("SET")+ .arg(key)+ .arg(value)+ .arg("PX")+ .arg(ttl_ms)+ .ignore();+ } else {+ pipe.cmd("SET").arg(key).arg(value).ignore();+ }+ }+ pipe.query_async(&mut connection).await+ })+ .await+ }++ pub(crate) async fn batch_delete(&self, keys: Vec<String>) -> CacheResult<()> {+ if keys.is_empty() {+ return Ok(());+ }+ self.execute("DEL", |mut connection| async move {+ let _: u64 = connection.del(keys).await?;+ Ok(())+ })+ .await+ }++ pub(crate) async fn close(&self) -> CacheResult<()> {+ if self.closed.swap(true, Ordering::AcqRel) {+ return Ok(());+ }+ let permits = Arc::clone(&self.concurrency)+ .acquire_many_owned(self.concurrency_limit)+ .await+ .map_err(|_| CacheError::Unavailable("Redis client is closing".into()))?;+ self.connection.write().await.take();+ drop(permits);+ Ok(())+ }+}++fn endpoint_url(config: &RedisConfig) -> CacheResult<String> {+ let endpoint = config.endpoints[0].clone();+ if config.username.is_empty() && config.password_env.is_empty() {+ return Ok(endpoint);+ }++ let mut url = Url::parse(&endpoint).map_err(|error| {+ CacheError::InvalidArgument(format!("Redis endpoint URL is invalid: {error}"))+ })?;+ if !config.username.is_empty() {+ url.set_username(&config.username)+ .map_err(|_| CacheError::InvalidArgument("Redis username is invalid".into()))?;+ }+ if !config.password_env.is_empty() {+ let password = env::var(&config.password_env).map_err(|_| {+ CacheError::InvalidArgument(format!(+ "Redis password_env {} is not set",+ config.password_env+ ))+ })?;+ url.set_password(Some(&password))+ .map_err(|_| CacheError::InvalidArgument("Redis password is invalid".into()))?;+ }+ Ok(url.to_string())+}++fn map_redis_error(operation: &str, error: RedisError) -> CacheError {+ if error.is_timeout() {+ return CacheError::Timeout(format!("Redis {operation} timed out: {error}"));+ }+ if error.is_connection_refusal()+ || error.is_connection_dropped()+ || error.is_cluster_error()+ || error.is_io_error()+ {+ return CacheError::Unavailable(format!("Redis {operation} unavailable: {error}"));+ }+ CacheError::Internal(format!("Redis {operation} failed: {error}"))+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn endpoint_uses_password_from_environment() {+ std::env::set_var("RAGFS_REDIS_TEST_PASSWORD", "secret");+ let config = RedisConfig {+ username: "user".into(),+ password_env: "RAGFS_REDIS_TEST_PASSWORD".into(),+ ..RedisConfig::default()+ };++ let endpoint = endpoint_url(&config).unwrap();++ assert_eq!(endpoint, "redis://user:[email protected]:6379");+ std::env::remove_var("RAGFS_REDIS_TEST_PASSWORD");+ }+}diff --git a/crates/ragfs-cache-redis/src/config.rs b/crates/ragfs-cache-redis/src/config.rsnew file mode 100644index 000000000..faf2d7fad--- /dev/null+++ b/crates/ragfs-cache-redis/src/config.rs@@ -0,0 +1,133 @@+use ragfs::cache::{CacheError, CacheResult};++/// Connection and execution settings for a Redis provider.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct RedisConfig {+ /// Redis deployment mode. The first adapter stage supports standalone.+ pub mode: String,+ /// Redis endpoints. Standalone mode uses the first endpoint.+ pub endpoints: Vec<String>,+ /// Optional ACL username.+ pub username: String,+ /// Environment variable name containing the Redis password.+ pub password_env: String,+ /// Maximum number of concurrent Redis commands issued by this provider.+ pub pool_size: usize,+ /// Timeout used while establishing the Redis connection.+ pub connect_timeout_ms: u64,+ /// Timeout applied to each Redis command.+ pub command_timeout_ms: u64,+ /// Prefix prepended to every cache key before it reaches Redis.+ pub key_prefix: String,+ /// Redis TTL in seconds. Zero disables native Redis expiration.+ pub default_ttl_seconds: u64,+ /// Whether reads may use replicas. The first adapter stage is primary-only.+ pub read_from_replica: bool,+}++impl Default for RedisConfig {+ fn default() -> Self {+ Self {+ mode: "standalone".into(),+ endpoints: vec!["redis://127.0.0.1:6379".into()],+ username: String::new(),+ password_env: String::new(),+ pool_size: 32,+ connect_timeout_ms: 1_000,+ command_timeout_ms: 20,+ key_prefix: "ragfs-cache".into(),+ default_ttl_seconds: 3_600,+ read_from_replica: false,+ }+ }+}++impl RedisConfig {+ pub(crate) fn validate(&self) -> CacheResult<()> {+ if self.mode != "standalone" {+ return Err(CacheError::InvalidArgument(+ "Redis mode must be standalone in this adapter stage".into(),+ ));+ }+ if self.endpoints.is_empty() {+ return Err(CacheError::InvalidArgument(+ "Redis endpoints must not be empty".into(),+ ));+ }+ if self+ .endpoints+ .iter()+ .any(|endpoint| endpoint.trim().is_empty())+ {+ return Err(CacheError::InvalidArgument(+ "Redis endpoints must not contain empty values".into(),+ ));+ }+ if self.pool_size == 0 {+ return Err(CacheError::InvalidArgument(+ "Redis pool_size must be greater than zero".into(),+ ));+ }+ if self.connect_timeout_ms == 0 {+ return Err(CacheError::InvalidArgument(+ "Redis connect_timeout_ms must be greater than zero".into(),+ ));+ }+ if self.command_timeout_ms == 0 {+ return Err(CacheError::InvalidArgument(+ "Redis command_timeout_ms must be greater than zero".into(),+ ));+ }+ if self.key_prefix.trim().is_empty() {+ return Err(CacheError::InvalidArgument(+ "Redis key_prefix must not be empty".into(),+ ));+ }+ if self.read_from_replica {+ return Err(CacheError::InvalidArgument(+ "Redis read_from_replica is not supported in standalone mode".into(),+ ));+ }+ Ok(())+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn validates_required_redis_settings() {+ let config = RedisConfig {+ mode: "cluster".into(),+ ..RedisConfig::default()+ };+ assert!(matches!(+ config.validate().unwrap_err(),+ CacheError::InvalidArgument(_)+ ));++ let mut config = RedisConfig::default();+ config.endpoints.clear();+ assert!(matches!(+ config.validate().unwrap_err(),+ CacheError::InvalidArgument(_)+ ));++ let config = RedisConfig {+ pool_size: 0,+ ..RedisConfig::default()+ };+ assert!(matches!(+ config.validate().unwrap_err(),+ CacheError::InvalidArgument(_)+ ));++ let mut config = RedisConfig::default();+ config.key_prefix.clear();+ assert!(matches!(+ config.validate().unwrap_err(),+ CacheError::InvalidArgument(_)+ ));+ }+}diff --git a/crates/ragfs-cache-redis/src/lib.rs b/crates/ragfs-cache-redis/src/lib.rsnew file mode 100644index 000000000..6efa84140--- /dev/null+++ b/crates/ragfs-cache-redis/src/lib.rs@@ -0,0 +1,8 @@+//! Redis cache provider adapter for RAGFS.++mod client;+mod config;+mod provider;++pub use config::RedisConfig;+pub use provider::RedisProvider;diff --git a/crates/ragfs-cache-redis/src/provider.rs b/crates/ragfs-cache-redis/src/provider.rsnew file mode 100644index 000000000..8c61f64b8--- /dev/null+++ b/crates/ragfs-cache-redis/src/provider.rs@@ -0,0 +1,193 @@+use crate::client::RedisClient;+use crate::RedisConfig;+use async_trait::async_trait;+use bytes::Bytes;+use ragfs::cache::{CacheError, CacheProvider, CacheResult, ProviderCapabilities};+use std::collections::HashSet;+use std::fmt;+use std::sync::{Arc, Mutex, MutexGuard};++/// Redis implementation of the common RAGFS cache provider contract.+pub struct RedisProvider {+ client: Arc<RedisClient>,+ key_prefix: String,+ ttl_ms: Option<u64>,+ known_keys: Mutex<HashSet<String>>,+}++impl fmt::Debug for RedisProvider {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter+ .debug_struct("RedisProvider")+ .field("key_prefix", &self.key_prefix)+ .field("ttl_ms", &self.ttl_ms)+ .finish_non_exhaustive()+ }+}++impl RedisProvider {+ /// Connect to Redis and validate the provider with PING.+ pub async fn connect(config: RedisConfig) -> CacheResult<Self> {+ config.validate()?;+ let key_prefix = normalized_prefix(&config.key_prefix);+ let ttl_ms = ttl_ms(config.default_ttl_seconds)?;+ let client = Arc::new(RedisClient::connect(&config).await?);+ Ok(Self {+ client,+ key_prefix,+ ttl_ms,+ known_keys: Mutex::new(HashSet::new()),+ })+ }++ /// Check whether the connected Redis service is healthy.+ pub async fn health_check(&self) -> CacheResult<()> {+ self.client.health_check().await+ }++ fn redis_key(&self, key: &str) -> String {+ format!("{}{}", self.key_prefix, key)+ }+}++fn normalized_prefix(prefix: &str) -> String {+ if prefix.ends_with(':') {+ prefix.to_owned()+ } else {+ format!("{prefix}:")+ }+}++fn ttl_ms(ttl_seconds: u64) -> CacheResult<Option<u64>> {+ if ttl_seconds == 0 {+ return Ok(None);+ }+ ttl_seconds+ .checked_mul(1_000)+ .map(Some)+ .ok_or_else(|| CacheError::InvalidArgument("Redis default_ttl_seconds is too large".into()))+}++fn lock_known_keys(+ known_keys: &Mutex<HashSet<String>>,+) -> CacheResult<MutexGuard<'_, HashSet<String>>> {+ known_keys+ .lock()+ .map_err(|_| CacheError::Internal("Redis key tracker is poisoned".into()))+}++#[async_trait]+impl CacheProvider for RedisProvider {+ fn name(&self) -> &'static str {+ "redis"+ }++ fn capabilities(&self) -> ProviderCapabilities {+ ProviderCapabilities {+ batch_get: true,+ batch_put: true,+ native_ttl: self.ttl_ms.is_some(),+ }+ }++ async fn get(&self, key: &str) -> CacheResult<Option<Bytes>> {+ Ok(self.client.get(self.redis_key(key)).await?.map(Bytes::from))+ }++ async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> {+ self.client+ .set(self.redis_key(key), value.to_vec(), self.ttl_ms)+ .await?;+ lock_known_keys(&self.known_keys)?.insert(key.to_owned());+ Ok(())+ }++ async fn delete(&self, key: &str) -> CacheResult<()> {+ self.client.delete(self.redis_key(key)).await?;+ lock_known_keys(&self.known_keys)?.remove(key);+ Ok(())+ }++ async fn exists(&self, key: &str) -> CacheResult<bool> {+ self.client.exists(self.redis_key(key)).await+ }++ async fn batch_get(&self, keys: &[String]) -> CacheResult<Vec<Option<Bytes>>> {+ let redis_keys = keys.iter().map(|key| self.redis_key(key)).collect();+ Ok(self+ .client+ .batch_get(redis_keys)+ .await?+ .into_iter()+ .map(|value| value.map(Bytes::from))+ .collect())+ }++ async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> {+ let redis_entries = entries+ .iter()+ .map(|(key, value)| (self.redis_key(key), value.to_vec()))+ .collect();+ self.client.batch_set(redis_entries, self.ttl_ms).await?;+ lock_known_keys(&self.known_keys)?.extend(entries.into_iter().map(|(key, _)| key));+ Ok(())+ }++ async fn invalidate(&self, keys: &[String]) -> CacheResult<()> {+ let redis_keys = keys.iter().map(|key| self.redis_key(key)).collect();+ self.client.batch_delete(redis_keys).await?;+ let mut known_keys = lock_known_keys(&self.known_keys)?;+ for key in keys {+ known_keys.remove(key);+ }+ Ok(())+ }++ async fn flush(&self) -> CacheResult<()> {+ let keys = lock_known_keys(&self.known_keys)?+ .iter()+ .cloned()+ .collect::<Vec<_>>();+ self.invalidate(&keys).await+ }++ async fn close(&self) -> CacheResult<()> {+ lock_known_keys(&self.known_keys)?.clear();+ self.client.close().await+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::panic::{self, AssertUnwindSafe};++ #[test]+ fn normalizes_key_prefix_once() {+ assert_eq!(normalized_prefix("ragfs-cache"), "ragfs-cache:");+ assert_eq!(normalized_prefix("ragfs-cache:"), "ragfs-cache:");+ }++ #[test]+ fn ttl_zero_disables_native_expiration() {+ assert_eq!(ttl_ms(0).unwrap(), None);+ assert_eq!(ttl_ms(2).unwrap(), Some(2_000));+ }++ #[test]+ fn lock_known_keys_returns_internal_error_when_poisoned() {+ let known_keys = Mutex::new(HashSet::new());++ let _ = panic::catch_unwind(AssertUnwindSafe(|| {+ let _guard = known_keys.lock().unwrap();+ panic!("poison known_keys");+ }));++ let error = lock_known_keys(&known_keys).unwrap_err();++ assert!(matches!(+ error,+ CacheError::Internal(message) if message == "Redis key tracker is poisoned"+ ));+ }+}diff --git a/crates/ragfs-cache-redis/tests/cached_filesystem.rs b/crates/ragfs-cache-redis/tests/cached_filesystem.rsnew file mode 100644index 000000000..df3c2ceae--- /dev/null+++ b/crates/ragfs-cache-redis/tests/cached_filesystem.rs@@ -0,0 +1,226 @@+use async_trait::async_trait;+use ragfs::cache::{CacheNamespace, CachePolicy, CacheProvider, CachedFileSystem};+use ragfs::core::{GrepResult, TreeEntry};+use ragfs::plugins::MemFileSystem;+use ragfs::{FileInfo, FileSystem, Result as FsResult, WriteFlag};+use ragfs_cache_redis::{RedisConfig, RedisProvider};+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::Arc;++#[derive(Clone)]+struct CountingFileSystem {+ inner: Arc<MemFileSystem>,+ reads: Arc<AtomicU64>,+ read_dirs: Arc<AtomicU64>,+}++impl CountingFileSystem {+ fn new() -> Self {+ Self {+ inner: Arc::new(MemFileSystem::new()),+ reads: Arc::new(AtomicU64::new(0)),+ read_dirs: Arc::new(AtomicU64::new(0)),+ }+ }+}++#[async_trait]+impl FileSystem for CountingFileSystem {+ async fn create(&self, path: &str) -> FsResult<()> {+ self.inner.create(path).await+ }++ async fn mkdir(&self, path: &str, mode: u32) -> FsResult<()> {+ self.inner.mkdir(path, mode).await+ }++ async fn remove(&self, path: &str) -> FsResult<()> {+ self.inner.remove(path).await+ }++ async fn remove_all(&self, path: &str) -> FsResult<()> {+ self.inner.remove_all(path).await+ }++ async fn read(&self, path: &str, offset: u64, size: u64) -> FsResult<Vec<u8>> {+ self.reads.fetch_add(1, Ordering::SeqCst);+ self.inner.read(path, offset, size).await+ }++ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> FsResult<u64> {+ self.inner.write(path, data, offset, flags).await+ }++ async fn read_dir(&self, path: &str) -> FsResult<Vec<FileInfo>> {+ self.read_dirs.fetch_add(1, Ordering::SeqCst);+ self.inner.read_dir(path).await+ }++ async fn stat(&self, path: &str) -> FsResult<FileInfo> {+ self.inner.stat(path).await+ }++ async fn rename(&self, old_path: &str, new_path: &str) -> FsResult<()> {+ self.inner.rename(old_path, new_path).await+ }++ async fn chmod(&self, path: &str, mode: u32) -> FsResult<()> {+ self.inner.chmod(path, mode).await+ }++ async fn truncate(&self, path: &str, size: u64) -> FsResult<()> {+ self.inner.truncate(path, size).await+ }++ async fn grep(+ &self,+ path: &str,+ pattern: &str,+ recursive: bool,+ case_insensitive: bool,+ node_limit: Option<usize>,+ exclude_path: Option<&str>,+ level_limit: Option<usize>,+ ) -> FsResult<GrepResult> {+ self.inner+ .grep(+ path,+ pattern,+ recursive,+ case_insensitive,+ node_limit,+ exclude_path,+ level_limit,+ )+ .await+ }++ async fn tree_directory(+ &self,+ path: &str,+ show_hidden: bool,+ node_limit: Option<usize>,+ level_limit: Option<usize>,+ ) -> FsResult<Vec<TreeEntry>> {+ self.inner+ .tree_directory(path, show_hidden, node_limit, level_limit)+ .await+ }+}++fn config(test_name: &str) -> Option<RedisConfig> {+ let endpoint = std::env::var("REDIS_URL").ok()?;+ Some(RedisConfig {+ endpoints: vec![endpoint],+ key_prefix: format!("ragfs-cache-fs-test:{}:{}", std::process::id(), test_name),+ connect_timeout_ms: 30_000,+ command_timeout_ms: 1_000,+ default_ttl_seconds: 60,+ ..RedisConfig::default()+ })+}++async fn cached_fs(backend: CountingFileSystem, test_name: &str) -> Option<CachedFileSystem> {+ let provider: Arc<dyn CacheProvider> =+ Arc::new(RedisProvider::connect(config(test_name)?).await.unwrap());+ Some(CachedFileSystem::new(+ Box::new(backend),+ provider,+ CacheNamespace::new(test_name),+ CachePolicy::default(),+ ))+}++#[tokio::test]+async fn redis_hit_miss_fill_and_write_after_read_are_consistent() {+ let backend = CountingFileSystem::new();+ backend+ .write("/value.md", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let Some(fs) = cached_fs(backend, "redis-read-write").await else {+ return;+ };++ assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old");+ assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old");+ assert_eq!(probe.reads.load(Ordering::SeqCst), 1);++ fs.write("/value.md", b"new", 0, WriteFlag::Truncate)+ .await+ .unwrap();+ assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"new");+ assert_eq!(probe.reads.load(Ordering::SeqCst), 1);+ fs.provider().flush().await.unwrap();+ fs.provider().close().await.unwrap();+}++#[tokio::test]+async fn redis_directory_and_mutation_invalidation_stay_consistent() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/root", 0o755).await.unwrap();+ backend.mkdir("/root/tree", 0o755).await.unwrap();+ backend+ .write("/root/tree/leaf", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let Some(fs) = cached_fs(backend, "redis-invalidation").await else {+ return;+ };++ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1);+ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1);+ assert_eq!(direct.read_dirs.load(Ordering::SeqCst), 1);+ fs.mkdir("/root/created", 0o755).await.unwrap();+ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);+ assert_eq!(direct.read_dirs.load(Ordering::SeqCst), 2);++ assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"old");+ fs.rename("/root/tree/leaf", "/root/tree/moved")+ .await+ .unwrap();+ assert!(fs.read("/root/tree/leaf", 0, 0).await.is_err());+ assert_eq!(fs.read("/root/tree/moved", 0, 0).await.unwrap(), b"old");+ fs.remove("/root/tree/moved").await.unwrap();+ assert!(fs.read("/root/tree/moved", 0, 0).await.is_err());++ direct+ .write("/root/tree/leaf", b"stale", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"stale");+ fs.remove_all("/root/tree").await.unwrap();+ direct.mkdir("/root/tree", 0o755).await.unwrap();+ direct+ .write("/root/tree/leaf", b"fresh", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"fresh");++ fs.rename("/root/tree", "/root/renamed").await.unwrap();+ assert!(fs.read("/root/tree/leaf", 0, 0).await.is_err());+ assert_eq!(fs.read("/root/renamed/leaf", 0, 0).await.unwrap(), b"fresh");+ fs.provider().flush().await.unwrap();+ fs.provider().close().await.unwrap();+}++#[tokio::test]+async fn closed_redis_provider_falls_back_without_breaking_backend_reads() {+ let backend = CountingFileSystem::new();+ backend+ .write("/available.md", b"backend", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let Some(fs) = cached_fs(backend, "redis-fallback").await else {+ return;+ };+ fs.provider().close().await.unwrap();++ assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend");+ assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend");+ assert_eq!(probe.reads.load(Ordering::SeqCst), 2);+ assert!(fs.metrics().snapshot().errors >= 1);+}diff --git a/crates/ragfs-cache-redis/tests/provider_contract.rs b/crates/ragfs-cache-redis/tests/provider_contract.rsnew file mode 100644index 000000000..da130af54--- /dev/null+++ b/crates/ragfs-cache-redis/tests/provider_contract.rs@@ -0,0 +1,101 @@+use bytes::Bytes;+use ragfs::cache::CacheProvider;+use ragfs_cache_redis::{RedisConfig, RedisProvider};++fn config(test_name: &str) -> Option<RedisConfig> {+ let endpoint = std::env::var("REDIS_URL").ok()?;+ Some(RedisConfig {+ endpoints: vec![endpoint],+ key_prefix: format!("ragfs-cache-test:{}:{}", std::process::id(), test_name),+ connect_timeout_ms: 30_000,+ command_timeout_ms: 1_000,+ default_ttl_seconds: 60,+ ..RedisConfig::default()+ })+}++async fn provider(test_name: &str) -> Option<RedisProvider> {+ Some(RedisProvider::connect(config(test_name)?).await.unwrap())+}++#[tokio::test]+async fn hit_miss_write_delete_and_exists_map_to_redis_operations() {+ let Some(provider) = provider("contract-basic").await else {+ return;+ };++ assert_eq!(provider.get("missing").await.unwrap(), None);+ provider+ .put("hit", Bytes::from_static(b"value"))+ .await+ .unwrap();+ assert!(provider.exists("hit").await.unwrap());+ assert_eq!(+ provider.get("hit").await.unwrap(),+ Some(Bytes::from_static(b"value"))+ );++ provider.delete("hit").await.unwrap();+ provider.delete("hit").await.unwrap();+ assert!(!provider.exists("hit").await.unwrap());+ assert_eq!(provider.get("hit").await.unwrap(), None);+ provider.flush().await.unwrap();+ provider.close().await.unwrap();+}++#[tokio::test]+async fn batch_operations_preserve_order_and_flush_only_known_keys() {+ let Some(provider) = provider("contract-batch").await else {+ return;+ };++ provider+ .batch_put(vec![+ ("one".into(), Bytes::from_static(b"1")),+ ("two".into(), Bytes::from_static(b"2")),+ ])+ .await+ .unwrap();++ assert_eq!(+ provider+ .batch_get(&["two".into(), "missing".into(), "one".into()])+ .await+ .unwrap(),+ vec![+ Some(Bytes::from_static(b"2")),+ None,+ Some(Bytes::from_static(b"1"))+ ]+ );+ assert!(provider.capabilities().batch_get);+ assert!(provider.capabilities().batch_put);+ assert!(provider.capabilities().native_ttl);++ provider.flush().await.unwrap();+ assert_eq!(provider.get("one").await.unwrap(), None);+ assert_eq!(provider.get("two").await.unwrap(), None);+ provider.close().await.unwrap();+}++#[tokio::test]+async fn set_px_ttl_expires_to_cache_miss() {+ let Some(mut config) = config("contract-ttl") else {+ return;+ };+ config.default_ttl_seconds = 1;+ let provider = RedisProvider::connect(config).await.unwrap();++ provider+ .put("ttl", Bytes::from_static(b"short"))+ .await+ .unwrap();+ assert_eq!(+ provider.get("ttl").await.unwrap(),+ Some(Bytes::from_static(b"short"))+ );++ tokio::time::sleep(std::time::Duration::from_millis(1_200)).await;+ assert_eq!(provider.get("ttl").await.unwrap(), None);+ provider.close().await.unwrap();+}diff --git a/crates/ragfs-cache-yuanrong-sys/Cargo.toml b/crates/ragfs-cache-yuanrong-sys/Cargo.tomlnew file mode 100644index 000000000..76d03847d--- /dev/null+++ b/crates/ragfs-cache-yuanrong-sys/Cargo.toml@@ -0,0 +1,16 @@+[package]+name = "ragfs-cache-yuanrong-sys"+version = "0.1.0"+edition = "2021"+rust-version = "1.91.1"+license = "Apache-2.0"+description = "Native C ABI bridge for the Yuanrong DataSystem C++ SDK"+links = "openviking_yuanrong_bridge"+build = "build.rs"++[build-dependencies]+cc = "1"++[features]+default = []+native = []diff --git a/crates/ragfs-cache-yuanrong-sys/build.rs b/crates/ragfs-cache-yuanrong-sys/build.rsnew file mode 100644index 000000000..c0d2315ef--- /dev/null+++ b/crates/ragfs-cache-yuanrong-sys/build.rs@@ -0,0 +1,41 @@+use std::env;+use std::path::PathBuf;++fn main() {+ println!("cargo:rerun-if-changed=native/yuanrong_bridge.cpp");+ println!("cargo:rerun-if-changed=native/yuanrong_bridge.h");+ println!("cargo:rerun-if-env-changed=YUANRONG_SDK_INCLUDE");+ println!("cargo:rerun-if-env-changed=YUANRONG_SDK_LIB_DIR");+ println!("cargo:rerun-if-env-changed=YUANRONG_SDK_LIB_NAME");++ if env::var_os("CARGO_FEATURE_NATIVE").is_none() {+ return;+ }++ let include = required_path("YUANRONG_SDK_INCLUDE");+ let lib_dir = required_path("YUANRONG_SDK_LIB_DIR");+ let lib_name = env::var("YUANRONG_SDK_LIB_NAME").unwrap_or_else(|_| "datasystem".into());++ cc::Build::new()+ .cpp(true)+ .std("c++17")+ .warnings(true)+ .include("native")+ .include(include)+ .file("native/yuanrong_bridge.cpp")+ .compile("openviking_yuanrong_bridge");++ println!("cargo:rustc-link-search=native={}", lib_dir.display());+ println!("cargo:rustc-link-lib=dylib={lib_name}");+}++fn required_path(name: &str) -> PathBuf {+ let value = env::var_os(name).unwrap_or_else(|| {+ panic!("{name} must point to the Yuanrong SDK directory when feature `native` is enabled")+ });+ let path = PathBuf::from(value);+ if !path.exists() {+ panic!("{name} does not exist: {}", path.display());+ }+ path+}diff --git a/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.cpp b/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.cppnew file mode 100644index 000000000..7e69dcd17--- /dev/null+++ b/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.cpp@@ -0,0 +1,530 @@+#include "yuanrong_bridge.h"++#include <algorithm>+#include <atomic>+#include <cstdlib>+#include <cstring>+#include <memory>+#include <mutex>+#include <new>+#include <string>+#include <vector>++#include "datasystem/datasystem.h"++using datasystem::ConnectOptions;+using datasystem::DsClient;+using datasystem::MSetParam;+using datasystem::Optional;+using datasystem::ReadOnlyBuffer;+using datasystem::Status;+using datasystem::StatusCode;+using datasystem::StringView;++struct YrClientHandle {+ std::shared_ptr<DsClient> client;+ // Serialize all SDK calls made through this native client handle. A single+ // native YuanrongProvider owns one handle today, so sdk_concurrency > 1 in+ // Rust does not create true backend concurrency for that provider.+ //+ // If the Rust layer creates multiple native clients to provide concurrent+ // channels, those channels must not be treated as a global ordering+ // guarantee for concurrent conflicting writes.+ std::mutex call_mutex;+ std::atomic<bool> shutdown { false };+};++namespace {+thread_local std::string last_error;++void set_error(const Status &status)+{+ last_error = status.ToString();+}++void set_error(const char *message)+{+ last_error = message;+}++int map_status(const Status &status)+{+ if (status.IsOk()) {+ last_error.clear();+ return YR_OK;+ }+ set_error(status);+ switch (status.GetCode()) {+ case StatusCode::K_NOT_FOUND:+ case StatusCode::K_NOT_FOUND_IN_L2CACHE:+ return YR_NOT_FOUND;+ case StatusCode::K_INVALID:+ case StatusCode::K_OUT_OF_RANGE:+ case StatusCode::K_FILE_NAME_TOO_LONG:+ return YR_INVALID_ARGUMENT;+ case StatusCode::K_MASTER_TIMEOUT:+ case StatusCode::K_RPC_DEADLINE_EXCEEDED:+ case StatusCode::K_FUTURE_TIMEOUT:+ return YR_TIMEOUT;+ case StatusCode::K_NOT_READY:+ case StatusCode::K_SHUTTING_DOWN:+ case StatusCode::K_WORKER_ABNORMAL:+ case StatusCode::K_CLIENT_WORKER_DISCONNECT:+ case StatusCode::K_RPC_UNAVAILABLE:+ case StatusCode::K_URMA_ERROR:+ case StatusCode::K_RDMA_ERROR:+ return YR_UNAVAILABLE;+ default:+ return YR_INTERNAL;+ }+}++bool valid_client(YrClientHandle *client)+{+ if (client == nullptr || client->client == nullptr) {+ set_error("Yuanrong client handle is null");+ return false;+ }+ if (client->shutdown.load()) {+ set_error("Yuanrong client is shut down");+ return false;+ }+ return true;+}++bool valid_bytes(const uint8_t *data, size_t len, const char *name)+{+ if (data == nullptr || len == 0) {+ last_error = std::string(name) + " must not be empty";+ return false;+ }+ return true;+}++std::string to_string(const uint8_t *data, size_t len)+{+ return std::string(reinterpret_cast<const char *>(data), len);+}++int copy_value(const void *source, size_t size, uint8_t **out)+{+ auto *copy = static_cast<uint8_t *>(std::malloc(size));+ if (copy == nullptr) {+ set_error("failed to allocate Yuanrong result buffer");+ return YR_INTERNAL;+ }+ std::memcpy(copy, source, size);+ *out = copy;+ return YR_OK;+}++template <typename Function>+int protect(Function &&function) noexcept+{+ try {+ return function();+ } catch (const std::exception &error) {+ last_error = std::string("Yuanrong bridge exception: ") + error.what();+ return YR_INTERNAL;+ } catch (...) {+ set_error("Yuanrong bridge caught an unknown exception");+ return YR_INTERNAL;+ }+}++template <typename Function>+void protect_void(Function &&function) noexcept+{+ try {+ function();+ } catch (const std::exception &error) {+ last_error = std::string("Yuanrong bridge exception: ") + error.what();+ } catch (...) {+ set_error("Yuanrong bridge caught an unknown exception");+ }+}+} // namespace++int client_create_impl(const char *host, uint16_t port, int32_t connect_timeout_ms,+ int32_t request_timeout_ms, YrClientHandle **out)+{+ if (host == nullptr || host[0] == '\0' || port == 0 || connect_timeout_ms <= 0+ || request_timeout_ms <= 0 || out == nullptr) {+ set_error("invalid Yuanrong connection options");+ return YR_INVALID_ARGUMENT;+ }+ *out = nullptr;+ ConnectOptions options;+ options.host = host;+ options.port = port;+ options.connectTimeoutMs = connect_timeout_ms;+ options.requestTimeoutMs = request_timeout_ms;+ auto handle = std::make_unique<YrClientHandle>();+ handle->client = std::make_shared<DsClient>(options);+ int code = map_status(handle->client->Init());+ if (code != YR_OK) {+ return code;+ }+ *out = handle.release();+ return YR_OK;+}++int client_health_check_impl(YrClientHandle *client)+{+ if (!valid_client(client)) {+ return YR_UNAVAILABLE;+ }+ std::lock_guard<std::mutex> guard(client->call_mutex);+ return map_status(client->client->KV()->HealthCheck());+}++int client_get_impl(YrClientHandle *client, const uint8_t *key, size_t key_len,+ uint8_t **data, size_t *size)+{+ if (!valid_client(client)) {+ return YR_UNAVAILABLE;+ }+ if (!valid_bytes(key, key_len, "key") || data == nullptr || size == nullptr) {+ return YR_INVALID_ARGUMENT;+ }+ *data = nullptr;+ *size = 0;+ std::lock_guard<std::mutex> guard(client->call_mutex);+ std::string value;+ int code = map_status(client->client->KV()->Get(to_string(key, key_len), value));+ if (code != YR_OK) {+ return code;+ }+ if (value.empty()) {+ set_error("Yuanrong returned an empty cache value");+ return YR_INTERNAL;+ }+ code = copy_value(value.data(), value.size(), data);+ if (code == YR_OK) {+ *size = value.size();+ }+ return code;+}++int client_set_impl(YrClientHandle *client, const uint8_t *key, size_t key_len,+ const uint8_t *data, size_t size)+{+ if (!valid_client(client)) {+ return YR_UNAVAILABLE;+ }+ if (!valid_bytes(key, key_len, "key") || !valid_bytes(data, size, "value")) {+ return YR_INVALID_ARGUMENT;+ }+ std::lock_guard<std::mutex> guard(client->call_mutex);+ StringView value(reinterpret_cast<const char *>(data), size);+ return map_status(client->client->KV()->Set(to_string(key, key_len), value));+}++int client_delete_impl(YrClientHandle *client, const uint8_t *key, size_t key_len)+{+ if (!valid_client(client)) {+ return YR_UNAVAILABLE;+ }+ if (!valid_bytes(key, key_len, "key")) {+ return YR_INVALID_ARGUMENT;+ }+ std::lock_guard<std::mutex> guard(client->call_mutex);+ int code = map_status(client->client->KV()->Del(to_string(key, key_len)));+ return code == YR_NOT_FOUND ? YR_OK : code;+}++int client_exists_impl(YrClientHandle *client, const uint8_t *key, size_t key_len,+ uint8_t *exists)+{+ if (!valid_client(client)) {+ return YR_UNAVAILABLE;+ }+ if (!valid_bytes(key, key_len, "key") || exists == nullptr) {+ return YR_INVALID_ARGUMENT;+ }+ std::lock_guard<std::mutex> guard(client->call_mutex);+ std::vector<bool> results;+ int code = map_status(client->client->KV()->Exist({ to_string(key, key_len) }, results));+ if (code == YR_NOT_FOUND) {+ *exists = 0;+ return YR_OK;+ }+ if (code != YR_OK) {+ return code;+ }+ if (results.size() != 1) {+ set_error("Yuanrong Exist returned an unexpected result count");+ return YR_INTERNAL;+ }+ *exists = results[0] ? 1 : 0;+ return YR_OK;+}++int client_mget_impl(YrClientHandle *client, const uint8_t *const *keys,+ const size_t *key_lens, size_t count, YrBuffer **values)+{+ if (!valid_client(client)) {+ return YR_UNAVAILABLE;+ }+ if (keys == nullptr || key_lens == nullptr || count == 0 || values == nullptr) {+ set_error("invalid Yuanrong mget arguments");+ return YR_INVALID_ARGUMENT;+ }+ *values = nullptr;+ std::vector<std::string> native_keys;+ native_keys.reserve(count);+ for (size_t i = 0; i < count; ++i) {+ if (!valid_bytes(keys[i], key_lens[i], "key")) {+ return YR_INVALID_ARGUMENT;+ }+ native_keys.push_back(to_string(keys[i], key_lens[i]));+ }++ std::lock_guard<std::mutex> guard(client->call_mutex);+ std::vector<Optional<ReadOnlyBuffer>> buffers;+ int code = map_status(client->client->KV()->Get(native_keys, buffers));+ if (code == YR_NOT_FOUND) {+ buffers.resize(count);+ code = YR_OK;+ }+ if (code != YR_OK) {+ return code;+ }+ if (buffers.size() != count) {+ set_error("Yuanrong MGet returned an unexpected result count");+ return YR_INTERNAL;+ }++ auto *results = static_cast<YrBuffer *>(std::calloc(count, sizeof(YrBuffer)));+ if (results == nullptr) {+ set_error("failed to allocate Yuanrong mget result array");+ return YR_INTERNAL;+ }+ for (size_t i = 0; i < count; ++i) {+ if (!buffers[i]) {+ continue;+ }+ Status latch = buffers[i]->RLatch();+ if (latch.IsError()) {+ set_error(latch);+ yr_buffers_free(results, count);+ return map_status(latch);+ }+ const auto size = static_cast<size_t>(buffers[i]->GetSize());+ code = size == 0 ? YR_INTERNAL+ : copy_value(buffers[i]->ImmutableData(), size, &results[i].data);+ Status unlatch = buffers[i]->UnRLatch();+ if (code != YR_OK) {+ yr_buffers_free(results, count);+ return code;+ }+ if (unlatch.IsError()) {+ set_error(unlatch);+ yr_buffers_free(results, count);+ return map_status(unlatch);+ }+ results[i].len = size;+ results[i].found = 1;+ }+ *values = results;+ last_error.clear();+ return YR_OK;+}++int client_mset_impl(YrClientHandle *client, const uint8_t *const *keys,+ const size_t *key_lens, const uint8_t *const *values,+ const size_t *value_lens, size_t count)+{+ if (!valid_client(client)) {+ return YR_UNAVAILABLE;+ }+ if (keys == nullptr || key_lens == nullptr || values == nullptr || value_lens == nullptr+ || count == 0) {+ set_error("invalid Yuanrong mset arguments");+ return YR_INVALID_ARGUMENT;+ }+ std::vector<std::string> native_keys;+ std::vector<StringView> native_values;+ native_keys.reserve(count);+ native_values.reserve(count);+ for (size_t i = 0; i < count; ++i) {+ if (!valid_bytes(keys[i], key_lens[i], "key")+ || !valid_bytes(values[i], value_lens[i], "value")) {+ return YR_INVALID_ARGUMENT;+ }+ native_keys.push_back(to_string(keys[i], key_lens[i]));+ native_values.emplace_back(reinterpret_cast<const char *>(values[i]), value_lens[i]);+ }+ std::lock_guard<std::mutex> guard(client->call_mutex);+ std::vector<std::string> failed_keys;+ int code = map_status(client->client->KV()->MSet(native_keys, native_values, failed_keys,+ MSetParam {}));+ if (code != YR_OK) {+ return code;+ }+ for (const auto &failed_key : failed_keys) {+ auto position = std::find(native_keys.begin(), native_keys.end(), failed_key);+ if (position == native_keys.end()) {+ set_error("Yuanrong MSet returned an unknown failed key");+ return YR_INTERNAL;+ }+ const auto index = static_cast<size_t>(std::distance(native_keys.begin(), position));+ code = map_status(client->client->KV()->Set(failed_key, native_values[index]));+ if (code != YR_OK) {+ return code;+ }+ }+ return YR_OK;+}++int client_mdelete_impl(YrClientHandle *client, const uint8_t *const *keys,+ const size_t *key_lens, size_t count)+{+ if (!valid_client(client)) {+ return YR_UNAVAILABLE;+ }+ if (keys == nullptr || key_lens == nullptr || count == 0) {+ set_error("invalid Yuanrong mdelete arguments");+ return YR_INVALID_ARGUMENT;+ }+ std::vector<std::string> native_keys;+ native_keys.reserve(count);+ for (size_t i = 0; i < count; ++i) {+ if (!valid_bytes(keys[i], key_lens[i], "key")) {+ return YR_INVALID_ARGUMENT;+ }+ native_keys.push_back(to_string(keys[i], key_lens[i]));+ }+ std::lock_guard<std::mutex> guard(client->call_mutex);+ std::vector<std::string> failed_keys;+ int code = map_status(client->client->KV()->Del(native_keys, failed_keys));+ if (code == YR_NOT_FOUND) {+ return YR_OK;+ }+ if (code != YR_OK) {+ return code;+ }+ for (const auto &failed_key : failed_keys) {+ code = map_status(client->client->KV()->Del(failed_key));+ if (code != YR_OK && code != YR_NOT_FOUND) {+ return code;+ }+ }+ return YR_OK;+}++int client_shutdown_impl(YrClientHandle *client)+{+ if (client == nullptr || client->client == nullptr) {+ set_error("Yuanrong client handle is null");+ return YR_INVALID_ARGUMENT;+ }+ std::lock_guard<std::mutex> guard(client->call_mutex);+ if (client->shutdown.load()) {+ return YR_OK;+ }+ int code = map_status(client->client->ShutDown());+ if (code == YR_OK) {+ client->shutdown.store(true);+ }+ return code;+}++void client_destroy_impl(YrClientHandle *client)+{+ if (client == nullptr) {+ return;+ }+ if (!client->shutdown.load() && client->client != nullptr) {+ std::lock_guard<std::mutex> guard(client->call_mutex);+ (void)client->client->ShutDown();+ client->shutdown.store(true);+ }+ delete client;+}++extern "C" int yr_client_create(const char *host, uint16_t port, int32_t connect_timeout_ms,+ int32_t request_timeout_ms, YrClientHandle **out)+{+ return protect(+ [&] { return client_create_impl(host, port, connect_timeout_ms, request_timeout_ms, out); });+}++extern "C" int yr_client_health_check(YrClientHandle *client)+{+ return protect([&] { return client_health_check_impl(client); });+}++extern "C" int yr_client_get(YrClientHandle *client, const uint8_t *key, size_t key_len,+ uint8_t **data, size_t *size)+{+ return protect([&] { return client_get_impl(client, key, key_len, data, size); });+}++extern "C" int yr_client_set(YrClientHandle *client, const uint8_t *key, size_t key_len,+ const uint8_t *data, size_t size)+{+ return protect([&] { return client_set_impl(client, key, key_len, data, size); });+}++extern "C" int yr_client_delete(YrClientHandle *client, const uint8_t *key, size_t key_len)+{+ return protect([&] { return client_delete_impl(client, key, key_len); });+}++extern "C" int yr_client_exists(YrClientHandle *client, const uint8_t *key, size_t key_len,+ uint8_t *exists)+{+ return protect([&] { return client_exists_impl(client, key, key_len, exists); });+}++extern "C" int yr_client_mget(YrClientHandle *client, const uint8_t *const *keys,+ const size_t *key_lens, size_t count, YrBuffer **values)+{+ return protect([&] { return client_mget_impl(client, keys, key_lens, count, values); });+}++extern "C" int yr_client_mset(YrClientHandle *client, const uint8_t *const *keys,+ const size_t *key_lens, const uint8_t *const *values,+ const size_t *value_lens, size_t count)+{+ return protect(+ [&] { return client_mset_impl(client, keys, key_lens, values, value_lens, count); });+}++extern "C" int yr_client_mdelete(YrClientHandle *client, const uint8_t *const *keys,+ const size_t *key_lens, size_t count)+{+ return protect([&] { return client_mdelete_impl(client, keys, key_lens, count); });+}++extern "C" int yr_client_shutdown(YrClientHandle *client)+{+ return protect([&] { return client_shutdown_impl(client); });+}++extern "C" void yr_client_destroy(YrClientHandle *client)+{+ protect_void([&] { client_destroy_impl(client); });+}++extern "C" void yr_buffer_free(void *data)+{+ std::free(data);+}++extern "C" void yr_buffers_free(YrBuffer *values, size_t count)+{+ if (values == nullptr) {+ return;+ }+ for (size_t i = 0; i < count; ++i) {+ std::free(values[i].data);+ }+ std::free(values);+}++extern "C" const char *yr_last_error(YrClientHandle *)+{+ return last_error.c_str();+}diff --git a/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.h b/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.hnew file mode 100644index 000000000..68632f008--- /dev/null+++ b/crates/ragfs-cache-yuanrong-sys/native/yuanrong_bridge.h@@ -0,0 +1,55 @@+#ifndef OPENVIKING_YUANRONG_BRIDGE_H+#define OPENVIKING_YUANRONG_BRIDGE_H++#include <stddef.h>+#include <stdint.h>++#ifdef __cplusplus+extern "C" {+#endif++typedef struct YrClientHandle YrClientHandle;++typedef struct YrBuffer {+ uint8_t *data;+ size_t len;+ uint8_t found;+} YrBuffer;++enum YrStatus {+ YR_OK = 0,+ YR_NOT_FOUND = 1,+ YR_INVALID_ARGUMENT = 2,+ YR_UNAVAILABLE = 3,+ YR_TIMEOUT = 4,+ YR_INTERNAL = 5,+};++int yr_client_create(const char *host, uint16_t port, int32_t connect_timeout_ms,+ int32_t request_timeout_ms, YrClientHandle **out);+int yr_client_health_check(YrClientHandle *client);+int yr_client_get(YrClientHandle *client, const uint8_t *key, size_t key_len,+ uint8_t **data, size_t *size);+int yr_client_set(YrClientHandle *client, const uint8_t *key, size_t key_len,+ const uint8_t *data, size_t size);+int yr_client_delete(YrClientHandle *client, const uint8_t *key, size_t key_len);+int yr_client_exists(YrClientHandle *client, const uint8_t *key, size_t key_len,+ uint8_t *exists);+int yr_client_mget(YrClientHandle *client, const uint8_t *const *keys,+ const size_t *key_lens, size_t count, YrBuffer **values);+int yr_client_mset(YrClientHandle *client, const uint8_t *const *keys,+ const size_t *key_lens, const uint8_t *const *values,+ const size_t *value_lens, size_t count);+int yr_client_mdelete(YrClientHandle *client, const uint8_t *const *keys,+ const size_t *key_lens, size_t count);+int yr_client_shutdown(YrClientHandle *client);+void yr_client_destroy(YrClientHandle *client);+void yr_buffer_free(void *data);+void yr_buffers_free(YrBuffer *values, size_t count);+const char *yr_last_error(YrClientHandle *client);++#ifdef __cplusplus+}+#endif++#endifdiff --git a/crates/ragfs-cache-yuanrong-sys/src/lib.rs b/crates/ragfs-cache-yuanrong-sys/src/lib.rsnew file mode 100644index 000000000..5f9aca8a2--- /dev/null+++ b/crates/ragfs-cache-yuanrong-sys/src/lib.rs@@ -0,0 +1,113 @@+//! Unsafe bindings for OpenViking's stable Yuanrong C ABI bridge.++#[cfg(feature = "native")]+use std::ffi::{c_char, c_void};+use std::ffi::{c_int, c_uchar};++/// Operation completed successfully.+pub const YR_OK: c_int = 0;+/// Requested key does not exist.+pub const YR_NOT_FOUND: c_int = 1;+/// Yuanrong rejected an argument.+pub const YR_INVALID_ARGUMENT: c_int = 2;+/// Worker or client connection is unavailable.+pub const YR_UNAVAILABLE: c_int = 3;+/// Yuanrong operation timed out.+pub const YR_TIMEOUT: c_int = 4;+/// Yuanrong returned another internal error.+pub const YR_INTERNAL: c_int = 5;++/// Opaque Yuanrong client owned by the C++ bridge.+#[repr(C)]+pub struct YrClientHandle {+ _private: [u8; 0],+}++/// Buffer allocated by the C++ bridge.+#[repr(C)]+#[derive(Debug, Clone, Copy)]+pub struct YrBuffer {+ /// Buffer address, or null when the corresponding key is missing.+ pub data: *mut c_uchar,+ /// Buffer length.+ pub len: usize,+ /// One when the key exists, zero when it is missing.+ pub found: c_uchar,+}++#[cfg(feature = "native")]+unsafe extern "C" {+ /// Create and initialize a Yuanrong `DsClient`.+ pub fn yr_client_create(+ host: *const c_char,+ port: u16,+ connect_timeout_ms: i32,+ request_timeout_ms: i32,+ out: *mut *mut YrClientHandle,+ ) -> c_int;+ /// Check the local worker connection.+ pub fn yr_client_health_check(client: *mut YrClientHandle) -> c_int;+ /// Read one complete KV value.+ pub fn yr_client_get(+ client: *mut YrClientHandle,+ key: *const c_uchar,+ key_len: usize,+ data: *mut *mut c_uchar,+ size: *mut usize,+ ) -> c_int;+ /// Store one complete KV value.+ pub fn yr_client_set(+ client: *mut YrClientHandle,+ key: *const c_uchar,+ key_len: usize,+ data: *const c_uchar,+ size: usize,+ ) -> c_int;+ /// Delete one KV value.+ pub fn yr_client_delete(+ client: *mut YrClientHandle,+ key: *const c_uchar,+ key_len: usize,+ ) -> c_int;+ /// Check whether one KV value exists.+ pub fn yr_client_exists(+ client: *mut YrClientHandle,+ key: *const c_uchar,+ key_len: usize,+ exists: *mut c_uchar,+ ) -> c_int;+ /// Read multiple values, preserving key order.+ pub fn yr_client_mget(+ client: *mut YrClientHandle,+ keys: *const *const c_uchar,+ key_lens: *const usize,+ count: usize,+ values: *mut *mut YrBuffer,+ ) -> c_int;+ /// Store multiple values.+ pub fn yr_client_mset(+ client: *mut YrClientHandle,+ keys: *const *const c_uchar,+ key_lens: *const usize,+ values: *const *const c_uchar,+ value_lens: *const usize,+ count: usize,+ ) -> c_int;+ /// Delete multiple values.+ pub fn yr_client_mdelete(+ client: *mut YrClientHandle,+ keys: *const *const c_uchar,+ key_lens: *const usize,+ count: usize,+ ) -> c_int;+ /// Shut down the SDK client.+ pub fn yr_client_shutdown(client: *mut YrClientHandle) -> c_int;+ /// Destroy the bridge handle.+ pub fn yr_client_destroy(client: *mut YrClientHandle);+ /// Free a single value returned by `yr_client_get`.+ pub fn yr_buffer_free(data: *mut c_void);+ /// Free values returned by `yr_client_mget`.+ pub fn yr_buffers_free(values: *mut YrBuffer, count: usize);+ /// Return diagnostics for the last bridge call on the current thread.+ pub fn yr_last_error(client: *mut YrClientHandle) -> *const c_char;+}diff --git a/crates/ragfs-cache-yuanrong/Cargo.toml b/crates/ragfs-cache-yuanrong/Cargo.tomlnew file mode 100644index 000000000..ca45f88d9--- /dev/null+++ b/crates/ragfs-cache-yuanrong/Cargo.toml@@ -0,0 +1,24 @@+[package]+name = "ragfs-cache-yuanrong"+version = "0.1.0"+edition = "2021"+rust-version = "1.91.1"+license = "Apache-2.0"+description = "Yuanrong DataSystem cache provider adapter for RAGFS"++[dependencies]+async-trait = "0.1"+bytes = "1.5"+ragfs = { path = "../ragfs", features = ["cache"] }+ragfs-cache-yuanrong-sys = { path = "../ragfs-cache-yuanrong-sys", optional = true }+sha2 = { version = "0.10", optional = true }+thiserror = "1.0"+tokio = { version = "1.38", features = ["rt-multi-thread", "sync", "time", "macros"] }++[features]+default = []+yuanrong-native = [+ "dep:ragfs-cache-yuanrong-sys",+ "dep:sha2",+ "ragfs-cache-yuanrong-sys/native",+]diff --git a/crates/ragfs-cache-yuanrong/src/client.rs b/crates/ragfs-cache-yuanrong/src/client.rsnew file mode 100644index 000000000..e4a6aac82--- /dev/null+++ b/crates/ragfs-cache-yuanrong/src/client.rs@@ -0,0 +1,164 @@+use crate::{YuanrongKvStore, YuanrongStoreError};+use ragfs::cache::{CacheError, CacheResult};+use std::sync::atomic::{AtomicBool, Ordering};+use std::sync::Arc;+use std::time::Duration;+use tokio::sync::{RwLock, Semaphore};++pub(crate) struct YuanrongClient {+ store: RwLock<Option<Arc<dyn YuanrongKvStore>>>,+ concurrency: Arc<Semaphore>,+ concurrency_limit: u32,+ timeout: Duration,+ closed: AtomicBool,+}++impl YuanrongClient {+ pub(crate) fn new(+ store: Arc<dyn YuanrongKvStore>,+ concurrency_limit: usize,+ timeout: Duration,+ ) -> Self {+ Self {+ store: RwLock::new(Some(store)),+ concurrency: Arc::new(Semaphore::new(concurrency_limit)),+ concurrency_limit: concurrency_limit as u32,+ timeout,+ closed: AtomicBool::new(false),+ }+ }++ async fn execute<T, F>(&self, operation: &'static str, call: F) -> CacheResult<T>+ where+ T: Send + 'static,+ F: FnOnce(Arc<dyn YuanrongKvStore>) -> Result<T, YuanrongStoreError> + Send + 'static,+ {+ if self.closed.load(Ordering::Acquire) {+ return Err(CacheError::Unavailable(+ "Yuanrong provider is closed".into(),+ ));+ }++ let work = async {+ let permit = Arc::clone(&self.concurrency)+ .acquire_owned()+ .await+ .map_err(|_| CacheError::Unavailable("Yuanrong client is closing".into()))?;+ if self.closed.load(Ordering::Acquire) {+ return Err(CacheError::Unavailable(+ "Yuanrong provider is closed".into(),+ ));+ }+ let store = self.store.read().await.clone().ok_or_else(|| {+ CacheError::Unavailable("Yuanrong KV client has been released".into())+ })?;+ tokio::task::spawn_blocking(move || {+ let _permit = permit;+ call(store)+ })+ .await+ .map_err(|error| {+ CacheError::Internal(format!(+ "Yuanrong {operation} blocking task failed: {error}"+ ))+ })?+ .map_err(map_store_error)+ };++ tokio::time::timeout(self.timeout, work)+ .await+ .map_err(|_| {+ CacheError::Timeout(format!(+ "Yuanrong {operation} exceeded {} ms",+ self.timeout.as_millis()+ ))+ })?+ }++ pub(crate) async fn health_check(&self) -> CacheResult<()> {+ self.execute("health_check", |store| store.health_check())+ .await+ }++ pub(crate) async fn get(&self, key: &str) -> CacheResult<Option<Vec<u8>>> {+ let key = key.to_owned();+ self.execute("get", move |store| store.get(&key)).await+ }++ pub(crate) async fn set(&self, key: &str, value: &[u8]) -> CacheResult<()> {+ let key = key.to_owned();+ let value = value.to_vec();+ self.execute("set", move |store| store.set(&key, &value))+ .await+ }++ pub(crate) async fn delete(&self, key: &str) -> CacheResult<()> {+ let key = key.to_owned();+ self.execute("delete", move |store| store.delete(&key))+ .await+ }++ pub(crate) async fn exists(&self, key: &str) -> CacheResult<bool> {+ let key = key.to_owned();+ self.execute("exists", move |store| store.exists(&key))+ .await+ }++ pub(crate) async fn batch_get(&self, keys: &[String]) -> CacheResult<Vec<Option<Vec<u8>>>> {+ let keys = keys.to_vec();+ self.execute("batch_get", move |store| store.batch_get(&keys))+ .await+ }++ pub(crate) async fn batch_set(&self, entries: Vec<(String, Vec<u8>)>) -> CacheResult<()> {+ self.execute("batch_set", move |store| store.batch_set(&entries))+ .await+ }++ pub(crate) async fn batch_delete(&self, keys: &[String]) -> CacheResult<()> {+ let keys = keys.to_vec();+ self.execute("batch_delete", move |store| store.batch_delete(&keys))+ .await+ }++ pub(crate) async fn close(&self) -> CacheResult<()> {+ if self.closed.swap(true, Ordering::AcqRel) {+ return Ok(());+ }+ let permits = Arc::clone(&self.concurrency)+ .acquire_many_owned(self.concurrency_limit)+ .await+ .map_err(|_| CacheError::Unavailable("Yuanrong client is closing".into()))?;+ let store = self.store.write().await.take();+ let result = if let Some(store) = store {+ tokio::time::timeout(+ self.timeout,+ tokio::task::spawn_blocking(move || store.shutdown()),+ )+ .await+ .map_err(|_| {+ CacheError::Timeout(format!(+ "Yuanrong shutdown exceeded {} ms",+ self.timeout.as_millis()+ ))+ })?+ .map_err(|error| {+ CacheError::Internal(format!("Yuanrong shutdown task failed: {error}"))+ })?+ .map_err(map_store_error)+ } else {+ Ok(())+ };+ drop(permits);+ result+ }+}++fn map_store_error(error: YuanrongStoreError) -> CacheError {+ match error {+ YuanrongStoreError::Unavailable(message) => CacheError::Unavailable(message),+ YuanrongStoreError::Timeout(message) => CacheError::Timeout(message),+ YuanrongStoreError::InvalidArgument(message) => CacheError::InvalidArgument(message),+ YuanrongStoreError::Internal(message) => CacheError::Internal(message),+ }+}diff --git a/crates/ragfs-cache-yuanrong/src/config.rs b/crates/ragfs-cache-yuanrong/src/config.rsnew file mode 100644index 000000000..0acbf46c9--- /dev/null+++ b/crates/ragfs-cache-yuanrong/src/config.rs@@ -0,0 +1,52 @@+use ragfs::cache::{CacheError, CacheResult};++/// Connection and execution settings for a Yuanrong provider.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct YuanrongConfig {+ /// Yuanrong worker host.+ pub host: String,+ /// Yuanrong worker port.+ pub port: u16,+ /// Timeout used while connecting to the worker.+ pub connect_timeout_ms: u64,+ /// Timeout applied to each KV operation.+ pub request_timeout_ms: u64,+ /// Maximum number of concurrent blocking tasks allowed by the Rust client.+ ///+ /// The current native bridge creates one native client handle per+ /// `YuanrongProvider`; calls through that handle are serialized in C++.+ /// Therefore `sdk_concurrency > 1` does not provide true Yuanrong backend+ /// concurrency for a single native provider instance.+ pub sdk_concurrency: usize,+}++impl YuanrongConfig {+ pub(crate) fn validate(&self) -> CacheResult<()> {+ if self.host.trim().is_empty() {+ return Err(CacheError::InvalidArgument(+ "Yuanrong host must not be empty".into(),+ ));+ }+ if self.port == 0 {+ return Err(CacheError::InvalidArgument(+ "Yuanrong port must be greater than zero".into(),+ ));+ }+ if self.connect_timeout_ms == 0 || self.connect_timeout_ms > i32::MAX as u64 {+ return Err(CacheError::InvalidArgument(+ "Yuanrong connect_timeout_ms must be between 1 and i32::MAX".into(),+ ));+ }+ if self.request_timeout_ms == 0 || self.request_timeout_ms > i32::MAX as u64 {+ return Err(CacheError::InvalidArgument(+ "Yuanrong request_timeout_ms must be between 1 and i32::MAX".into(),+ ));+ }+ if self.sdk_concurrency == 0 || self.sdk_concurrency > u32::MAX as usize {+ return Err(CacheError::InvalidArgument(+ "Yuanrong sdk_concurrency must be between 1 and u32::MAX".into(),+ ));+ }+ Ok(())+ }+}diff --git a/crates/ragfs-cache-yuanrong/src/error.rs b/crates/ragfs-cache-yuanrong/src/error.rsnew file mode 100644index 000000000..07d0e62f8--- /dev/null+++ b/crates/ragfs-cache-yuanrong/src/error.rs@@ -0,0 +1,16 @@+/// Errors returned by the synchronous Yuanrong KV boundary.+#[derive(Debug, thiserror::Error)]+pub enum YuanrongStoreError {+ /// The worker or client connection is unavailable.+ #[error("Yuanrong unavailable: {0}")]+ Unavailable(String),+ /// A Yuanrong SDK operation timed out.+ #[error("Yuanrong operation timed out: {0}")]+ Timeout(String),+ /// Yuanrong rejected an argument.+ #[error("invalid Yuanrong argument: {0}")]+ InvalidArgument(String),+ /// Yuanrong returned an unspecified operation failure.+ #[error("Yuanrong operation failed: {0}")]+ Internal(String),+}diff --git a/crates/ragfs-cache-yuanrong/src/lib.rs b/crates/ragfs-cache-yuanrong/src/lib.rsnew file mode 100644index 000000000..bf604e1b3--- /dev/null+++ b/crates/ragfs-cache-yuanrong/src/lib.rs@@ -0,0 +1,14 @@+//! Yuanrong DataSystem cache provider adapter for RAGFS.++mod client;+mod config;+mod error;+#[cfg(feature = "yuanrong-native")]+mod native;+mod provider;+mod store;++pub use config::YuanrongConfig;+pub use error::YuanrongStoreError;+pub use provider::YuanrongProvider;+pub use store::YuanrongKvStore;diff --git a/crates/ragfs-cache-yuanrong/src/native.rs b/crates/ragfs-cache-yuanrong/src/native.rsnew file mode 100644index 000000000..d55a132da--- /dev/null+++ b/crates/ragfs-cache-yuanrong/src/native.rs@@ -0,0 +1,344 @@+use crate::{YuanrongConfig, YuanrongKvStore, YuanrongProvider, YuanrongStoreError};+use ragfs::cache::{CacheError, CacheResult};+use ragfs_cache_yuanrong_sys as sys;+use sha2::{Digest, Sha256};+use std::ffi::{CStr, CString};+use std::ptr::{self, NonNull};+use std::slice;+use std::sync::Arc;+use std::time::Duration;++struct NativeYuanrongStore {+ handle: NonNull<sys::YrClientHandle>,+}++unsafe impl Send for NativeYuanrongStore {}+unsafe impl Sync for NativeYuanrongStore {}++impl NativeYuanrongStore {+ const VALUE_FORMAT_V1: u8 = 1;++ fn connect(config: &YuanrongConfig) -> Result<Self, YuanrongStoreError> {+ let host = CString::new(config.host.as_str()).map_err(|_| {+ YuanrongStoreError::InvalidArgument("host contains an embedded NUL byte".into())+ })?;+ let mut handle = ptr::null_mut();+ let code = unsafe {+ sys::yr_client_create(+ host.as_ptr(),+ config.port,+ config.connect_timeout_ms as i32,+ config.request_timeout_ms as i32,+ &mut handle,+ )+ };+ let handle = NonNull::new(handle);+ if code != sys::YR_OK {+ return Err(map_native_error(code, handle));+ }+ Ok(Self {+ handle: handle.ok_or_else(|| {+ YuanrongStoreError::Internal(+ "Yuanrong create succeeded without returning a handle".into(),+ )+ })?,+ })+ }++ fn call(&self, code: i32) -> Result<(), YuanrongStoreError> {+ if code == sys::YR_OK {+ Ok(())+ } else {+ Err(map_native_error(code, Some(self.handle)))+ }+ }++ fn native_key(key: &str) -> Result<String, YuanrongStoreError> {+ if key.is_empty() {+ return Err(YuanrongStoreError::InvalidArgument(+ "Yuanrong cache key must not be empty".into(),+ ));+ }+ let digest = Sha256::digest(key.as_bytes());+ let mut encoded = String::with_capacity(67);+ encoded.push_str("ov_");+ for byte in digest {+ use std::fmt::Write;+ write!(&mut encoded, "{byte:02x}").expect("writing to String cannot fail");+ }+ Ok(encoded)+ }++ fn native_keys(keys: &[String]) -> Result<Vec<String>, YuanrongStoreError> {+ keys.iter().map(|key| Self::native_key(key)).collect()+ }++ fn key_parts(keys: &[String]) -> (Vec<*const u8>, Vec<usize>) {+ (+ keys.iter().map(|key| key.as_ptr()).collect(),+ keys.iter().map(String::len).collect(),+ )+ }++ fn encode_value(value: &[u8]) -> Vec<u8> {+ let mut encoded = Vec::with_capacity(value.len() + 1);+ encoded.push(Self::VALUE_FORMAT_V1);+ encoded.extend_from_slice(value);+ encoded+ }++ fn decode_value(value: &[u8]) -> Result<Vec<u8>, YuanrongStoreError> {+ match value.split_first() {+ Some((&Self::VALUE_FORMAT_V1, data)) => Ok(data.to_vec()),+ _ => Err(YuanrongStoreError::Internal(+ "Yuanrong cache value has an unsupported format".into(),+ )),+ }+ }+}++impl Drop for NativeYuanrongStore {+ fn drop(&mut self) {+ unsafe {+ sys::yr_client_destroy(self.handle.as_ptr());+ }+ }+}++impl YuanrongKvStore for NativeYuanrongStore {+ fn health_check(&self) -> Result<(), YuanrongStoreError> {+ self.call(unsafe { sys::yr_client_health_check(self.handle.as_ptr()) })+ }++ fn get(&self, key: &str) -> Result<Option<Vec<u8>>, YuanrongStoreError> {+ let key = Self::native_key(key)?;+ let mut data = ptr::null_mut();+ let mut size = 0;+ let code = unsafe {+ sys::yr_client_get(+ self.handle.as_ptr(),+ key.as_ptr(),+ key.len(),+ &mut data,+ &mut size,+ )+ };+ if code == sys::YR_NOT_FOUND {+ return Ok(None);+ }+ self.call(code)?;+ let data = NonNull::new(data).ok_or_else(|| {+ YuanrongStoreError::Internal("Yuanrong get succeeded without returning a buffer".into())+ })?;+ let value = Self::decode_value(unsafe { slice::from_raw_parts(data.as_ptr(), size) });+ unsafe {+ sys::yr_buffer_free(data.as_ptr().cast());+ }+ Ok(Some(value?))+ }++ fn set(&self, key: &str, value: &[u8]) -> Result<(), YuanrongStoreError> {+ let key = Self::native_key(key)?;+ let value = Self::encode_value(value);+ self.call(unsafe {+ sys::yr_client_set(+ self.handle.as_ptr(),+ key.as_ptr(),+ key.len(),+ value.as_ptr(),+ value.len(),+ )+ })+ }++ fn delete(&self, key: &str) -> Result<(), YuanrongStoreError> {+ let key = Self::native_key(key)?;+ self.call(unsafe { sys::yr_client_delete(self.handle.as_ptr(), key.as_ptr(), key.len()) })+ }++ fn exists(&self, key: &str) -> Result<bool, YuanrongStoreError> {+ let key = Self::native_key(key)?;+ let mut exists = 0;+ self.call(unsafe {+ sys::yr_client_exists(self.handle.as_ptr(), key.as_ptr(), key.len(), &mut exists)+ })?;+ Ok(exists != 0)+ }++ fn batch_get(&self, keys: &[String]) -> Result<Vec<Option<Vec<u8>>>, YuanrongStoreError> {+ if keys.is_empty() {+ return Ok(Vec::new());+ }+ let native_keys = Self::native_keys(keys)?;+ let (key_ptrs, key_lens) = Self::key_parts(&native_keys);+ let mut values = ptr::null_mut();+ self.call(unsafe {+ sys::yr_client_mget(+ self.handle.as_ptr(),+ key_ptrs.as_ptr(),+ key_lens.as_ptr(),+ keys.len(),+ &mut values,+ )+ })?;+ let values = NonNull::new(values).ok_or_else(|| {+ YuanrongStoreError::Internal("Yuanrong mget succeeded without returning results".into())+ })?;+ let native = unsafe { slice::from_raw_parts(values.as_ptr(), keys.len()) };+ let result = native+ .iter()+ .map(|value| {+ if value.found == 0 {+ Ok(None)+ } else {+ let data = NonNull::new(value.data).ok_or_else(|| {+ YuanrongStoreError::Internal(+ "Yuanrong mget returned a null value buffer".into(),+ )+ })?;+ Self::decode_value(unsafe { slice::from_raw_parts(data.as_ptr(), value.len) })+ .map(Some)+ }+ })+ .collect();+ unsafe {+ sys::yr_buffers_free(values.as_ptr(), keys.len());+ }+ result+ }++ fn batch_set(&self, entries: &[(String, Vec<u8>)]) -> Result<(), YuanrongStoreError> {+ const MAX_MSET_KEYS: usize = 1_999;+ const MAX_MSET_ENCODED_VALUE_SIZE: usize = 500 * 1024;++ for chunk in entries.chunks(MAX_MSET_KEYS) {+ if chunk+ .iter()+ .any(|(_, value)| value.len() + 1 >= MAX_MSET_ENCODED_VALUE_SIZE)+ {+ for (key, value) in chunk {+ self.set(key, value)?;+ }+ continue;+ }+ let logical_keys = chunk.iter().map(|(key, _)| key.clone()).collect::<Vec<_>>();+ let keys = Self::native_keys(&logical_keys)?;+ let (key_ptrs, key_lens) = Self::key_parts(&keys);+ let values = chunk+ .iter()+ .map(|(_, value)| Self::encode_value(value))+ .collect::<Vec<_>>();+ let value_ptrs = values.iter().map(Vec::as_ptr).collect::<Vec<_>>();+ let value_lens = values.iter().map(Vec::len).collect::<Vec<_>>();+ self.call(unsafe {+ sys::yr_client_mset(+ self.handle.as_ptr(),+ key_ptrs.as_ptr(),+ key_lens.as_ptr(),+ value_ptrs.as_ptr(),+ value_lens.as_ptr(),+ chunk.len(),+ )+ })?;+ }+ Ok(())+ }++ fn batch_delete(&self, keys: &[String]) -> Result<(), YuanrongStoreError> {+ const MAX_BATCH_KEYS: usize = 10_000;++ for chunk in keys.chunks(MAX_BATCH_KEYS) {+ if chunk.is_empty() {+ continue;+ }+ let native_keys = Self::native_keys(chunk)?;+ let (key_ptrs, key_lens) = Self::key_parts(&native_keys);+ self.call(unsafe {+ sys::yr_client_mdelete(+ self.handle.as_ptr(),+ key_ptrs.as_ptr(),+ key_lens.as_ptr(),+ chunk.len(),+ )+ })?;+ }+ Ok(())+ }++ fn shutdown(&self) -> Result<(), YuanrongStoreError> {+ self.call(unsafe { sys::yr_client_shutdown(self.handle.as_ptr()) })+ }+}++impl YuanrongProvider {+ /// Connect to Yuanrong through the native C ABI bridge.+ pub async fn connect(config: YuanrongConfig) -> CacheResult<Self> {+ config.validate()?;+ let setup_config = config.clone();+ let setup_timeout = Duration::from_millis(config.connect_timeout_ms);+ let store = tokio::time::timeout(+ setup_timeout,+ tokio::task::spawn_blocking(move || NativeYuanrongStore::connect(&setup_config)),+ )+ .await+ .map_err(|_| {+ CacheError::Timeout(format!(+ "Yuanrong connection exceeded {} ms",+ setup_timeout.as_millis()+ ))+ })?+ .map_err(|error| CacheError::Internal(format!("Yuanrong connection task failed: {error}")))?+ .map_err(map_cache_error)?;++ // A native provider currently owns one native client handle. The C+++ // bridge serializes SDK calls on that handle, so sdk_concurrency only+ // bounds Rust blocking tasks here; it is not backend concurrency.+ Self::from_store(config, Arc::new(store)).await+ }+}++fn map_native_error(code: i32, handle: Option<NonNull<sys::YrClientHandle>>) -> YuanrongStoreError {+ let raw_handle = handle.map_or(ptr::null_mut(), NonNull::as_ptr);+ let value = unsafe { sys::yr_last_error(raw_handle) };+ let message = (!value.is_null())+ .then(|| {+ unsafe { CStr::from_ptr(value) }+ .to_string_lossy()+ .into_owned()+ })+ .filter(|message| !message.is_empty())+ .unwrap_or_else(|| format!("Yuanrong bridge returned status {code}"));++ match code {+ sys::YR_INVALID_ARGUMENT => YuanrongStoreError::InvalidArgument(message),+ sys::YR_UNAVAILABLE => YuanrongStoreError::Unavailable(message),+ sys::YR_TIMEOUT => YuanrongStoreError::Timeout(message),+ sys::YR_NOT_FOUND => {+ YuanrongStoreError::Internal(format!("unexpected not-found status: {message}"))+ }+ _ => YuanrongStoreError::Internal(message),+ }+}++fn map_cache_error(error: YuanrongStoreError) -> CacheError {+ match error {+ YuanrongStoreError::Unavailable(message) => CacheError::Unavailable(message),+ YuanrongStoreError::Timeout(message) => CacheError::Timeout(message),+ YuanrongStoreError::InvalidArgument(message) => CacheError::InvalidArgument(message),+ YuanrongStoreError::Internal(message) => CacheError::Internal(message),+ }+}++#[cfg(test)]+mod tests {+ use super::NativeYuanrongStore;++ #[test]+ fn native_value_codec_round_trips_empty_and_binary_values() {+ for value in [Vec::new(), vec![0, 1, 2, 255]] {+ let encoded = NativeYuanrongStore::encode_value(&value);+ assert!(!encoded.is_empty());+ assert_eq!(NativeYuanrongStore::decode_value(&encoded).unwrap(), value);+ }+ }+}diff --git a/crates/ragfs-cache-yuanrong/src/provider.rs b/crates/ragfs-cache-yuanrong/src/provider.rsnew file mode 100644index 000000000..9b6e2a43f--- /dev/null+++ b/crates/ragfs-cache-yuanrong/src/provider.rs@@ -0,0 +1,167 @@+use crate::client::YuanrongClient;+use crate::{YuanrongConfig, YuanrongKvStore};+use async_trait::async_trait;+use bytes::Bytes;+use ragfs::cache::{CacheError, CacheProvider, CacheResult, ProviderCapabilities};+use std::collections::HashSet;+use std::fmt;+use std::sync::{Arc, Mutex, MutexGuard};+use std::time::Duration;++/// Yuanrong implementation of the common RAGFS cache provider contract.+pub struct YuanrongProvider {+ client: Arc<YuanrongClient>,+ known_keys: Mutex<HashSet<String>>,+}++impl fmt::Debug for YuanrongProvider {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter+ .debug_struct("YuanrongProvider")+ .finish_non_exhaustive()+ }+}++impl YuanrongProvider {+ /// Construct a provider over a Yuanrong-compatible synchronous KV store.+ ///+ /// Construction validates configuration and performs a health check.+ pub async fn from_store(+ config: YuanrongConfig,+ store: Arc<dyn YuanrongKvStore>,+ ) -> CacheResult<Self> {+ config.validate()?;+ let client = Arc::new(YuanrongClient::new(+ store,+ config.sdk_concurrency,+ Duration::from_millis(config.request_timeout_ms),+ ));+ client.health_check().await?;+ Ok(Self {+ client,+ known_keys: Mutex::new(HashSet::new()),+ })+ }++ /// Return a startup error when the native Yuanrong bridge is not compiled.+ #[cfg(not(feature = "yuanrong-native"))]+ pub async fn connect(config: YuanrongConfig) -> CacheResult<Self> {+ config.validate()?;+ Err(CacheError::Unavailable(+ "Yuanrong support requires the yuanrong-native feature".into(),+ ))+ }++ /// Check whether the connected Yuanrong worker is healthy.+ pub async fn health_check(&self) -> CacheResult<()> {+ self.client.health_check().await+ }+}++fn lock_known_keys(+ known_keys: &Mutex<HashSet<String>>,+) -> CacheResult<MutexGuard<'_, HashSet<String>>> {+ known_keys+ .lock()+ .map_err(|_| CacheError::Internal("Yuanrong key tracker is poisoned".into()))+}++#[async_trait]+impl CacheProvider for YuanrongProvider {+ fn name(&self) -> &'static str {+ "yuanrong"+ }++ fn capabilities(&self) -> ProviderCapabilities {+ ProviderCapabilities {+ batch_get: true,+ batch_put: true,+ native_ttl: false,+ }+ }++ async fn get(&self, key: &str) -> CacheResult<Option<Bytes>> {+ Ok(self.client.get(key).await?.map(Bytes::from))+ }++ async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> {+ self.client.set(key, &value).await?;+ lock_known_keys(&self.known_keys)?.insert(key.to_owned());+ Ok(())+ }++ async fn delete(&self, key: &str) -> CacheResult<()> {+ self.client.delete(key).await?;+ lock_known_keys(&self.known_keys)?.remove(key);+ Ok(())+ }++ async fn exists(&self, key: &str) -> CacheResult<bool> {+ self.client.exists(key).await+ }++ async fn batch_get(&self, keys: &[String]) -> CacheResult<Vec<Option<Bytes>>> {+ Ok(self+ .client+ .batch_get(keys)+ .await?+ .into_iter()+ .map(|value| value.map(Bytes::from))+ .collect())+ }++ async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> {+ let native_entries = entries+ .iter()+ .map(|(key, value)| (key.clone(), value.to_vec()))+ .collect();+ self.client.batch_set(native_entries).await?;+ lock_known_keys(&self.known_keys)?.extend(entries.into_iter().map(|(key, _)| key));+ Ok(())+ }++ async fn invalidate(&self, keys: &[String]) -> CacheResult<()> {+ self.client.batch_delete(keys).await?;+ let mut known_keys = lock_known_keys(&self.known_keys)?;+ for key in keys {+ known_keys.remove(key);+ }+ Ok(())+ }++ async fn flush(&self) -> CacheResult<()> {+ let keys = lock_known_keys(&self.known_keys)?+ .iter()+ .cloned()+ .collect::<Vec<_>>();+ self.invalidate(&keys).await+ }++ async fn close(&self) -> CacheResult<()> {+ self.client.close().await+ }+}++#[cfg(test)]+mod tests {+ use super::*;+ use std::panic::{self, AssertUnwindSafe};++ #[test]+ fn lock_known_keys_returns_internal_error_when_poisoned() {+ let known_keys = Mutex::new(HashSet::new());++ let _ = panic::catch_unwind(AssertUnwindSafe(|| {+ let _guard = known_keys.lock().unwrap();+ panic!("poison known_keys");+ }));++ let error = lock_known_keys(&known_keys).unwrap_err();++ assert!(matches!(+ error,+ CacheError::Internal(message)+ if message == "Yuanrong key tracker is poisoned"+ ));+ }+}diff --git a/crates/ragfs-cache-yuanrong/src/store.rs b/crates/ragfs-cache-yuanrong/src/store.rsnew file mode 100644index 000000000..8d62b2ff7--- /dev/null+++ b/crates/ragfs-cache-yuanrong/src/store.rs@@ -0,0 +1,26 @@+use crate::YuanrongStoreError;++/// Synchronous Yuanrong KV operations used by the async provider.+///+/// The production implementation delegates to the Yuanrong C++ SDK. Tests use+/// this boundary to exercise the provider without requiring a worker process.+pub trait YuanrongKvStore: Send + Sync + 'static {+ /// Check that the connected worker is healthy.+ fn health_check(&self) -> Result<(), YuanrongStoreError>;+ /// Read one complete value. A missing key returns `Ok(None)`.+ fn get(&self, key: &str) -> Result<Option<Vec<u8>>, YuanrongStoreError>;+ /// Store one complete value.+ fn set(&self, key: &str, value: &[u8]) -> Result<(), YuanrongStoreError>;+ /// Delete one key. Missing keys are treated as success.+ fn delete(&self, key: &str) -> Result<(), YuanrongStoreError>;+ /// Check whether one key exists.+ fn exists(&self, key: &str) -> Result<bool, YuanrongStoreError>;+ /// Read multiple values while preserving input order.+ fn batch_get(&self, keys: &[String]) -> Result<Vec<Option<Vec<u8>>>, YuanrongStoreError>;+ /// Store multiple values.+ fn batch_set(&self, entries: &[(String, Vec<u8>)]) -> Result<(), YuanrongStoreError>;+ /// Delete multiple keys.+ fn batch_delete(&self, keys: &[String]) -> Result<(), YuanrongStoreError>;+ /// Shut down the SDK client.+ fn shutdown(&self) -> Result<(), YuanrongStoreError>;+}diff --git a/crates/ragfs-cache-yuanrong/tests/cached_filesystem.rs b/crates/ragfs-cache-yuanrong/tests/cached_filesystem.rsnew file mode 100644index 000000000..c0a8539b7--- /dev/null+++ b/crates/ragfs-cache-yuanrong/tests/cached_filesystem.rs@@ -0,0 +1,381 @@+use async_trait::async_trait;+use ragfs::cache::{CacheNamespace, CachePolicy, CacheProvider, CachedFileSystem};+use ragfs::core::{GrepResult, TreeEntry};+use ragfs::plugins::MemFileSystem;+use ragfs::{FileInfo, FileSystem, Result as FsResult, WriteFlag};+use ragfs_cache_yuanrong::{YuanrongConfig, YuanrongKvStore, YuanrongProvider, YuanrongStoreError};+use std::collections::HashMap;+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};+use std::sync::{Arc, Mutex};++#[derive(Default)]+struct SharedKvStore {+ values: Mutex<HashMap<String, Vec<u8>>>,+ unavailable: AtomicBool,+}++impl SharedKvStore {+ fn check(&self) -> std::result::Result<(), YuanrongStoreError> {+ if self.unavailable.load(Ordering::SeqCst) {+ Err(YuanrongStoreError::Unavailable("worker unavailable".into()))+ } else {+ Ok(())+ }+ }+}++impl YuanrongKvStore for SharedKvStore {+ fn health_check(&self) -> std::result::Result<(), YuanrongStoreError> {+ self.check()+ }++ fn get(&self, key: &str) -> std::result::Result<Option<Vec<u8>>, YuanrongStoreError> {+ self.check()?;+ Ok(self.values.lock().unwrap().get(key).cloned())+ }++ fn set(&self, key: &str, value: &[u8]) -> std::result::Result<(), YuanrongStoreError> {+ self.check()?;+ self.values+ .lock()+ .unwrap()+ .insert(key.to_owned(), value.to_vec());+ Ok(())+ }++ fn delete(&self, key: &str) -> std::result::Result<(), YuanrongStoreError> {+ self.check()?;+ self.values.lock().unwrap().remove(key);+ Ok(())+ }++ fn exists(&self, key: &str) -> std::result::Result<bool, YuanrongStoreError> {+ self.check()?;+ Ok(self.values.lock().unwrap().contains_key(key))+ }++ fn batch_get(+ &self,+ keys: &[String],+ ) -> std::result::Result<Vec<Option<Vec<u8>>>, YuanrongStoreError> {+ self.check()?;+ let values = self.values.lock().unwrap();+ Ok(keys.iter().map(|key| values.get(key).cloned()).collect())+ }++ fn batch_set(+ &self,+ entries: &[(String, Vec<u8>)],+ ) -> std::result::Result<(), YuanrongStoreError> {+ self.check()?;+ self.values.lock().unwrap().extend(entries.iter().cloned());+ Ok(())+ }++ fn batch_delete(&self, keys: &[String]) -> std::result::Result<(), YuanrongStoreError> {+ self.check()?;+ let mut values = self.values.lock().unwrap();+ for key in keys {+ values.remove(key);+ }+ Ok(())+ }++ fn shutdown(&self) -> std::result::Result<(), YuanrongStoreError> {+ Ok(())+ }+}++#[derive(Clone)]+struct CountingFileSystem {+ inner: Arc<MemFileSystem>,+ reads: Arc<AtomicU64>,+ read_dirs: Arc<AtomicU64>,+}++impl CountingFileSystem {+ fn new() -> Self {+ Self {+ inner: Arc::new(MemFileSystem::new()),+ reads: Arc::new(AtomicU64::new(0)),+ read_dirs: Arc::new(AtomicU64::new(0)),+ }+ }+}++#[async_trait]+impl FileSystem for CountingFileSystem {+ async fn create(&self, path: &str) -> FsResult<()> {+ self.inner.create(path).await+ }++ async fn mkdir(&self, path: &str, mode: u32) -> FsResult<()> {+ self.inner.mkdir(path, mode).await+ }++ async fn remove(&self, path: &str) -> FsResult<()> {+ self.inner.remove(path).await+ }++ async fn remove_all(&self, path: &str) -> FsResult<()> {+ self.inner.remove_all(path).await+ }++ async fn read(&self, path: &str, offset: u64, size: u64) -> FsResult<Vec<u8>> {+ self.reads.fetch_add(1, Ordering::SeqCst);+ self.inner.read(path, offset, size).await+ }++ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> FsResult<u64> {+ self.inner.write(path, data, offset, flags).await+ }++ async fn read_dir(&self, path: &str) -> FsResult<Vec<FileInfo>> {+ self.read_dirs.fetch_add(1, Ordering::SeqCst);+ self.inner.read_dir(path).await+ }++ async fn stat(&self, path: &str) -> FsResult<FileInfo> {+ self.inner.stat(path).await+ }++ async fn rename(&self, old_path: &str, new_path: &str) -> FsResult<()> {+ self.inner.rename(old_path, new_path).await+ }++ async fn chmod(&self, path: &str, mode: u32) -> FsResult<()> {+ self.inner.chmod(path, mode).await+ }++ async fn truncate(&self, path: &str, size: u64) -> FsResult<()> {+ self.inner.truncate(path, size).await+ }++ async fn grep(+ &self,+ path: &str,+ pattern: &str,+ recursive: bool,+ case_insensitive: bool,+ node_limit: Option<usize>,+ exclude_path: Option<&str>,+ level_limit: Option<usize>,+ ) -> FsResult<GrepResult> {+ self.inner+ .grep(+ path,+ pattern,+ recursive,+ case_insensitive,+ node_limit,+ exclude_path,+ level_limit,+ )+ .await+ }++ async fn tree_directory(+ &self,+ path: &str,+ show_hidden: bool,+ node_limit: Option<usize>,+ level_limit: Option<usize>,+ ) -> FsResult<Vec<TreeEntry>> {+ self.inner+ .tree_directory(path, show_hidden, node_limit, level_limit)+ .await+ }+}++fn config() -> YuanrongConfig {+ YuanrongConfig {+ host: "127.0.0.1".into(),+ port: 9088,+ connect_timeout_ms: 1_000,+ request_timeout_ms: 100,+ sdk_concurrency: 4,+ }+}++async fn cached_fs(+ backend: CountingFileSystem,+ store: Arc<SharedKvStore>,+ namespace: &str,+) -> CachedFileSystem {+ let provider: Arc<dyn CacheProvider> =+ Arc::new(YuanrongProvider::from_store(config(), store).await.unwrap());+ CachedFileSystem::new(+ Box::new(backend),+ provider,+ CacheNamespace::new(namespace),+ CachePolicy::default(),+ )+}++#[tokio::test]+async fn yuanrong_hit_miss_fill_and_write_after_read_are_consistent() {+ let backend = CountingFileSystem::new();+ backend+ .write("/value.md", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let fs = cached_fs(backend, Arc::new(SharedKvStore::default()), "read-write").await;++ assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old");+ assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"old");+ assert_eq!(probe.reads.load(Ordering::SeqCst), 1);++ fs.write("/value.md", b"new", 0, WriteFlag::Truncate)+ .await+ .unwrap();+ assert_eq!(fs.read("/value.md", 0, 0).await.unwrap(), b"new");+ assert_eq!(probe.reads.load(Ordering::SeqCst), 1);+}++#[tokio::test]+async fn delete_rename_remove_all_and_directory_changes_invalidate_yuanrong_keys() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/root", 0o755).await.unwrap();+ backend.mkdir("/root/tree", 0o755).await.unwrap();+ backend+ .write("/root/tree/leaf", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let fs = cached_fs(backend, Arc::new(SharedKvStore::default()), "invalidation").await;++ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1);+ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1);+ assert_eq!(direct.read_dirs.load(Ordering::SeqCst), 1);+ fs.mkdir("/root/created", 0o755).await.unwrap();+ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);+ assert_eq!(direct.read_dirs.load(Ordering::SeqCst), 2);++ assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"old");+ fs.rename("/root/tree/leaf", "/root/tree/moved")+ .await+ .unwrap();+ assert!(fs.read("/root/tree/leaf", 0, 0).await.is_err());+ assert_eq!(fs.read("/root/tree/moved", 0, 0).await.unwrap(), b"old");+ fs.remove("/root/tree/moved").await.unwrap();+ assert!(fs.read("/root/tree/moved", 0, 0).await.is_err());++ direct+ .write("/root/tree/leaf", b"stale", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"stale");+ fs.remove_all("/root/tree").await.unwrap();+ direct.mkdir("/root/tree", 0o755).await.unwrap();+ direct+ .write("/root/tree/leaf", b"fresh", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(fs.read("/root/tree/leaf", 0, 0).await.unwrap(), b"fresh");++ fs.rename("/root/tree", "/root/renamed").await.unwrap();+ assert!(fs.read("/root/tree/leaf", 0, 0).await.is_err());+ assert_eq!(fs.read("/root/renamed/leaf", 0, 0).await.unwrap(), b"fresh");+}++#[tokio::test]+async fn unavailable_yuanrong_falls_back_without_breaking_backend_reads() {+ let backend = CountingFileSystem::new();+ backend+ .write("/available.md", b"backend", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let store = Arc::new(SharedKvStore::default());+ let fs = cached_fs(backend, store.clone(), "fallback").await;+ store.unavailable.store(true, Ordering::SeqCst);++ assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend");+ assert_eq!(fs.read("/available.md", 0, 0).await.unwrap(), b"backend");+ assert_eq!(probe.reads.load(Ordering::SeqCst), 2);+ assert!(fs.metrics().snapshot().errors >= 1);+}++#[cfg(feature = "yuanrong-native")]+#[tokio::test]+async fn native_yuanrong_cached_filesystem_hits_fills_writes_and_invalidates() {+ if std::env::var("OPENVIKING_RUN_YUANRONG_INTEGRATION").as_deref() != Ok("true") {+ return;+ }++ let backend = CountingFileSystem::new();+ backend.mkdir("/native", 0o755).await.unwrap();+ backend+ .write("/native/value", b"backend-old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let native_config = YuanrongConfig {+ host: std::env::var("YUANRONG_WORKER_HOST").unwrap_or_else(|_| "127.0.0.1".into()),+ port: std::env::var("YUANRONG_WORKER_PORT")+ .ok()+ .and_then(|value| value.parse().ok())+ .unwrap_or(31501),+ connect_timeout_ms: 5_000,+ request_timeout_ms: 5_000,+ sdk_concurrency: 4,+ };+ let provider: Arc<dyn CacheProvider> =+ Arc::new(YuanrongProvider::connect(native_config).await.unwrap());+ let namespace = format!("native-fs-{}", std::process::id());+ let fs = CachedFileSystem::new(+ Box::new(backend),+ provider.clone(),+ CacheNamespace::new(namespace),+ CachePolicy::default(),+ );++ assert_eq!(+ fs.read("/native/value", 0, 0).await.unwrap(),+ b"backend-old"+ );+ direct+ .write("/native/value", b"backend-mutated", 0, WriteFlag::Truncate)+ .await+ .unwrap();+ assert_eq!(+ fs.read("/native/value", 0, 0).await.unwrap(),+ b"backend-old",+ "second read must come from Yuanrong rather than the mutated backend"+ );++ fs.write("/native/value", b"write-through", 0, WriteFlag::Truncate)+ .await+ .unwrap();+ assert_eq!(+ fs.read("/native/value", 0, 0).await.unwrap(),+ b"write-through"+ );+ fs.write("/native/empty", b"", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(fs.read("/native/empty", 0, 0).await.unwrap(), b"");+ fs.rename("/native/value", "/native/moved").await.unwrap();+ assert!(fs.read("/native/value", 0, 0).await.is_err());+ assert_eq!(+ fs.read("/native/moved", 0, 0).await.unwrap(),+ b"write-through"+ );+ fs.remove("/native/moved").await.unwrap();+ assert!(fs.read("/native/moved", 0, 0).await.is_err());++ direct+ .write("/native/leaf", b"stale", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(fs.read("/native/leaf", 0, 0).await.unwrap(), b"stale");+ fs.remove_all("/native").await.unwrap();+ direct.mkdir("/native", 0o755).await.unwrap();+ direct+ .write("/native/leaf", b"fresh", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(fs.read("/native/leaf", 0, 0).await.unwrap(), b"fresh");+ provider.close().await.unwrap();+}diff --git a/crates/ragfs-cache-yuanrong/tests/native_smoke.rs b/crates/ragfs-cache-yuanrong/tests/native_smoke.rsnew file mode 100644index 000000000..0531d3cf4--- /dev/null+++ b/crates/ragfs-cache-yuanrong/tests/native_smoke.rs@@ -0,0 +1,76 @@+#![cfg(feature = "yuanrong-native")]++use bytes::Bytes;+use ragfs::cache::CacheProvider;+use ragfs_cache_yuanrong::{YuanrongConfig, YuanrongProvider};++fn integration_enabled() -> bool {+ std::env::var("OPENVIKING_RUN_YUANRONG_INTEGRATION").as_deref() == Ok("true")+}++fn config() -> YuanrongConfig {+ YuanrongConfig {+ host: std::env::var("YUANRONG_WORKER_HOST").unwrap_or_else(|_| "127.0.0.1".into()),+ port: std::env::var("YUANRONG_WORKER_PORT")+ .ok()+ .and_then(|value| value.parse().ok())+ .unwrap_or(31501),+ connect_timeout_ms: 5_000,+ request_timeout_ms: 5_000,+ sdk_concurrency: 4,+ }+}++#[tokio::test]+async fn native_yuanrong_provider_round_trips_kv_and_batch_operations() {+ if !integration_enabled() {+ return;+ }++ let provider = YuanrongProvider::connect(config()).await.unwrap();+ provider.health_check().await.unwrap();+ let prefix = format!("openviking_native_{}", std::process::id());+ let first = format!("{prefix}_first");+ let second = format!("{prefix}_second");++ assert_eq!(provider.get(&first).await.unwrap(), None);+ provider+ .put(&first, Bytes::from_static(b"first-value"))+ .await+ .unwrap();+ assert_eq!(+ provider.get(&first).await.unwrap(),+ Some(Bytes::from_static(b"first-value"))+ );+ let empty = format!("{prefix}_empty");+ provider.put(&empty, Bytes::new()).await.unwrap();+ assert_eq!(provider.get(&empty).await.unwrap(), Some(Bytes::new()));++ provider+ .batch_put(vec![+ (first.clone(), Bytes::from_static(b"updated")),+ (second.clone(), Bytes::from_static(b"second-value")),+ ])+ .await+ .unwrap();+ assert_eq!(+ provider+ .batch_get(&[second.clone(), format!("{prefix}_missing"), first.clone()])+ .await+ .unwrap(),+ vec![+ Some(Bytes::from_static(b"second-value")),+ None,+ Some(Bytes::from_static(b"updated")),+ ]+ );++ provider+ .invalidate(&[first.clone(), second.clone(), empty.clone()])+ .await+ .unwrap();+ assert_eq!(provider.get(&first).await.unwrap(), None);+ assert_eq!(provider.get(&second).await.unwrap(), None);+ assert_eq!(provider.get(&empty).await.unwrap(), None);+ provider.close().await.unwrap();+}diff --git a/crates/ragfs-cache-yuanrong/tests/provider_contract.rs b/crates/ragfs-cache-yuanrong/tests/provider_contract.rsnew file mode 100644index 000000000..fb9b40ab6--- /dev/null+++ b/crates/ragfs-cache-yuanrong/tests/provider_contract.rs@@ -0,0 +1,274 @@+use bytes::Bytes;+use ragfs::cache::{CacheError, CacheProvider};+use ragfs_cache_yuanrong::{YuanrongConfig, YuanrongKvStore, YuanrongProvider, YuanrongStoreError};+use std::collections::HashMap;+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+use std::sync::{Arc, Mutex};+use std::time::Duration;++#[derive(Default)]+struct FakeKvStore {+ values: Mutex<HashMap<String, Vec<u8>>>,+ healthy: AtomicBool,+ available: AtomicBool,+ delay_ms: AtomicUsize,+ active: AtomicUsize,+ max_active: AtomicUsize,+ batch_get_calls: AtomicUsize,+ batch_set_calls: AtomicUsize,+ batch_delete_calls: AtomicUsize,+ shutdown_calls: AtomicUsize,+}++impl FakeKvStore {+ fn available() -> Self {+ Self {+ healthy: AtomicBool::new(true),+ available: AtomicBool::new(true),+ ..Self::default()+ }+ }++ fn enter(&self) -> Result<ActiveGuard<'_>, YuanrongStoreError> {+ if !self.available.load(Ordering::SeqCst) {+ return Err(YuanrongStoreError::Unavailable("worker unavailable".into()));+ }+ let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;+ self.max_active.fetch_max(active, Ordering::SeqCst);+ let delay = self.delay_ms.load(Ordering::SeqCst);+ if delay > 0 {+ std::thread::sleep(Duration::from_millis(delay as u64));+ }+ Ok(ActiveGuard { store: self })+ }+}++struct ActiveGuard<'a> {+ store: &'a FakeKvStore,+}++impl Drop for ActiveGuard<'_> {+ fn drop(&mut self) {+ self.store.active.fetch_sub(1, Ordering::SeqCst);+ }+}++impl YuanrongKvStore for FakeKvStore {+ fn health_check(&self) -> Result<(), YuanrongStoreError> {+ let _guard = self.enter()?;+ if self.healthy.load(Ordering::SeqCst) {+ Ok(())+ } else {+ Err(YuanrongStoreError::Unavailable("unhealthy worker".into()))+ }+ }++ fn get(&self, key: &str) -> Result<Option<Vec<u8>>, YuanrongStoreError> {+ let _guard = self.enter()?;+ Ok(self.values.lock().unwrap().get(key).cloned())+ }++ fn set(&self, key: &str, value: &[u8]) -> Result<(), YuanrongStoreError> {+ let _guard = self.enter()?;+ self.values+ .lock()+ .unwrap()+ .insert(key.to_owned(), value.to_vec());+ Ok(())+ }++ fn delete(&self, key: &str) -> Result<(), YuanrongStoreError> {+ let _guard = self.enter()?;+ self.values.lock().unwrap().remove(key);+ Ok(())+ }++ fn exists(&self, key: &str) -> Result<bool, YuanrongStoreError> {+ let _guard = self.enter()?;+ Ok(self.values.lock().unwrap().contains_key(key))+ }++ fn batch_get(&self, keys: &[String]) -> Result<Vec<Option<Vec<u8>>>, YuanrongStoreError> {+ let _guard = self.enter()?;+ self.batch_get_calls.fetch_add(1, Ordering::SeqCst);+ let values = self.values.lock().unwrap();+ Ok(keys.iter().map(|key| values.get(key).cloned()).collect())+ }++ fn batch_set(&self, entries: &[(String, Vec<u8>)]) -> Result<(), YuanrongStoreError> {+ let _guard = self.enter()?;+ self.batch_set_calls.fetch_add(1, Ordering::SeqCst);+ self.values.lock().unwrap().extend(entries.iter().cloned());+ Ok(())+ }++ fn batch_delete(&self, keys: &[String]) -> Result<(), YuanrongStoreError> {+ let _guard = self.enter()?;+ self.batch_delete_calls.fetch_add(1, Ordering::SeqCst);+ let mut values = self.values.lock().unwrap();+ for key in keys {+ values.remove(key);+ }+ Ok(())+ }++ fn shutdown(&self) -> Result<(), YuanrongStoreError> {+ self.shutdown_calls.fetch_add(1, Ordering::SeqCst);+ Ok(())+ }+}++fn config() -> YuanrongConfig {+ YuanrongConfig {+ host: "127.0.0.1".into(),+ port: 9088,+ connect_timeout_ms: 1_000,+ request_timeout_ms: 100,+ sdk_concurrency: 2,+ }+}++async fn provider(store: Arc<FakeKvStore>) -> YuanrongProvider {+ YuanrongProvider::from_store(config(), store).await.unwrap()+}++#[tokio::test]+async fn initialization_validates_config_and_health() {+ let mut invalid = config();+ invalid.host.clear();+ let error = YuanrongProvider::from_store(invalid, Arc::new(FakeKvStore::available()))+ .await+ .unwrap_err();+ assert!(matches!(error, CacheError::InvalidArgument(_)));++ let unhealthy = Arc::new(FakeKvStore::available());+ unhealthy.healthy.store(false, Ordering::SeqCst);+ let error = YuanrongProvider::from_store(config(), unhealthy)+ .await+ .unwrap_err();+ assert!(matches!(error, CacheError::Unavailable(_)));+}++#[tokio::test]+async fn hit_miss_write_delete_and_exists_map_to_kv_operations() {+ let store = Arc::new(FakeKvStore::available());+ store+ .values+ .lock()+ .unwrap()+ .insert("hit".into(), b"value".to_vec());+ let provider = provider(store).await;++ assert_eq!(+ provider.get("hit").await.unwrap(),+ Some(Bytes::from_static(b"value"))+ );+ assert_eq!(provider.get("missing").await.unwrap(), None);+ provider+ .put("written", Bytes::from_static(b"new"))+ .await+ .unwrap();+ assert!(provider.exists("written").await.unwrap());+ assert_eq!(+ provider.get("written").await.unwrap(),+ Some(Bytes::from_static(b"new"))+ );+ provider.delete("written").await.unwrap();+ provider.delete("written").await.unwrap();+ assert_eq!(provider.get("written").await.unwrap(), None);+}++#[tokio::test]+async fn batch_operations_use_native_kv_batch_calls_and_preserve_order() {+ let store = Arc::new(FakeKvStore::available());+ let provider = provider(store.clone()).await;++ provider+ .batch_put(vec![+ ("one".into(), Bytes::from_static(b"1")),+ ("two".into(), Bytes::from_static(b"2")),+ ])+ .await+ .unwrap();+ assert_eq!(+ provider+ .batch_get(&["two".into(), "missing".into(), "one".into()])+ .await+ .unwrap(),+ vec![+ Some(Bytes::from_static(b"2")),+ None,+ Some(Bytes::from_static(b"1"))+ ]+ );+ provider+ .invalidate(&["one".into(), "two".into()])+ .await+ .unwrap();++ assert_eq!(store.batch_set_calls.load(Ordering::SeqCst), 1);+ assert_eq!(store.batch_get_calls.load(Ordering::SeqCst), 1);+ assert_eq!(store.batch_delete_calls.load(Ordering::SeqCst), 1);+ assert!(provider.capabilities().batch_get);+ assert!(provider.capabilities().batch_put);+}++#[tokio::test]+async fn synchronous_calls_are_bounded_and_timeout_is_not_a_miss() {+ let store = Arc::new(FakeKvStore::available());+ store+ .values+ .lock()+ .unwrap()+ .insert("slow".into(), b"value".to_vec());+ store.delay_ms.store(30, Ordering::SeqCst);+ let mut bounded = config();+ bounded.request_timeout_ms = 500;+ let provider = Arc::new(+ YuanrongProvider::from_store(bounded, store.clone())+ .await+ .unwrap(),+ );++ let tasks = (0..8)+ .map(|_| {+ let provider = provider.clone();+ tokio::spawn(async move { provider.get("slow").await.unwrap() })+ })+ .collect::<Vec<_>>();+ for task in tasks {+ assert_eq!(task.await.unwrap(), Some(Bytes::from_static(b"value")));+ }+ assert!(store.max_active.load(Ordering::SeqCst) <= 2);++ let timeout_store = Arc::new(FakeKvStore::available());+ let mut timeout_config = config();+ timeout_config.request_timeout_ms = 10;+ let provider = YuanrongProvider::from_store(timeout_config, timeout_store.clone())+ .await+ .unwrap();+ timeout_store.delay_ms.store(80, Ordering::SeqCst);+ let error = provider.get("slow").await.unwrap_err();+ assert!(matches!(error, CacheError::Timeout(_)));+}++#[tokio::test]+async fn close_shuts_down_store_and_rejects_new_operations() {+ let store = Arc::new(FakeKvStore::available());+ let provider = provider(store.clone()).await;++ provider.close().await.unwrap();+ assert_eq!(store.shutdown_calls.load(Ordering::SeqCst), 1);+ let error = provider.get("key").await.unwrap_err();+ assert!(matches!(error, CacheError::Unavailable(_)));+}++#[cfg(not(feature = "yuanrong-native"))]+#[tokio::test]+async fn connect_without_native_feature_returns_startup_error() {+ let error = YuanrongProvider::connect(config()).await.unwrap_err();+ assert!(matches!(+ error,+ CacheError::Unavailable(message) if message.contains("yuanrong-native")+ ));+}diff --git a/crates/ragfs-python-native/Cargo.toml b/crates/ragfs-python-native/Cargo.tomlnew file mode 100644index 000000000..853c00f15--- /dev/null+++ b/crates/ragfs-python-native/Cargo.toml@@ -0,0 +1,38 @@+[workspace]+resolver = "2"++[package]+name = "ragfs-python"+version = "0.1.0"+edition = "2021"+rust-version = "1.91.1"+description = "Python bindings for RAGFS with explicit native cache providers"+publish = false++[lib]+name = "ragfs_python"+path = "../ragfs-python/src/lib.rs"+crate-type = ["cdylib"]++[features]+default = ["extension-module", "s3", "cache-redis"]+extension-module = ["pyo3/extension-module"]+s3 = ["ragfs/s3"]+cache-redis = ["ragfs-cache-redis"]+yuanrong-native = [+ "ragfs-cache-yuanrong",+ "ragfs-cache-yuanrong/yuanrong-native",+]+mooncake-native = [+ "ragfs-cache-mooncake",+ "ragfs-cache-mooncake/mooncake-native",+]++[dependencies]+ragfs = { path = "../ragfs", features = ["cache"] }+ragfs-cache-redis = { path = "../ragfs-cache-redis", optional = true }+ragfs-cache-yuanrong = { path = "../ragfs-cache-yuanrong", optional = true }+ragfs-cache-mooncake = { path = "../ragfs-cache-mooncake", optional = true }+pyo3 = { version = "0.27", features = ["abi3", "abi3-py310"] }+tokio = { version = "1", features = ["full"] }+serde_json = "1.0"diff --git a/crates/ragfs-python/Cargo.toml b/crates/ragfs-python/Cargo.tomlindex b82c41cbe..acfe14d23 100644--- a/crates/ragfs-python/Cargo.toml+++ b/crates/ragfs-python/Cargo.toml@@ -11,11 +11,17 @@ name = "ragfs_python"crate-type = ["cdylib"][features]-default = ["s3"]+default = ["extension-module", "s3", "cache-redis"]+extension-module = ["pyo3/extension-module"]s3 = ["ragfs/s3"]+cache-redis = ["ragfs-cache-redis"][dependencies]-ragfs = { path = "../ragfs" }-pyo3 = { version = "0.27", features = ["extension-module", "abi3", "abi3-py310"] }+ragfs = { path = "../ragfs", features = ["cache"] }+ragfs-cache-redis = { path = "../ragfs-cache-redis", optional = true }+pyo3 = { version = "0.27", features = ["abi3", "abi3-py310"] }tokio = { version = "1", features = ["full"] }serde_json = "1.0"++[lints.rust]+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(feature, values("yuanrong-native", "mooncake-native"))'] }diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rsindex 5f580485c..b1d4b01e8 100644--- a/crates/ragfs-python/src/lib.rs+++ b/crates/ragfs-python/src/lib.rs@@ -8,17 +8,494 @@ use pyo3::exceptions::{PyRuntimeError, PyValueError};use pyo3::prelude::*;use pyo3::types::{PyBytes, PyDict, PyList, PyType};use std::collections::HashMap;+use std::fs;use std::future::Future;use std::sync::Arc;use std::time::UNIX_EPOCH;+use ragfs::cache::{+ CacheError, CacheNamespace, CachePolicy, CacheProvider, CacheResult, CacheTraversalMode,+ MemoryCacheProvider,+};use ragfs::core::builder::EncryptionConfig;use ragfs::core::{- build_default_stack, ConfigValue, FileInfo, FileSystem, FilesystemStats, FsContext,- FsContextInner, FsOperation, GrepResult, MountableFS, OperationStats, PluginConfig,- RagfsConfig, TreeEntry, WriteFlag, FS_CTX,+ build_default_stack, register_builtin_plugins, ConfigValue, FileInfo, FileSystem,+ FilesystemStats, FsContext, FsContextInner, FsOperation, GrepResult, MountableFS,+ OperationStats, PluginConfig, RagfsConfig, StatsWrappedFS, TreeEntry, WriteFlag, FS_CTX,};+#[derive(Debug, Clone, PartialEq, Eq)]+enum CacheProviderKind {+ Memory,+ Yuanrong,+ Mooncake,+ Redis,+}++#[derive(Debug, Clone, PartialEq, Eq)]+struct RagfsCacheConfig {+ enabled: bool,+ provider: CacheProviderKind,+ namespace: String,+ max_file_size_bytes: usize,+ traversal_mode: CacheTraversalMode,+ bypass_prefixes: Vec<String>,+ yuanrong: YuanrongCacheConfig,+ mooncake: MooncakeCacheConfig,+ redis: RedisCacheConfig,+}++#[derive(Debug, Clone, PartialEq, Eq)]+struct YuanrongCacheConfig {+ host: String,+ port: u16,+ connect_timeout_ms: u64,+ request_timeout_ms: u64,+ sdk_concurrency: usize,+}++#[derive(Debug, Clone, PartialEq, Eq)]+struct MooncakeCacheConfig {+ local_hostname: String,+ metadata_server: String,+ master_server_addr: String,+ protocol: String,+ device_name: String,+ global_segment_size: u64,+ local_buffer_size: u64,+ replica_num: usize,+ sdk_concurrency: usize,+ operation_timeout_ms: u64,+}++#[derive(Debug, Clone, PartialEq, Eq)]+struct RedisCacheConfig {+ mode: String,+ endpoints: Vec<String>,+ username: String,+ password_env: String,+ pool_size: usize,+ connect_timeout_ms: u64,+ command_timeout_ms: u64,+ key_prefix: String,+ default_ttl_seconds: u64,+ read_from_replica: bool,+}++impl Default for RagfsCacheConfig {+ fn default() -> Self {+ Self {+ enabled: false,+ provider: CacheProviderKind::Memory,+ namespace: "openviking".to_string(),+ max_file_size_bytes: CachePolicy::default().max_file_size(),+ traversal_mode: CacheTraversalMode::Backend,+ bypass_prefixes: Vec::new(),+ yuanrong: YuanrongCacheConfig::default(),+ mooncake: MooncakeCacheConfig::default(),+ redis: RedisCacheConfig::default(),+ }+ }+}++impl Default for YuanrongCacheConfig {+ fn default() -> Self {+ Self {+ host: "127.0.0.1".to_string(),+ port: 31501,+ connect_timeout_ms: 5_000,+ request_timeout_ms: 5_000,+ sdk_concurrency: 4,+ }+ }+}++impl Default for MooncakeCacheConfig {+ fn default() -> Self {+ Self {+ local_hostname: "127.0.0.1".to_string(),+ metadata_server: "http://127.0.0.1:8080/metadata".to_string(),+ master_server_addr: "127.0.0.1:50051".to_string(),+ protocol: "tcp".to_string(),+ device_name: String::new(),+ global_segment_size: 512 << 20,+ local_buffer_size: 128 << 20,+ replica_num: 2,+ sdk_concurrency: 4,+ operation_timeout_ms: 5_000,+ }+ }+}++impl Default for RedisCacheConfig {+ fn default() -> Self {+ Self {+ mode: "standalone".to_string(),+ endpoints: vec!["redis://127.0.0.1:6379".to_string()],+ username: String::new(),+ password_env: String::new(),+ pool_size: 32,+ connect_timeout_ms: 1_000,+ command_timeout_ms: 20,+ key_prefix: "ragfs-cache".to_string(),+ default_ttl_seconds: 3_600,+ read_from_replica: false,+ }+ }+}++struct CacheProviderFactory;++impl CacheProviderFactory {+ async fn create(config: &RagfsCacheConfig) -> CacheResult<Arc<dyn CacheProvider>> {+ match config.provider {+ CacheProviderKind::Memory => Ok(Arc::new(MemoryCacheProvider::new())),+ CacheProviderKind::Yuanrong => create_yuanrong_provider(config).await,+ CacheProviderKind::Mooncake => create_mooncake_provider(config).await,+ CacheProviderKind::Redis => create_redis_provider(config).await,+ }+ }+}++#[cfg(feature = "yuanrong-native")]+async fn create_yuanrong_provider(+ config: &RagfsCacheConfig,+) -> CacheResult<Arc<dyn CacheProvider>> {+ use ragfs_cache_yuanrong::{YuanrongConfig, YuanrongProvider};++ let provider = YuanrongProvider::connect(YuanrongConfig {+ host: config.yuanrong.host.clone(),+ port: config.yuanrong.port,+ connect_timeout_ms: config.yuanrong.connect_timeout_ms,+ request_timeout_ms: config.yuanrong.request_timeout_ms,+ sdk_concurrency: config.yuanrong.sdk_concurrency,+ })+ .await?;+ Ok(Arc::new(provider))+}++#[cfg(not(feature = "yuanrong-native"))]+async fn create_yuanrong_provider(+ _config: &RagfsCacheConfig,+) -> CacheResult<Arc<dyn CacheProvider>> {+ Err(CacheError::Unavailable(+ "Yuanrong support requires the yuanrong-native feature".to_string(),+ ))+}++#[cfg(feature = "mooncake-native")]+async fn create_mooncake_provider(+ config: &RagfsCacheConfig,+) -> CacheResult<Arc<dyn CacheProvider>> {+ use ragfs_cache_mooncake::{MooncakeConfig, MooncakeProvider};++ let provider = MooncakeProvider::connect(MooncakeConfig {+ local_hostname: config.mooncake.local_hostname.clone(),+ metadata_server: config.mooncake.metadata_server.clone(),+ master_server_addr: config.mooncake.master_server_addr.clone(),+ protocol: config.mooncake.protocol.clone(),+ device_name: config.mooncake.device_name.clone(),+ global_segment_size: config.mooncake.global_segment_size,+ local_buffer_size: config.mooncake.local_buffer_size,+ replica_num: config.mooncake.replica_num,+ sdk_concurrency: config.mooncake.sdk_concurrency,+ operation_timeout_ms: config.mooncake.operation_timeout_ms,+ })+ .await?;+ Ok(Arc::new(provider))+}++#[cfg(not(feature = "mooncake-native"))]+async fn create_mooncake_provider(+ _config: &RagfsCacheConfig,+) -> CacheResult<Arc<dyn CacheProvider>> {+ Err(CacheError::Unavailable(+ "Mooncake support requires the mooncake-native feature".to_string(),+ ))+}++#[cfg(feature = "cache-redis")]+async fn create_redis_provider(config: &RagfsCacheConfig) -> CacheResult<Arc<dyn CacheProvider>> {+ use ragfs_cache_redis::{RedisConfig, RedisProvider};++ let provider = RedisProvider::connect(RedisConfig {+ mode: config.redis.mode.clone(),+ endpoints: config.redis.endpoints.clone(),+ username: config.redis.username.clone(),+ password_env: config.redis.password_env.clone(),+ pool_size: config.redis.pool_size,+ connect_timeout_ms: config.redis.connect_timeout_ms,+ command_timeout_ms: config.redis.command_timeout_ms,+ key_prefix: config.redis.key_prefix.clone(),+ default_ttl_seconds: config.redis.default_ttl_seconds,+ read_from_replica: config.redis.read_from_replica,+ })+ .await?;+ Ok(Arc::new(provider))+}++#[cfg(not(feature = "cache-redis"))]+async fn create_redis_provider(_config: &RagfsCacheConfig) -> CacheResult<Arc<dyn CacheProvider>> {+ Err(CacheError::Unavailable(+ "Redis support requires the cache-redis feature".to_string(),+ ))+}++fn cache_config_from_ov_conf(path: &str) -> Result<RagfsCacheConfig, String> {+ let raw = fs::read_to_string(path)+ .map_err(|error| format!("failed to read OpenViking config {path}: {error}"))?;+ let json: serde_json::Value = serde_json::from_str(&raw)+ .map_err(|error| format!("failed to parse OpenViking config {path}: {error}"))?;++ match json+ .get("storage")+ .and_then(|storage| storage.get("agfs"))+ .and_then(|agfs| agfs.get("cache"))+ {+ Some(cache) => cache_config_from_value(cache),+ None => Ok(RagfsCacheConfig::default()),+ }+}++fn cache_config_from_value(cache: &serde_json::Value) -> Result<RagfsCacheConfig, String> {+ if cache.is_null() {+ return Ok(RagfsCacheConfig::default());+ }+ let cache = cache+ .as_object()+ .ok_or_else(|| "storage.agfs.cache must be an object".to_string())?;++ let mut config = RagfsCacheConfig::default();+ config.enabled = bool_field(cache, "enabled", config.enabled)?;+ config.provider = provider_kind(string_field(cache, "provider", "memory")?)?;+ config.namespace = string_field(cache, "namespace", &config.namespace)?;+ if config.namespace.trim().is_empty() {+ return Err("storage.agfs.cache.namespace must not be empty".to_string());+ }+ config.max_file_size_bytes =+ usize_field(cache, "max_file_size_bytes", config.max_file_size_bytes)?;+ config.traversal_mode = traversal_mode(string_field(cache, "traversal_mode", "backend")?)?;+ config.bypass_prefixes = string_array_field(cache, "bypass_prefixes")?;++ if let Some(yuanrong) = cache.get("yuanrong") {+ let yuanrong = yuanrong+ .as_object()+ .ok_or_else(|| "storage.agfs.cache.yuanrong must be an object".to_string())?;+ config.yuanrong.host = string_field(yuanrong, "host", &config.yuanrong.host)?;+ config.yuanrong.port = u16_field(yuanrong, "port", config.yuanrong.port)?;+ config.yuanrong.connect_timeout_ms = u64_field(+ yuanrong,+ "connect_timeout_ms",+ config.yuanrong.connect_timeout_ms,+ )?;+ config.yuanrong.request_timeout_ms = u64_field(+ yuanrong,+ "request_timeout_ms",+ config.yuanrong.request_timeout_ms,+ )?;+ config.yuanrong.sdk_concurrency =+ usize_field(yuanrong, "sdk_concurrency", config.yuanrong.sdk_concurrency)?;+ }++ if let Some(mooncake) = cache.get("mooncake") {+ let mooncake = mooncake+ .as_object()+ .ok_or_else(|| "storage.agfs.cache.mooncake must be an object".to_string())?;+ config.mooncake.local_hostname =+ string_field(mooncake, "local_hostname", &config.mooncake.local_hostname)?;+ config.mooncake.metadata_server = string_field(+ mooncake,+ "metadata_server",+ &config.mooncake.metadata_server,+ )?;+ config.mooncake.master_server_addr = string_field(+ mooncake,+ "master_server_addr",+ &config.mooncake.master_server_addr,+ )?;+ config.mooncake.protocol = string_field(mooncake, "protocol", &config.mooncake.protocol)?;+ config.mooncake.device_name =+ string_field(mooncake, "device_name", &config.mooncake.device_name)?;+ config.mooncake.global_segment_size = u64_field(+ mooncake,+ "global_segment_size",+ config.mooncake.global_segment_size,+ )?;+ config.mooncake.local_buffer_size = u64_field(+ mooncake,+ "local_buffer_size",+ config.mooncake.local_buffer_size,+ )?;+ config.mooncake.replica_num =+ usize_field(mooncake, "replica_num", config.mooncake.replica_num)?;+ config.mooncake.sdk_concurrency =+ usize_field(mooncake, "sdk_concurrency", config.mooncake.sdk_concurrency)?;+ config.mooncake.operation_timeout_ms = u64_field(+ mooncake,+ "operation_timeout_ms",+ config.mooncake.operation_timeout_ms,+ )?;+ }++ if let Some(redis) = cache.get("redis") {+ let redis = redis+ .as_object()+ .ok_or_else(|| "storage.agfs.cache.redis must be an object".to_string())?;+ config.redis.mode = string_field(redis, "mode", &config.redis.mode)?;+ config.redis.endpoints =+ string_array_field_or_default(redis, "endpoints", &config.redis.endpoints)?;+ config.redis.username = string_field(redis, "username", &config.redis.username)?;+ config.redis.password_env =+ string_field(redis, "password_env", &config.redis.password_env)?;+ config.redis.pool_size = usize_field(redis, "pool_size", config.redis.pool_size)?;+ config.redis.connect_timeout_ms =+ u64_field(redis, "connect_timeout_ms", config.redis.connect_timeout_ms)?;+ config.redis.command_timeout_ms =+ u64_field(redis, "command_timeout_ms", config.redis.command_timeout_ms)?;+ config.redis.key_prefix = string_field(redis, "key_prefix", &config.redis.key_prefix)?;+ config.redis.default_ttl_seconds = u64_field_allow_zero(+ redis,+ "default_ttl_seconds",+ config.redis.default_ttl_seconds,+ )?;+ config.redis.read_from_replica =+ bool_field(redis, "read_from_replica", config.redis.read_from_replica)?;+ }++ Ok(config)+}++fn provider_kind(value: String) -> Result<CacheProviderKind, String> {+ match value.as_str() {+ "memory" => Ok(CacheProviderKind::Memory),+ "yuanrong" => Ok(CacheProviderKind::Yuanrong),+ "mooncake" => Ok(CacheProviderKind::Mooncake),+ "redis" => Ok(CacheProviderKind::Redis),+ other => Err(format!(+ "unsupported storage.agfs.cache.provider: {other}; expected memory, yuanrong, mooncake, or redis"+ )),+ }+}++fn traversal_mode(value: String) -> Result<CacheTraversalMode, String> {+ match value.as_str() {+ "backend" => Ok(CacheTraversalMode::Backend),+ "cached_traversal" => Ok(CacheTraversalMode::CachedTraversal),+ other => Err(format!(+ "unsupported storage.agfs.cache.traversal_mode: {other}; expected backend or cached_traversal"+ )),+ }+}++fn bool_field(+ object: &serde_json::Map<String, serde_json::Value>,+ key: &str,+ default: bool,+) -> Result<bool, String> {+ match object.get(key) {+ Some(value) => value+ .as_bool()+ .ok_or_else(|| format!("{key} must be a boolean")),+ None => Ok(default),+ }+}++fn string_field(+ object: &serde_json::Map<String, serde_json::Value>,+ key: &str,+ default: &str,+) -> Result<String, String> {+ match object.get(key) {+ Some(value) => value+ .as_str()+ .map(ToOwned::to_owned)+ .ok_or_else(|| format!("{key} must be a string")),+ None => Ok(default.to_string()),+ }+}++fn u64_field(+ object: &serde_json::Map<String, serde_json::Value>,+ key: &str,+ default: u64,+) -> Result<u64, String> {+ match object.get(key) {+ Some(value) => value+ .as_u64()+ .filter(|value| *value > 0)+ .ok_or_else(|| format!("{key} must be a positive integer")),+ None => Ok(default),+ }+}++fn u64_field_allow_zero(+ object: &serde_json::Map<String, serde_json::Value>,+ key: &str,+ default: u64,+) -> Result<u64, String> {+ match object.get(key) {+ Some(value) => value+ .as_u64()+ .ok_or_else(|| format!("{key} must be a non-negative integer")),+ None => Ok(default),+ }+}++fn u16_field(+ object: &serde_json::Map<String, serde_json::Value>,+ key: &str,+ default: u16,+) -> Result<u16, String> {+ let value = u64_field(object, key, default as u64)?;+ u16::try_from(value).map_err(|_| format!("{key} must fit in u16"))+}++fn usize_field(+ object: &serde_json::Map<String, serde_json::Value>,+ key: &str,+ default: usize,+) -> Result<usize, String> {+ let value = u64_field(object, key, default as u64)?;+ usize::try_from(value).map_err(|_| format!("{key} must fit in usize"))+}++fn string_array_field(+ object: &serde_json::Map<String, serde_json::Value>,+ key: &str,+) -> Result<Vec<String>, String> {+ match object.get(key) {+ Some(value) => value+ .as_array()+ .ok_or_else(|| format!("{key} must be an array of strings"))?+ .iter()+ .map(|item| {+ item.as_str()+ .map(ToOwned::to_owned)+ .ok_or_else(|| format!("{key} must be an array of strings"))+ })+ .collect(),+ None => Ok(Vec::new()),+ }+}++fn string_array_field_or_default(+ object: &serde_json::Map<String, serde_json::Value>,+ key: &str,+ default: &[String],+) -> Result<Vec<String>, String> {+ match object.get(key) {+ Some(_) => string_array_field(object, key),+ None => Ok(default.to_vec()),+ }+}++fn cache_policy_from_config(config: &RagfsCacheConfig) -> CachePolicy {+ config.bypass_prefixes.iter().fold(+ CachePolicy::new(config.max_file_size_bytes).with_traversal_mode(config.traversal_mode),+ |policy, prefix| policy.with_bypass_prefix(prefix),+ )+}+fn py_detach_blocking<T, F>(py: Python<'_>, f: F) -> TwhereT: Send,@@ -367,7 +844,7 @@ impl RAGFSBindingClient {/// `RAGFSBindingClient(config_path=...)` do not fail. `config` is an optional sectioned dict/// (mirrors ov.conf). The `encryption` section, when present, carries `root_key` (32 bytes) +/// `provider_type` (int) and causes the stack to include an `EncryptionWrappedFS` layer.- /// Absence of the section yields a plaintext stack.+ /// Runtime `cache` configuration takes precedence over `config_path`.#[new]#[pyo3(signature = (config_path=None, config=None))]fn new(@@ -375,12 +852,12 @@ impl RAGFSBindingClient {config_path: Option<&str>,config: Option<HashMap<String, Py<PyAny>>>,) -> PyResult<Self> {- let _ = config_path;let rt = tokio::runtime::Runtime::new().map_err(|e| PyRuntimeError::new_err(format!("Failed to create runtime: {}", e)))?;// Phase A (holding GIL): parse the sectioned config into an owned RagfsConfig.let mut ragfs_cfg = RagfsConfig::default();+ let mut runtime_cache_config = None;if let Some(cfg) = config {if let Some(enc_obj) = cfg.get("encryption") {let enc: HashMap<String, Py<PyAny>> = enc_obj.extract(py)?;@@ -401,10 +878,54 @@ impl RAGFSBindingClient {provider_type,});}+ if let Some(cache_obj) = cfg.get("cache") {+ let cache_value = py_to_json_value(cache_obj.bind(py))?;+ runtime_cache_config =+ Some(cache_config_from_value(&cache_value).map_err(|error| {+ PyRuntimeError::new_err(format!("Invalid cache config: {error}"))+ })?);+ }}- // Phase B: build the stack (no encryption layer when the section is absent).- let stack = rt.block_on(build_default_stack(ragfs_cfg));+ let cache_config = match runtime_cache_config {+ Some(config) => config,+ None => match config_path {+ Some(path) => cache_config_from_ov_conf(path).map_err(|error| {+ PyRuntimeError::new_err(format!("Invalid cache config: {error}"))+ })?,+ None => RagfsCacheConfig::default(),+ },+ };++ // Phase B: build the stack. Cache, when enabled, replaces the mountable+ // layer with a cache-aware mountable while preserving the same plugin+ // registration and per-backend encryption setup.+ let stack = if cache_config.enabled {+ let provider = rt+ .block_on(CacheProviderFactory::create(&cache_config))+ .map_err(|error| {+ PyRuntimeError::new_err(format!("Failed to initialize cache provider: {error}"))+ })?;+ let mountable = Arc::new(MountableFS::with_cache(+ provider,+ CacheNamespace::new(&cache_config.namespace),+ cache_policy_from_config(&cache_config),+ ));+ rt.block_on(async {+ register_builtin_plugins(&mountable).await;+ if let Some(enc) = &ragfs_cfg.encryption {+ mountable+ .set_encryption_config(Some(enc.root_key), Some(enc.provider_type))+ .await;+ }+ });+ let top: Arc<dyn FileSystem> = Arc::new(StatsWrappedFS::with_arc(+ mountable.clone() as Arc<dyn FileSystem>+ ));+ ragfs::core::RagfsStack { mountable, top }+ } else {+ rt.block_on(build_default_stack(ragfs_cfg))+ };Ok(Self {mountable: stack.mountable,top: stack.top,@@ -1113,6 +1634,7 @@ fn ragfs_python(m: &Bound<'_, PyModule>) -> PyResult<()> {mod tests {use super::*;use pyo3::types::PyDict;+ use std::fs;#[test]fn detach_blocking_helper_runs_without_python_objects() {@@ -1124,14 +1646,245 @@ mod tests {#[test]fn constructor_accepts_legacy_config_path_keyword() {+ let path = std::env::temp_dir().join(format!(+ "openviking-legacy-config-path-{}.json",+ std::process::id()+ ));+ fs::write(&path, r#"{"storage": {"agfs": {"backend": "local"}}}"#).unwrap();+Python::attach(|py| {let ty = py.get_type::<RAGFSBindingClient>();let kwargs = PyDict::new(py);kwargs- .set_item("config_path", "/tmp/legacy-ov.conf")+ .set_item("config_path", path.to_str().unwrap()).unwrap();let instance = ty.call((), Some(&kwargs));assert!(instance.is_ok());});++ fs::remove_file(path).unwrap();+ }++ #[tokio::test]+ async fn cache_provider_factory_creates_memory_provider_from_ov_conf() {+ let path = std::env::temp_dir().join(format!(+ "openviking-cache-config-{}.json",+ std::process::id()+ ));+ fs::write(+ &path,+ r#"{+ "storage": {+ "agfs": {+ "cache": {+ "enabled": true,+ "provider": "memory",+ "namespace": "ov-test",+ "traversal_mode": "cached_traversal"+ }+ }+ }+ }"#,+ )+ .unwrap();++ let cache_config = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap();+ let provider = CacheProviderFactory::create(&cache_config).await.unwrap();++ assert!(cache_config.enabled);+ assert_eq!(cache_config.provider, CacheProviderKind::Memory);+ assert_eq!(cache_config.namespace, "ov-test");+ assert_eq!(+ cache_config.traversal_mode,+ CacheTraversalMode::CachedTraversal+ );+ assert_eq!(provider.name(), "memory");++ fs::remove_file(path).unwrap();+ }++ #[test]+ fn missing_cache_config_defaults_to_disabled_memory_config() {+ let path = std::env::temp_dir().join(format!(+ "openviking-no-cache-config-{}.json",+ std::process::id()+ ));+ fs::write(&path, r#"{"storage": {"agfs": {"backend": "local"}}}"#).unwrap();++ let cache_config = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap();++ assert!(!cache_config.enabled);+ assert_eq!(cache_config.provider, CacheProviderKind::Memory);+ assert_eq!(cache_config.namespace, "openviking");+ assert_eq!(cache_config.traversal_mode, CacheTraversalMode::Backend);++ fs::remove_file(path).unwrap();+ }++ #[test]+ fn cache_config_rejects_invalid_traversal_mode() {+ let path = std::env::temp_dir().join(format!(+ "openviking-invalid-cache-traversal-{}.json",+ std::process::id()+ ));+ fs::write(+ &path,+ r#"{"storage": {"agfs": {"cache": {"traversal_mode": "bogus"}}}}"#,+ )+ .unwrap();++ let error = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap_err();++ assert!(error.contains("traversal_mode"));++ fs::remove_file(path).unwrap();+ }++ #[test]+ fn constructor_runtime_cache_overrides_config_path() {+ Python::initialize();+ let path = std::env::temp_dir().join(format!(+ "openviking-runtime-cache-override-{}.json",+ std::process::id()+ ));+ fs::write(+ &path,+ r#"{"storage": {"agfs": {"cache": {"enabled": true, "provider": "invalid"}}}}"#,+ )+ .unwrap();++ Python::attach(|py| {+ let ty = py.get_type::<RAGFSBindingClient>();+ let cache = PyDict::new(py);+ cache.set_item("enabled", true).unwrap();+ cache.set_item("provider", "memory").unwrap();+ cache.set_item("namespace", "runtime-cache").unwrap();+ let config = PyDict::new(py);+ config.set_item("cache", cache).unwrap();+ let kwargs = PyDict::new(py);+ kwargs+ .set_item("config_path", path.to_str().unwrap())+ .unwrap();+ kwargs.set_item("config", config).unwrap();++ let instance = ty.call((), Some(&kwargs));+ assert!(instance.is_ok());+ });++ fs::remove_file(path).unwrap();+ }++ #[test]+ fn redis_cache_config_is_parsed_from_ov_conf() {+ let path = std::env::temp_dir().join(format!(+ "openviking-redis-cache-config-{}.json",+ std::process::id()+ ));+ fs::write(+ &path,+ r#"{+ "storage": {+ "agfs": {+ "cache": {+ "enabled": true,+ "provider": "redis",+ "namespace": "ov-test",+ "redis": {+ "mode": "standalone",+ "endpoints": ["redis://127.0.0.1:6379"],+ "pool_size": 8,+ "connect_timeout_ms": 1000,+ "command_timeout_ms": 20,+ "key_prefix": "ragfs-cache",+ "default_ttl_seconds": 3600,+ "read_from_replica": false+ }+ }+ }+ }+ }"#,+ )+ .unwrap();++ let cache_config = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap();++ assert!(cache_config.enabled);+ assert_eq!(cache_config.provider, CacheProviderKind::Redis);+ assert_eq!(cache_config.redis.mode, "standalone");+ assert_eq!(cache_config.redis.endpoints, vec!["redis://127.0.0.1:6379"]);+ assert_eq!(cache_config.redis.pool_size, 8);+ assert_eq!(cache_config.redis.connect_timeout_ms, 1000);+ assert_eq!(cache_config.redis.command_timeout_ms, 20);+ assert_eq!(cache_config.redis.key_prefix, "ragfs-cache");+ assert_eq!(cache_config.redis.default_ttl_seconds, 3600);+ assert!(!cache_config.redis.read_from_replica);++ fs::remove_file(path).unwrap();+ }++ #[cfg(not(feature = "cache-redis"))]+ #[tokio::test]+ async fn redis_provider_requires_cache_redis_feature() {+ let config = RagfsCacheConfig {+ enabled: true,+ provider: CacheProviderKind::Redis,+ ..RagfsCacheConfig::default()+ };++ let error = match CacheProviderFactory::create(&config).await {+ Ok(provider) => panic!("unexpected provider: {}", provider.name()),+ Err(error) => error,+ };++ assert!(matches!(+ error,+ CacheError::Unavailable(message)+ if message == "Redis support requires the cache-redis feature"+ ));+ }++ #[cfg(feature = "cache-redis")]+ #[tokio::test]+ async fn cache_provider_factory_creates_redis_provider_from_ov_conf() {+ let Ok(endpoint) = std::env::var("REDIS_URL") else {+ return;+ };+ let path = std::env::temp_dir().join(format!(+ "openviking-cache-redis-factory-{}.json",+ std::process::id()+ ));+ fs::write(+ &path,+ format!(+ r#"{{+ "storage": {{+ "agfs": {{+ "cache": {{+ "enabled": true,+ "provider": "redis",+ "namespace": "ov-test",+ "redis": {{+ "mode": "standalone",+ "endpoints": ["{endpoint}"],+ "connect_timeout_ms": 30000,+ "command_timeout_ms": 1000,+ "key_prefix": "ragfs-python-cache-test",+ "default_ttl_seconds": 60+ }}+ }}+ }}+ }}+ }}"#+ ),+ )+ .unwrap();++ let cache_config = cache_config_from_ov_conf(path.to_str().unwrap()).unwrap();+ let provider = CacheProviderFactory::create(&cache_config).await.unwrap();++ assert_eq!(provider.name(), "redis");+ provider.close().await.unwrap();++ fs::remove_file(path).unwrap();}}diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.tomlindex 86cfcb1fd..64c1db03a 100644--- a/crates/ragfs/Cargo.toml+++ b/crates/ragfs/Cargo.toml@@ -18,6 +18,11 @@ path = "src/lib.rs"name = "ragfs-shell"path = "src/shell/main.rs"+[[test]]+name = "cache_wrapper"+path = "tests/cache_wrapper.rs"+required-features = ["cache"]+[dependencies]# Async runtimetokio = { version = "1.38", features = ["full"] }@@ -88,5 +93,6 @@ criterion = "0.5"[features]default = []+cache = []s3 = ["aws-sdk-s3", "aws-config", "aws-types"]-full = ["s3"]+full = ["s3", "cache"]diff --git a/crates/ragfs/src/cache/envelope.rs b/crates/ragfs/src/cache/envelope.rsnew file mode 100644index 000000000..3510fdbc8--- /dev/null+++ b/crates/ragfs/src/cache/envelope.rs@@ -0,0 +1,105 @@+//! Stable values stored behind the provider contract.++use super::{CacheError, CacheResult};+use crate::core::FileInfo;+use bytes::Bytes;+use serde::{Deserialize, Serialize};++const CACHE_ENVELOPE_VERSION: u8 = 1;++#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]+pub(crate) enum CacheObjectKind {+ File,+ Directory,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub(crate) struct GenerationSnapshot {+ pub key: String,+ pub value: u64,+}++#[derive(Debug, Clone, Serialize, Deserialize)]+enum CachePayload {+ File(Vec<u8>),+ Directory(Vec<FileInfo>),+}++#[derive(Debug, Clone, Serialize, Deserialize)]+pub(crate) struct CacheEnvelope {+ version: u8,+ kind: CacheObjectKind,+ path: String,+ generations: Vec<GenerationSnapshot>,+ payload: CachePayload,+}++impl CacheEnvelope {+ pub fn file(path: String, data: Vec<u8>, generations: Vec<GenerationSnapshot>) -> Self {+ Self {+ version: CACHE_ENVELOPE_VERSION,+ kind: CacheObjectKind::File,+ path,+ generations,+ payload: CachePayload::File(data),+ }+ }++ pub fn directory(+ path: String,+ entries: Vec<FileInfo>,+ generations: Vec<GenerationSnapshot>,+ ) -> Self {+ Self {+ version: CACHE_ENVELOPE_VERSION,+ kind: CacheObjectKind::Directory,+ path,+ generations,+ payload: CachePayload::Directory(entries),+ }+ }++ pub fn encode(&self) -> CacheResult<Bytes> {+ serde_json::to_vec(self)+ .map(Bytes::from)+ .map_err(|error| CacheError::InvalidData(error.to_string()))+ }++ pub fn decode(value: &[u8]) -> CacheResult<Self> {+ let envelope: Self = serde_json::from_slice(value)+ .map_err(|error| CacheError::InvalidData(error.to_string()))?;+ if envelope.version != CACHE_ENVELOPE_VERSION {+ return Err(CacheError::InvalidData(format!(+ "unsupported envelope version {}",+ envelope.version+ )));+ }+ Ok(envelope)+ }++ pub fn matches(&self, kind: CacheObjectKind, path: &str) -> bool {+ self.kind == kind && self.path == path+ }++ pub fn generations(&self) -> &[GenerationSnapshot] {+ &self.generations+ }++ pub fn into_file(self) -> CacheResult<Vec<u8>> {+ match self.payload {+ CachePayload::File(data) => Ok(data),+ CachePayload::Directory(_) => {+ Err(CacheError::InvalidData("expected file payload".to_string()))+ }+ }+ }++ pub fn into_directory(self) -> CacheResult<Vec<FileInfo>> {+ match self.payload {+ CachePayload::Directory(entries) => Ok(entries),+ CachePayload::File(_) => Err(CacheError::InvalidData(+ "expected directory payload".to_string(),+ )),+ }+ }+}diff --git a/crates/ragfs/src/cache/memory.rs b/crates/ragfs/src/cache/memory.rsnew file mode 100644index 000000000..9f66730ec--- /dev/null+++ b/crates/ragfs/src/cache/memory.rs@@ -0,0 +1,132 @@+//! In-process cache provider used by tests and smoke validation.++use super::{CacheError, CacheProvider, CacheResult, ProviderCapabilities};+use async_trait::async_trait;+use bytes::Bytes;+use std::collections::HashMap;+use std::sync::atomic::{AtomicBool, Ordering};+use tokio::sync::RwLock;++/// A thread-safe in-memory implementation of [`CacheProvider`].+pub struct MemoryCacheProvider {+ values: RwLock<HashMap<String, Bytes>>,+ closed: AtomicBool,+}++/// Test-oriented name for the in-process cache provider.+pub type MemoryMockProvider = MemoryCacheProvider;++impl MemoryCacheProvider {+ /// Create an empty provider.+ pub fn new() -> Self {+ Self {+ values: RwLock::new(HashMap::new()),+ closed: AtomicBool::new(false),+ }+ }++ fn ensure_open(&self) -> CacheResult<()> {+ if self.closed.load(Ordering::Acquire) {+ Err(CacheError::Unavailable(+ "memory provider is closed".to_string(),+ ))+ } else {+ Ok(())+ }+ }++ /// Return the current number of stored objects.+ pub async fn len(&self) -> usize {+ self.values.read().await.len()+ }++ /// Return whether the provider currently stores no objects.+ pub async fn is_empty(&self) -> bool {+ self.len().await == 0+ }++ /// Return a snapshot of stored keys for diagnostics and smoke tests.+ pub async fn keys(&self) -> Vec<String> {+ self.values.read().await.keys().cloned().collect()+ }+}++impl Default for MemoryCacheProvider {+ fn default() -> Self {+ Self::new()+ }+}++#[async_trait]+impl CacheProvider for MemoryCacheProvider {+ fn name(&self) -> &'static str {+ "memory"+ }++ fn capabilities(&self) -> ProviderCapabilities {+ ProviderCapabilities {+ batch_get: true,+ batch_put: true,+ native_ttl: false,+ }+ }++ async fn get(&self, key: &str) -> CacheResult<Option<Bytes>> {+ self.ensure_open()?;+ Ok(self.values.read().await.get(key).cloned())+ }++ async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> {+ self.ensure_open()?;+ let mut values = self.values.write().await;+ self.ensure_open()?;+ values.insert(key.to_string(), value);+ Ok(())+ }++ async fn delete(&self, key: &str) -> CacheResult<()> {+ self.ensure_open()?;+ let mut values = self.values.write().await;+ self.ensure_open()?;+ values.remove(key);+ Ok(())+ }++ async fn batch_get(&self, keys: &[String]) -> CacheResult<Vec<Option<Bytes>>> {+ self.ensure_open()?;+ let values = self.values.read().await;+ Ok(keys.iter().map(|key| values.get(key).cloned()).collect())+ }++ async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> {+ self.ensure_open()?;+ let mut values = self.values.write().await;+ self.ensure_open()?;+ values.extend(entries);+ Ok(())+ }++ async fn invalidate(&self, keys: &[String]) -> CacheResult<()> {+ self.ensure_open()?;+ let mut values = self.values.write().await;+ self.ensure_open()?;+ for key in keys {+ values.remove(key);+ }+ Ok(())+ }++ async fn flush(&self) -> CacheResult<()> {+ self.ensure_open()?;+ let mut values = self.values.write().await;+ self.ensure_open()?;+ values.clear();+ Ok(())+ }++ async fn close(&self) -> CacheResult<()> {+ self.closed.store(true, Ordering::Release);+ self.values.write().await.clear();+ Ok(())+ }+}diff --git a/crates/ragfs/src/cache/metrics.rs b/crates/ragfs/src/cache/metrics.rsnew file mode 100644index 000000000..3779fc766--- /dev/null+++ b/crates/ragfs/src/cache/metrics.rs@@ -0,0 +1,164 @@+//! Lightweight metrics owned by the cache wrapper.++use std::sync::atomic::{AtomicU64, Ordering};+use std::time::Duration;++/// Snapshot of cache wrapper counters.+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct CacheMetricsSnapshot {+ /// Successful file cache hits.+ pub file_hits: u64,+ /// File cache misses.+ pub file_misses: u64,+ /// Successful directory cache hits.+ pub read_dir_hits: u64,+ /// Directory cache misses.+ pub read_dir_misses: u64,+ /// Backend loads caused by cache misses.+ pub backend_fallbacks: u64,+ /// Cache writes attempted.+ pub puts: u64,+ /// Cache deletes attempted.+ pub deletes: u64,+ /// Logical invalidations completed.+ pub invalidations: u64,+ /// Provider or cache-data errors hidden by bypass mode.+ pub errors: u64,+ /// Reads deliberately bypassed by policy or range semantics.+ pub policy_bypasses: u64,+ /// Backend bytes returned on cacheable reads.+ pub backend_bytes: u64,+ /// Bytes returned from cache objects.+ pub cache_bytes: u64,+ /// Total provider get latency in nanoseconds.+ pub get_latency_ns: u64,+ /// Total provider put latency in nanoseconds.+ pub put_latency_ns: u64,+ /// Total provider delete latency in nanoseconds.+ pub delete_latency_ns: u64,+ /// Inflight miss leaders.+ pub inflight_leaders: u64,+ /// Inflight miss followers.+ pub inflight_followers: u64,+ /// Backend reads avoided by inflight followers.+ pub inflight_backend_saved: u64,+}++/// Thread-safe cache metric counters.+#[derive(Default)]+pub struct CacheMetrics {+ file_hits: AtomicU64,+ file_misses: AtomicU64,+ read_dir_hits: AtomicU64,+ read_dir_misses: AtomicU64,+ backend_fallbacks: AtomicU64,+ puts: AtomicU64,+ deletes: AtomicU64,+ invalidations: AtomicU64,+ errors: AtomicU64,+ policy_bypasses: AtomicU64,+ backend_bytes: AtomicU64,+ cache_bytes: AtomicU64,+ get_latency_ns: AtomicU64,+ put_latency_ns: AtomicU64,+ delete_latency_ns: AtomicU64,+ inflight_leaders: AtomicU64,+ inflight_followers: AtomicU64,+ inflight_backend_saved: AtomicU64,+}++impl CacheMetrics {+ /// Return an immutable snapshot.+ pub fn snapshot(&self) -> CacheMetricsSnapshot {+ CacheMetricsSnapshot {+ file_hits: self.file_hits.load(Ordering::Relaxed),+ file_misses: self.file_misses.load(Ordering::Relaxed),+ read_dir_hits: self.read_dir_hits.load(Ordering::Relaxed),+ read_dir_misses: self.read_dir_misses.load(Ordering::Relaxed),+ backend_fallbacks: self.backend_fallbacks.load(Ordering::Relaxed),+ puts: self.puts.load(Ordering::Relaxed),+ deletes: self.deletes.load(Ordering::Relaxed),+ invalidations: self.invalidations.load(Ordering::Relaxed),+ errors: self.errors.load(Ordering::Relaxed),+ policy_bypasses: self.policy_bypasses.load(Ordering::Relaxed),+ backend_bytes: self.backend_bytes.load(Ordering::Relaxed),+ cache_bytes: self.cache_bytes.load(Ordering::Relaxed),+ get_latency_ns: self.get_latency_ns.load(Ordering::Relaxed),+ put_latency_ns: self.put_latency_ns.load(Ordering::Relaxed),+ delete_latency_ns: self.delete_latency_ns.load(Ordering::Relaxed),+ inflight_leaders: self.inflight_leaders.load(Ordering::Relaxed),+ inflight_followers: self.inflight_followers.load(Ordering::Relaxed),+ inflight_backend_saved: self.inflight_backend_saved.load(Ordering::Relaxed),+ }+ }++ pub(crate) fn file_hit(&self, bytes: usize) {+ self.file_hits.fetch_add(1, Ordering::Relaxed);+ self.cache_bytes.fetch_add(bytes as u64, Ordering::Relaxed);+ }++ pub(crate) fn file_miss(&self) {+ self.file_misses.fetch_add(1, Ordering::Relaxed);+ }++ pub(crate) fn read_dir_hit(&self) {+ self.read_dir_hits.fetch_add(1, Ordering::Relaxed);+ }++ pub(crate) fn read_dir_miss(&self) {+ self.read_dir_misses.fetch_add(1, Ordering::Relaxed);+ }++ pub(crate) fn backend_fallback(&self, bytes: usize) {+ self.backend_fallbacks.fetch_add(1, Ordering::Relaxed);+ self.backend_bytes+ .fetch_add(bytes as u64, Ordering::Relaxed);+ }++ pub(crate) fn put(&self, elapsed: Duration) {+ self.puts.fetch_add(1, Ordering::Relaxed);+ self.put_latency_ns.fetch_add(+ elapsed.as_nanos().min(u64::MAX as u128) as u64,+ Ordering::Relaxed,+ );+ }++ pub(crate) fn get(&self, elapsed: Duration) {+ self.get_latency_ns.fetch_add(+ elapsed.as_nanos().min(u64::MAX as u128) as u64,+ Ordering::Relaxed,+ );+ }++ pub(crate) fn delete(&self, elapsed: Duration) {+ self.deletes.fetch_add(1, Ordering::Relaxed);+ self.delete_latency_ns.fetch_add(+ elapsed.as_nanos().min(u64::MAX as u128) as u64,+ Ordering::Relaxed,+ );+ }++ pub(crate) fn invalidation(&self) {+ self.invalidations.fetch_add(1, Ordering::Relaxed);+ }++ pub(crate) fn error(&self) {+ self.errors.fetch_add(1, Ordering::Relaxed);+ }++ pub(crate) fn policy_bypass(&self) {+ self.policy_bypasses.fetch_add(1, Ordering::Relaxed);+ }++ pub(crate) fn inflight_leader(&self) {+ self.inflight_leaders.fetch_add(1, Ordering::Relaxed);+ }++ pub(crate) fn inflight_follower(&self) {+ self.inflight_followers.fetch_add(1, Ordering::Relaxed);+ }++ pub(crate) fn inflight_backend_saved(&self) {+ self.inflight_backend_saved.fetch_add(1, Ordering::Relaxed);+ }+}diff --git a/crates/ragfs/src/cache/mod.rs b/crates/ragfs/src/cache/mod.rsnew file mode 100644index 000000000..2c58ea396--- /dev/null+++ b/crates/ragfs/src/cache/mod.rs@@ -0,0 +1,18 @@+//! Optional provider-independent caching for RAGFS filesystems.+//!+//! This module is intentionally not installed by [`crate::MountableFS`] by+//! default. Callers opt in by wrapping an existing [`crate::FileSystem`] with+//! [`CachedFileSystem`].++mod envelope;+mod memory;+mod metrics;+mod policy;+mod provider;+mod wrapper;++pub use memory::{MemoryCacheProvider, MemoryMockProvider};+pub use metrics::{CacheMetrics, CacheMetricsSnapshot};+pub use policy::{CacheDecision, CachePolicy, CacheTraversalMode, CacheTreeMode};+pub use provider::{CacheError, CacheProvider, CacheResult, ProviderCapabilities};+pub use wrapper::{CacheNamespace, CachedFileSystem};diff --git a/crates/ragfs/src/cache/policy.rs b/crates/ragfs/src/cache/policy.rsnew file mode 100644index 000000000..bbe21a5e7--- /dev/null+++ b/crates/ragfs/src/cache/policy.rs@@ -0,0 +1,181 @@+//! Provider-independent cache eligibility rules.++const DEFAULT_MAX_CACHED_DIR_ENTRIES: usize = 4096;++/// Cache admission decision for a filesystem object.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum CacheDecision {+ /// Do not access the cache for this object.+ Bypass,+ /// Cache the object with normal priority.+ Cache,+ /// Prefer this high-value object when a provider supports admission priority.+ Prefer,+}++impl CacheDecision {+ fn should_cache(self) -> bool {+ self != Self::Bypass+ }+}++/// Strategy used by [`super::CachedFileSystem`] for recursive traversal APIs.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum CacheTraversalMode {+ /// Delegate traversal APIs to the wrapped backend.+ Backend,+ /// Traverse through `CachedFileSystem` so directory and file caches can be reused.+ CachedTraversal,+}++/// Backwards-compatible alias for the earlier tree-only traversal name.+pub type CacheTreeMode = CacheTraversalMode;++/// Rules used by [`super::CachedFileSystem`] before accessing the cache.+#[derive(Debug, Clone)]+pub struct CachePolicy {+ max_file_size: usize,+ max_cached_dir_entries: usize,+ traversal_mode: CacheTraversalMode,+ bypass_prefixes: Vec<String>,+}++impl CachePolicy {+ /// Create a policy with the supplied maximum full-file cache size.+ pub fn new(max_file_size: usize) -> Self {+ Self {+ max_file_size,+ max_cached_dir_entries: DEFAULT_MAX_CACHED_DIR_ENTRIES,+ traversal_mode: CacheTraversalMode::Backend,+ bypass_prefixes: Vec::new(),+ }+ }++ /// Add a path prefix that always bypasses the cache.+ pub fn with_bypass_prefix(mut self, prefix: impl Into<String>) -> Self {+ self.bypass_prefixes.push(normalize_path(&prefix.into()));+ self+ }++ /// Set the traversal strategy for recursive APIs such as tree and grep.+ pub fn with_traversal_mode(mut self, mode: CacheTraversalMode) -> Self {+ self.traversal_mode = mode;+ self+ }++ /// Set the tree traversal strategy.+ pub fn with_tree_mode(self, mode: CacheTreeMode) -> Self {+ self.with_traversal_mode(mode)+ }++ /// Return the maximum cacheable file size.+ pub fn max_file_size(&self) -> usize {+ self.max_file_size+ }++ /// Return the maximum cacheable raw directory entry count.+ pub fn max_cached_dir_entries(&self) -> usize {+ self.max_cached_dir_entries+ }++ /// Return the traversal strategy for recursive APIs such as tree and grep.+ pub fn traversal_mode(&self) -> CacheTraversalMode {+ self.traversal_mode+ }++ /// Return the tree traversal strategy.+ pub fn tree_mode(&self) -> CacheTreeMode {+ self.traversal_mode()+ }++ /// Return the admission decision for a full-file object.+ pub fn file_decision(&self, path: &str, size: usize) -> CacheDecision {+ if size > self.max_file_size || !self.cache_path(path) {+ return CacheDecision::Bypass;+ }++ match normalize_path(path).rsplit('/').next().unwrap_or("") {+ ".abstract.md" | ".overview.md" => CacheDecision::Prefer,+ _ => CacheDecision::Cache,+ }+ }++ /// Return the admission decision for raw directory entries.+ pub fn directory_decision(&self, path: &str) -> CacheDecision {+ if self.cache_path(path) {+ CacheDecision::Prefer+ } else {+ CacheDecision::Bypass+ }+ }++ /// Return whether a full-file object is eligible for caching.+ pub fn cache_file(&self, path: &str, size: usize) -> bool {+ self.file_decision(path, size).should_cache()+ }++ /// Return whether raw directory entries are eligible for caching.+ pub fn cache_directory(&self, path: &str) -> bool {+ self.directory_decision(path).should_cache()+ }++ /// Return whether raw directory entries are small enough to cache.+ pub fn cache_directory_entries(&self, path: &str, entry_count: usize) -> bool {+ self.cache_directory(path) && entry_count <= self.max_cached_dir_entries+ }++ fn cache_path(&self, path: &str) -> bool {+ let normalized = normalize_path(path);+ if self+ .bypass_prefixes+ .iter()+ .any(|prefix| is_same_or_descendant(&normalized, prefix))+ {+ return false;+ }++ let name = normalized.rsplit('/').next().unwrap_or("");+ if name == ".path.ovlock"+ || name.ends_with(".lock")+ || name.ends_with(".lck")+ || matches!(+ name,+ "enqueue"+ | "dequeue"+ | "peek"+ | "ack"+ | "heartbeat"+ | "lease"+ | "cursor"+ | "offset"+ | "pid"+ )+ {+ return false;+ }++ true+ }+}++impl Default for CachePolicy {+ fn default() -> Self {+ Self::new(1024 * 1024)+ }+}++fn normalize_path(path: &str) -> String {+ if path.is_empty() || path == "/" {+ "/".to_string()+ } else {+ format!("/{}", path.trim_matches('/'))+ }+}++fn is_same_or_descendant(path: &str, prefix: &str) -> bool {+ path == prefix+ || prefix == "/"+ || path+ .strip_prefix(prefix)+ .is_some_and(|suffix| suffix.starts_with('/'))+}diff --git a/crates/ragfs/src/cache/provider.rs b/crates/ragfs/src/cache/provider.rsnew file mode 100644index 000000000..183a535b8--- /dev/null+++ b/crates/ragfs/src/cache/provider.rs@@ -0,0 +1,103 @@+//! Provider-independent cache primitives.++use async_trait::async_trait;+use bytes::Bytes;++/// Result type returned by cache providers.+pub type CacheResult<T> = std::result::Result<T, CacheError>;++/// Errors shared by all cache providers.+#[derive(Debug, thiserror::Error)]+pub enum CacheError {+ /// The provider is unavailable or has been closed.+ #[error("cache provider unavailable: {0}")]+ Unavailable(String),++ /// A provider operation timed out.+ #[error("cache provider operation timed out: {0}")]+ Timeout(String),++ /// A cache value could not be decoded or validated.+ #[error("invalid cache data: {0}")]+ InvalidData(String),++ /// A provider rejected an argument.+ #[error("invalid cache argument: {0}")]+ InvalidArgument(String),++ /// An unspecified provider failure occurred.+ #[error("cache provider internal error: {0}")]+ Internal(String),+}++/// Optional features exposed by a provider.+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub struct ProviderCapabilities {+ /// Whether batch reads have a native implementation.+ pub batch_get: bool,+ /// Whether batch writes have a native implementation.+ pub batch_put: bool,+ /// Whether the provider supports native expiration.+ pub native_ttl: bool,+}++/// Minimal contract implemented by local and remote cache providers.+#[async_trait]+pub trait CacheProvider: Send + Sync {+ /// Return a stable provider name for diagnostics.+ fn name(&self) -> &'static str;++ /// Return optional provider capabilities.+ fn capabilities(&self) -> ProviderCapabilities {+ ProviderCapabilities::default()+ }++ /// Read one cache object.+ async fn get(&self, key: &str) -> CacheResult<Option<Bytes>>;++ /// Write one cache object.+ async fn put(&self, key: &str, value: Bytes) -> CacheResult<()>;++ /// Delete one cache object. Missing keys are treated as success.+ async fn delete(&self, key: &str) -> CacheResult<()>;++ /// Check whether one cache object exists.+ async fn exists(&self, key: &str) -> CacheResult<bool> {+ Ok(self.get(key).await?.is_some())+ }++ /// Read multiple objects while preserving input order.+ async fn batch_get(&self, keys: &[String]) -> CacheResult<Vec<Option<Bytes>>> {+ let mut values = Vec::with_capacity(keys.len());+ for key in keys {+ values.push(self.get(key).await?);+ }+ Ok(values)+ }++ /// Write multiple objects.+ async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> {+ for (key, value) in entries {+ self.put(&key, value).await?;+ }+ Ok(())+ }++ /// Invalidate a known set of cache keys.+ async fn invalidate(&self, keys: &[String]) -> CacheResult<()> {+ for key in keys {+ self.delete(key).await?;+ }+ Ok(())+ }++ /// Remove all objects owned by this provider instance.+ async fn flush(&self) -> CacheResult<()> {+ Ok(())+ }++ /// Release provider resources.+ async fn close(&self) -> CacheResult<()> {+ Ok(())+ }+}diff --git a/crates/ragfs/src/cache/wrapper.rs b/crates/ragfs/src/cache/wrapper.rsnew file mode 100644index 000000000..b0d738b18--- /dev/null+++ b/crates/ragfs/src/cache/wrapper.rs@@ -0,0 +1,1291 @@+//! A transparent [`FileSystem`](crate::FileSystem) cache wrapper.++use super::envelope::{CacheEnvelope, CacheObjectKind, GenerationSnapshot};+use super::{+ CacheError, CacheMetrics, CachePolicy, CacheProvider, CacheResult, CacheTraversalMode,+};+use crate::core::filesystem::{+ compile_grep_regex, is_excluded_path, normalize_prefix_path, relative_depth,+ relative_match_file,+};+use crate::core::{+ FileInfo, FileSystem, GrepMatch, GrepResult, MultiWriteWrappedFS, Result, TreeEntry, WriteFlag,+};+use async_trait::async_trait;+use bytes::Bytes;+use futures::stream::{self, StreamExt};+use regex::Regex;+use std::collections::HashMap;+use std::sync::Arc;+use std::time::Instant;+use tokio::sync::{Mutex, RwLock};+use uuid::Uuid;++const GREP_CACHE_FILE_CONCURRENCY: usize = 8;++/// Namespace prepended to every provider key owned by one wrapper.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct CacheNamespace {+ value: String,+}++impl CacheNamespace {+ /// Create a namespace from a mount-, account-, or application-specific value.+ pub fn new(value: impl Into<String>) -> Self {+ Self {+ value: value.into(),+ }+ }++ /// Return the namespace text.+ pub fn as_str(&self) -> &str {+ &self.value+ }+}++/// A provider-independent read-through filesystem cache.+///+/// Construction is explicit. RAGFS does not install this wrapper in the+/// default mount path, so existing filesystem behavior remains unchanged.+pub struct CachedFileSystem {+ backend: Box<dyn FileSystem>,+ provider: Arc<dyn CacheProvider>,+ namespace: CacheNamespace,+ policy: CachePolicy,+ metrics: Arc<CacheMetrics>,+ operation_lock: RwLock<()>,+ inflight: Mutex<HashMap<String, Arc<Mutex<()>>>>,+ bypass_scopes: RwLock<Vec<String>>,+ generation_epoch: u64,+ generations: RwLock<HashMap<String, u64>>,+}++impl CachedFileSystem {+ /// Wrap an existing backend with a cache provider.+ pub fn new(+ backend: Box<dyn FileSystem>,+ provider: Arc<dyn CacheProvider>,+ namespace: CacheNamespace,+ policy: CachePolicy,+ ) -> Self {+ Self {+ backend,+ provider,+ namespace,+ policy,+ metrics: Arc::new(CacheMetrics::default()),+ operation_lock: RwLock::new(()),+ inflight: Mutex::new(HashMap::new()),+ bypass_scopes: RwLock::new(Vec::new()),+ generation_epoch: nonzero_random_u64(),+ generations: RwLock::new(HashMap::new()),+ }+ }++ /// Return this wrapper's cache metrics.+ pub fn metrics(&self) -> Arc<CacheMetrics> {+ Arc::clone(&self.metrics)+ }++ /// Return the provider used by this wrapper.+ pub fn provider(&self) -> Arc<dyn CacheProvider> {+ Arc::clone(&self.provider)+ }++ /// Return the wrapped filesystem for mount-stack capability discovery.+ pub(crate) fn inner_fs(&self) -> &dyn FileSystem {+ self.backend.as_ref()+ }++ fn wraps_multiwrite(&self) -> bool {+ let any = self.backend.as_ref() as &dyn std::any::Any;+ any.downcast_ref::<MultiWriteWrappedFS>().is_some()+ }++ /// Invalidate cache objects affected by a write that bypassed this wrapper.+ pub(crate) async fn invalidate_external_write(&self, path: &str) {+ self.invalidate_path_objects(path).await;+ self.invalidate_parent_directory(path).await;+ }++ async fn tree_directory_via_cache(+ &self,+ path: &str,+ show_hidden: bool,+ node_limit: Option<usize>,+ level_limit: Option<usize>,+ ) -> Result<Vec<TreeEntry>> {+ enum TreeTask {+ VisitDir(String),+ Emit(TreeEntry),+ }++ let base_path = normalize_prefix_path(path);+ let mut result = Vec::new();+ let mut stack = vec![TreeTask::VisitDir(base_path.clone())];++ while let Some(task) = stack.pop() {+ if node_limit.is_some_and(|limit| result.len() >= limit) {+ break;+ }++ match task {+ TreeTask::Emit(entry) => result.push(entry),+ TreeTask::VisitDir(current_path) => {+ if let Some(limit) = level_limit {+ let current_rel = relative_match_file(&base_path, ¤t_path);+ if relative_depth(¤t_rel) >= limit {+ continue;+ }+ }++ let entries = self.read_dir(¤t_path).await?;+ for entry in entries.into_iter().rev() {+ let is_hidden_file = !entry.is_dir && entry.name.starts_with('.');+ if is_hidden_file && !show_hidden {+ continue;+ }++ let entry_path = if current_path == "/" {+ format!("/{}", entry.name)+ } else {+ format!("{}/{}", current_path, entry.name)+ };+ let rel_path = relative_match_file(&base_path, &entry_path);+ let is_dir = entry.is_dir;+ let tree_entry = TreeEntry {+ path: entry_path.clone(),+ rel_path,+ info: entry,+ extra: HashMap::new(),+ };++ if is_dir {+ stack.push(TreeTask::VisitDir(entry_path));+ }+ stack.push(TreeTask::Emit(tree_entry));+ }+ }+ }+ }++ Ok(result)+ }++ async fn grep_via_cache(+ &self,+ path: &str,+ pattern: &str,+ recursive: bool,+ case_insensitive: bool,+ node_limit: Option<usize>,+ exclude_path: Option<&str>,+ level_limit: Option<usize>,+ ) -> Result<GrepResult> {+ enum GrepTask {+ Visit { path: String, is_dir: Option<bool> },+ }++ let re = compile_grep_regex(pattern, case_insensitive)?;+ let base_path = normalize_prefix_path(path);+ let normalized_exclude = exclude_path.map(normalize_prefix_path);+ let mut result = GrepResult::new();+ let generation_cache = Mutex::new(HashMap::new());+ let mut file_batch = Vec::new();+ let mut stack = vec![GrepTask::Visit {+ path: base_path.clone(),+ is_dir: None,+ }];++ while let Some(GrepTask::Visit {+ path: current_path,+ is_dir,+ }) = stack.pop()+ {+ if node_limit.is_some_and(|limit| result.count >= limit) {+ break;+ }++ if let Some(exclude) = normalized_exclude.as_deref() {+ if is_excluded_path(¤t_path, exclude) {+ continue;+ }+ }++ let is_dir = match is_dir {+ Some(is_dir) => is_dir,+ None => self.stat(¤t_path).await?.is_dir,+ };+ if is_dir {+ self.flush_grep_file_batch(+ &mut file_batch,+ &base_path,+ &re,+ node_limit,+ &mut result,+ &generation_cache,+ )+ .await?;+ if node_limit.is_some_and(|limit| result.count >= limit) {+ break;+ }++ if !recursive && current_path != base_path {+ continue;+ }++ if let Some(limit) = level_limit {+ let rel = relative_match_file(&base_path, ¤t_path);+ if relative_depth(&rel) >= limit {+ continue;+ }+ }++ let entries = self+ .read_dir_with_generation_cache(¤t_path, &generation_cache)+ .await?;+ for entry in entries.into_iter().rev() {+ let entry_path = if current_path == "/" {+ format!("/{}", entry.name)+ } else {+ format!("{}/{}", current_path, entry.name)+ };+ stack.push(GrepTask::Visit {+ path: entry_path,+ is_dir: Some(entry.is_dir),+ });+ }+ } else {+ if let Some(limit) = level_limit {+ let rel = relative_match_file(&base_path, ¤t_path);+ if relative_depth(&rel) > limit {+ continue;+ }+ }++ file_batch.push(current_path);+ if file_batch.len() >= GREP_CACHE_FILE_CONCURRENCY {+ self.flush_grep_file_batch(+ &mut file_batch,+ &base_path,+ &re,+ node_limit,+ &mut result,+ &generation_cache,+ )+ .await?;+ }+ }+ }++ self.flush_grep_file_batch(+ &mut file_batch,+ &base_path,+ &re,+ node_limit,+ &mut result,+ &generation_cache,+ )+ .await?;++ Ok(result)+ }++ async fn flush_grep_file_batch(+ &self,+ file_batch: &mut Vec<String>,+ base_path: &str,+ re: &Regex,+ node_limit: Option<usize>,+ result: &mut GrepResult,+ generation_cache: &Mutex<HashMap<String, u64>>,+ ) -> Result<()> {+ if file_batch.is_empty() || node_limit.is_some_and(|limit| result.count >= limit) {+ file_batch.clear();+ return Ok(());+ }++ let remaining_limit = node_limit+ .map(|limit| limit.saturating_sub(result.count))+ .unwrap_or(usize::MAX);+ let files = std::mem::take(file_batch);+ let mut indexed = stream::iter(files.into_iter().enumerate())+ .map(|(index, path)| async move {+ let matches = self+ .grep_cached_file(&path, base_path, re, remaining_limit, generation_cache)+ .await;+ (index, matches)+ })+ .buffer_unordered(GREP_CACHE_FILE_CONCURRENCY)+ .collect::<Vec<_>>()+ .await;++ indexed.sort_by_key(|(index, _)| *index);+ for (_, matches) in indexed {+ for item in matches? {+ if node_limit.is_some_and(|limit| result.count >= limit) {+ return Ok(());+ }+ result.matches.push(item);+ result.count += 1;+ }+ }++ Ok(())+ }++ async fn grep_cached_file(+ &self,+ path: &str,+ base_path: &str,+ re: &Regex,+ remaining_limit: usize,+ generation_cache: &Mutex<HashMap<String, u64>>,+ ) -> Result<Vec<GrepMatch>> {+ let content = self+ .read_with_generation_cache(path, 0, 0, generation_cache)+ .await?;+ let content_str = String::from_utf8_lossy(&content);+ let rel_file = relative_match_file(base_path, path);+ let mut matches = Vec::new();++ for (line_num, line) in content_str.lines().enumerate() {+ if matches.len() >= remaining_limit {+ break;+ }+ if re.is_match(line) {+ matches.push(GrepMatch {+ file: rel_file.clone(),+ line: (line_num + 1) as u64,+ content: line.to_string(),+ });+ }+ }++ Ok(matches)+ }++ async fn cache_get(&self, key: &str) -> CacheResult<Option<Bytes>> {+ let started = Instant::now();+ let result = self.provider.get(key).await;+ self.metrics.get(started.elapsed());+ result+ }++ async fn cache_batch_get(&self, keys: &[String]) -> CacheResult<Vec<Option<Bytes>>> {+ if keys.is_empty() {+ return Ok(Vec::new());+ }++ if keys.len() == 1 || !self.provider.capabilities().batch_get {+ let mut values = Vec::with_capacity(keys.len());+ for key in keys {+ values.push(self.cache_get(key).await?);+ }+ return Ok(values);+ }++ let started = Instant::now();+ let result = self.provider.batch_get(keys).await;+ self.metrics.get(started.elapsed());+ let values = result?;+ if values.len() != keys.len() {+ return Err(CacheError::InvalidData(format!(+ "batch_get returned {} values for {} keys",+ values.len(),+ keys.len()+ )));+ }+ Ok(values)+ }++ async fn cache_put(&self, key: &str, value: Bytes, affected_path: &str) -> bool {+ let started = Instant::now();+ let result = self.provider.put(key, value).await;+ self.metrics.put(started.elapsed());+ match result {+ Ok(()) => true,+ Err(_) => {+ self.metrics.error();+ self.mark_bypass(affected_path).await;+ false+ }+ }+ }++ async fn cache_delete(&self, key: &str, affected_path: &str) {+ let started = Instant::now();+ let result = self.provider.delete(key).await;+ self.metrics.delete(started.elapsed());+ match result {+ Ok(()) => self.metrics.invalidation(),+ Err(_) => {+ self.metrics.error();+ self.mark_bypass(affected_path).await;+ }+ }+ }++ async fn mark_bypass(&self, path: &str) {+ let normalized = normalize_path(path);+ let mut scopes = self.bypass_scopes.write().await;+ if scopes+ .iter()+ .any(|scope| is_same_or_descendant(&normalized, scope))+ {+ return;+ }+ scopes.retain(|scope| !is_same_or_descendant(scope, &normalized));+ scopes.push(normalized);+ }++ async fn is_runtime_bypassed(&self, path: &str) -> bool {+ let normalized = normalize_path(path);+ self.bypass_scopes+ .read()+ .await+ .iter()+ .any(|scope| is_same_or_descendant(&normalized, scope))+ }++ async fn current_generation(&self, key: &str) -> CacheResult<u64> {+ let values = self.current_generations(&[key.to_string()]).await?;+ Ok(values[0])+ }++ async fn current_generations(&self, keys: &[String]) -> CacheResult<Vec<u64>> {+ let provider_values = self.cache_batch_get(keys).await?;+ let local_generations = self.generations.read().await;+ let mut values = Vec::with_capacity(keys.len());+ let mut missing = Vec::new();++ for (key, provider_value) in keys.iter().zip(provider_values) {+ let value = match provider_value {+ None => {+ let value = local_generations+ .get(key)+ .copied()+ .unwrap_or(self.generation_epoch);+ missing.push((key.clone(), value));+ value+ }+ Some(value) if value.len() == std::mem::size_of::<u64>() => {+ let mut bytes = [0_u8; 8];+ bytes.copy_from_slice(&value);+ u64::from_be_bytes(bytes)+ }+ Some(_) => {+ return Err(CacheError::InvalidData(format!(+ "generation key {key} has invalid length"+ )))+ }+ };+ values.push(value);+ }+ drop(local_generations);++ {+ let mut local_generations = self.generations.write().await;+ for (key, value) in keys.iter().zip(values.iter()) {+ local_generations.insert(key.clone(), *value);+ }+ }++ self.put_missing_generations(missing).await;++ Ok(values)+ }++ async fn current_generations_memoized(+ &self,+ keys: &[String],+ generation_cache: &Mutex<HashMap<String, u64>>,+ ) -> CacheResult<Vec<u64>> {+ let mut values = vec![None; keys.len()];+ let mut missing_keys = Vec::new();+ let mut missing_positions = Vec::new();++ {+ let generation_cache = generation_cache.lock().await;+ for (index, key) in keys.iter().enumerate() {+ if let Some(value) = generation_cache.get(key) {+ values[index] = Some(*value);+ } else {+ missing_positions.push(index);+ missing_keys.push(key.clone());+ }+ }+ }++ let missing_values = self.current_generations(&missing_keys).await?;+ if !missing_values.is_empty() {+ let mut generation_cache = generation_cache.lock().await;+ for (index, value) in missing_positions.into_iter().zip(missing_values) {+ generation_cache.insert(keys[index].clone(), value);+ values[index] = Some(value);+ }+ }++ values+ .into_iter()+ .map(|value| {+ value.ok_or_else(|| CacheError::Internal("missing generation value".to_string()))+ })+ .collect()+ }++ async fn put_missing_generations(&self, missing: Vec<(String, u64)>) {+ if missing.is_empty() {+ return;+ }++ if missing.len() > 1 && self.provider.capabilities().batch_put {+ let entries = missing+ .into_iter()+ .map(|(key, value)| (key, Bytes::copy_from_slice(&value.to_be_bytes())))+ .collect();+ let started = Instant::now();+ if self.provider.batch_put(entries).await.is_err() {+ self.metrics.error();+ }+ self.metrics.put(started.elapsed());+ return;+ }++ for (key, value) in missing {+ let started = Instant::now();+ if self+ .provider+ .put(&key, Bytes::copy_from_slice(&value.to_be_bytes()))+ .await+ .is_err()+ {+ self.metrics.error();+ }+ self.metrics.put(started.elapsed());+ }+ }++ async fn generation_snapshots(&self, path: &str) -> CacheResult<Vec<GenerationSnapshot>> {+ let keys = ancestor_scopes(path)+ .into_iter()+ .map(|scope| self.generation_key(&scope))+ .collect::<Vec<_>>();+ let values = self.current_generations(&keys).await?;+ Ok(keys+ .into_iter()+ .zip(values)+ .map(|(key, value)| GenerationSnapshot { key, value })+ .collect())+ }++ async fn generations_match_with_cache(+ &self,+ envelope: &CacheEnvelope,+ generation_cache: Option<&Mutex<HashMap<String, u64>>>,+ ) -> CacheResult<bool> {+ let keys = envelope+ .generations()+ .iter()+ .map(|snapshot| snapshot.key.clone())+ .collect::<Vec<_>>();+ let values = match generation_cache {+ Some(generation_cache) => {+ self.current_generations_memoized(&keys, generation_cache)+ .await?+ }+ None => self.current_generations(&keys).await?,+ };++ for (snapshot, value) in envelope.generations().iter().zip(values) {+ if value != snapshot.value {+ return Ok(false);+ }+ }+ Ok(true)+ }++ async fn bump_generation(&self, path: &str) {+ let key = self.generation_key(path);+ let current = match self.current_generation(&key).await {+ Ok(value) => value,+ Err(_) => {+ self.metrics.error();+ self.mark_bypass(path).await;+ return;+ }+ };+ let next = current.wrapping_add(1);+ self.generations.write().await.insert(key.clone(), next);+ if self+ .cache_put(&key, Bytes::copy_from_slice(&next.to_be_bytes()), path)+ .await+ {+ self.metrics.invalidation();+ }+ }++ async fn probe_file(&self, key: &str, path: &str, record_hit: bool) -> Option<Vec<u8>> {+ self.probe_file_with_generation_cache(key, path, record_hit, None)+ .await+ }++ async fn probe_file_with_generation_cache(+ &self,+ key: &str,+ path: &str,+ record_hit: bool,+ generation_cache: Option<&Mutex<HashMap<String, u64>>>,+ ) -> Option<Vec<u8>> {+ let value = match self.cache_get(key).await {+ Ok(value) => value?,+ Err(_) => {+ self.metrics.error();+ self.mark_bypass(path).await;+ return None;+ }+ };++ let envelope = match CacheEnvelope::decode(&value) {+ Ok(envelope) if envelope.matches(CacheObjectKind::File, path) => envelope,+ _ => {+ self.metrics.error();+ self.cache_delete(key, path).await;+ return None;+ }+ };++ match self+ .generations_match_with_cache(&envelope, generation_cache)+ .await+ {+ Ok(true) => match envelope.into_file() {+ Ok(data) => {+ if record_hit {+ self.metrics.file_hit(data.len());+ }+ Some(data)+ }+ Err(_) => {+ self.metrics.error();+ self.cache_delete(key, path).await;+ None+ }+ },+ Ok(false) => {+ self.cache_delete(key, path).await;+ None+ }+ Err(_) => {+ self.metrics.error();+ self.mark_bypass(path).await;+ None+ }+ }+ }++ async fn probe_directory(+ &self,+ key: &str,+ path: &str,+ record_hit: bool,+ ) -> Option<Vec<FileInfo>> {+ self.probe_directory_with_generation_cache(key, path, record_hit, None)+ .await+ }++ async fn probe_directory_with_generation_cache(+ &self,+ key: &str,+ path: &str,+ record_hit: bool,+ generation_cache: Option<&Mutex<HashMap<String, u64>>>,+ ) -> Option<Vec<FileInfo>> {+ let value = match self.cache_get(key).await {+ Ok(value) => value?,+ Err(_) => {+ self.metrics.error();+ self.mark_bypass(path).await;+ return None;+ }+ };++ let envelope = match CacheEnvelope::decode(&value) {+ Ok(envelope) if envelope.matches(CacheObjectKind::Directory, path) => envelope,+ _ => {+ self.metrics.error();+ self.cache_delete(key, path).await;+ return None;+ }+ };++ match self+ .generations_match_with_cache(&envelope, generation_cache)+ .await+ {+ Ok(true) => match envelope.into_directory() {+ Ok(entries) => {+ if !self.policy.cache_directory_entries(path, entries.len()) {+ self.cache_delete(key, path).await;+ return None;+ }+ if record_hit {+ self.metrics.read_dir_hit();+ }+ Some(entries)+ }+ Err(_) => {+ self.metrics.error();+ self.cache_delete(key, path).await;+ None+ }+ },+ Ok(false) => {+ self.cache_delete(key, path).await;+ None+ }+ Err(_) => {+ self.metrics.error();+ self.mark_bypass(path).await;+ None+ }+ }+ }++ async fn acquire_inflight(&self, key: &str) -> (Arc<Mutex<()>>, bool) {+ let mut inflight = self.inflight.lock().await;+ if let Some(lock) = inflight.get(key) {+ (Arc::clone(lock), false)+ } else {+ let lock = Arc::new(Mutex::new(()));+ inflight.insert(key.to_string(), Arc::clone(&lock));+ (lock, true)+ }+ }++ async fn release_inflight(&self, key: &str, lock: &Arc<Mutex<()>>) {+ let mut inflight = self.inflight.lock().await;+ if inflight+ .get(key)+ .is_some_and(|current| Arc::ptr_eq(current, lock) && Arc::strong_count(lock) == 2)+ {+ inflight.remove(key);+ }+ }++ async fn read_with_generation_cache(+ &self,+ path: &str,+ offset: u64,+ size: u64,+ generation_cache: &Mutex<HashMap<String, u64>>,+ ) -> Result<Vec<u8>> {+ if offset != 0+ || size != 0+ || !self.policy.cache_file(path, 0)+ || self.is_runtime_bypassed(path).await+ {+ self.metrics.policy_bypass();+ return self.backend.read(path, offset, size).await;+ }++ let _operation_guard = self.operation_lock.read().await;+ if self.is_runtime_bypassed(path).await {+ self.metrics.policy_bypass();+ return self.backend.read(path, offset, size).await;+ }++ let normalized = normalize_path(path);+ let key = self.file_key(&normalized);+ if let Some(data) = self+ .probe_file_with_generation_cache(&key, &normalized, true, Some(generation_cache))+ .await+ {+ return Ok(data);+ }+ self.metrics.file_miss();++ let (inflight, leader) = self.acquire_inflight(&key).await;+ if leader {+ self.metrics.inflight_leader();+ } else {+ self.metrics.inflight_follower();+ }+ let inflight_guard = inflight.lock().await;++ if !leader {+ if let Some(data) = self+ .probe_file_with_generation_cache(&key, &normalized, false, Some(generation_cache))+ .await+ {+ self.metrics.inflight_backend_saved();+ drop(inflight_guard);+ self.release_inflight(&key, &inflight).await;+ return Ok(data);+ }+ }++ let data = self.backend.read(path, 0, 0).await;+ if let Ok(value) = &data {+ self.metrics.backend_fallback(value.len());+ self.fill_file(&key, &normalized, value).await;+ }+ drop(inflight_guard);+ self.release_inflight(&key, &inflight).await;+ data+ }++ async fn read_dir_with_generation_cache(+ &self,+ path: &str,+ generation_cache: &Mutex<HashMap<String, u64>>,+ ) -> Result<Vec<FileInfo>> {+ if !self.policy.cache_directory(path) || self.is_runtime_bypassed(path).await {+ self.metrics.policy_bypass();+ return self.backend.read_dir(path).await;+ }++ let _operation_guard = self.operation_lock.read().await;+ if self.is_runtime_bypassed(path).await {+ self.metrics.policy_bypass();+ return self.backend.read_dir(path).await;+ }++ let normalized = normalize_path(path);+ let key = self.directory_key(&normalized);+ if let Some(entries) = self+ .probe_directory_with_generation_cache(&key, &normalized, true, Some(generation_cache))+ .await+ {+ return Ok(entries);+ }+ self.metrics.read_dir_miss();++ let (inflight, leader) = self.acquire_inflight(&key).await;+ if leader {+ self.metrics.inflight_leader();+ } else {+ self.metrics.inflight_follower();+ }+ let inflight_guard = inflight.lock().await;++ if !leader {+ if let Some(entries) = self+ .probe_directory_with_generation_cache(+ &key,+ &normalized,+ false,+ Some(generation_cache),+ )+ .await+ {+ self.metrics.inflight_backend_saved();+ drop(inflight_guard);+ self.release_inflight(&key, &inflight).await;+ return Ok(entries);+ }+ }++ let entries = self.backend.read_dir(path).await;+ if let Ok(value) = &entries {+ self.metrics.backend_fallback(0);+ self.fill_directory(&key, &normalized, value).await;+ }+ drop(inflight_guard);+ self.release_inflight(&key, &inflight).await;+ entries+ }++ async fn fill_file(&self, key: &str, path: &str, data: &[u8]) {+ if !self.policy.cache_file(path, data.len()) {+ return;+ }+ let generations = match self.generation_snapshots(path).await {+ Ok(generations) => generations,+ Err(_) => {+ self.metrics.error();+ return;+ }+ };+ match CacheEnvelope::file(path.to_string(), data.to_vec(), generations).encode() {+ Ok(value) => {+ self.cache_put(key, value, path).await;+ }+ Err(_) => self.metrics.error(),+ }+ }++ async fn fill_directory(&self, key: &str, path: &str, entries: &[FileInfo]) {+ if !self.policy.cache_directory_entries(path, entries.len()) {+ return;+ }+ let generations = match self.generation_snapshots(path).await {+ Ok(generations) => generations,+ Err(_) => {+ self.metrics.error();+ return;+ }+ };+ match CacheEnvelope::directory(path.to_string(), entries.to_vec(), generations).encode() {+ Ok(value) => {+ self.cache_put(key, value, path).await;+ }+ Err(_) => self.metrics.error(),+ }+ }++ async fn invalidate_path_objects(&self, path: &str) {+ self.cache_delete(&self.file_key(path), path).await;+ self.cache_delete(&self.directory_key(path), path).await;+ }++ async fn invalidate_parent_directory(&self, path: &str) {+ let parent = parent_path(path);+ self.cache_delete(&self.directory_key(&parent), &parent)+ .await;+ }++ fn file_key(&self, path: &str) -> String {+ self.object_key("file", path)+ }++ fn directory_key(&self, path: &str) -> String {+ self.object_key("dir", path)+ }++ fn generation_key(&self, path: &str) -> String {+ self.object_key("subtree", path)+ }++ fn object_key(&self, kind: &str, path: &str) -> String {+ let normalized = normalize_path(path);+ format!(+ "ragfs:v1:{}:{}:{:016x}",+ self.namespace.as_str(),+ kind,+ stable_hash(normalized.as_bytes())+ )+ }+}++#[async_trait]+impl FileSystem for CachedFileSystem {+ async fn create(&self, path: &str) -> Result<()> {+ let _guard = self.operation_lock.write().await;+ self.backend.create(path).await?;+ self.invalidate_path_objects(path).await;+ self.invalidate_parent_directory(path).await;+ Ok(())+ }++ async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {+ let _guard = self.operation_lock.write().await;+ self.backend.mkdir(path, mode).await?;+ self.bump_generation(path).await;+ self.cache_delete(&self.directory_key(path), path).await;+ self.invalidate_parent_directory(path).await;+ Ok(())+ }++ async fn remove(&self, path: &str) -> Result<()> {+ let _guard = self.operation_lock.write().await;+ self.backend.remove(path).await?;+ self.bump_generation(path).await;+ self.invalidate_path_objects(path).await;+ self.invalidate_parent_directory(path).await;+ Ok(())+ }++ async fn remove_all(&self, path: &str) -> Result<()> {+ let _guard = self.operation_lock.write().await;+ self.backend.remove_all(path).await?;+ self.bump_generation(path).await;+ self.invalidate_path_objects(path).await;+ self.invalidate_parent_directory(path).await;+ Ok(())+ }++ async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {+ if offset != 0+ || size != 0+ || !self.policy.cache_file(path, 0)+ || self.is_runtime_bypassed(path).await+ {+ self.metrics.policy_bypass();+ return self.backend.read(path, offset, size).await;+ }++ let _operation_guard = self.operation_lock.read().await;+ if self.is_runtime_bypassed(path).await {+ self.metrics.policy_bypass();+ return self.backend.read(path, offset, size).await;+ }++ let normalized = normalize_path(path);+ let key = self.file_key(&normalized);+ if let Some(data) = self.probe_file(&key, &normalized, true).await {+ return Ok(data);+ }+ self.metrics.file_miss();++ let (inflight, leader) = self.acquire_inflight(&key).await;+ if leader {+ self.metrics.inflight_leader();+ } else {+ self.metrics.inflight_follower();+ }+ let inflight_guard = inflight.lock().await;++ if !leader {+ if let Some(data) = self.probe_file(&key, &normalized, false).await {+ self.metrics.inflight_backend_saved();+ drop(inflight_guard);+ self.release_inflight(&key, &inflight).await;+ return Ok(data);+ }+ }++ let data = self.backend.read(path, 0, 0).await;+ if let Ok(value) = &data {+ self.metrics.backend_fallback(value.len());+ self.fill_file(&key, &normalized, value).await;+ }+ drop(inflight_guard);+ self.release_inflight(&key, &inflight).await;+ data+ }++ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {+ let _guard = self.operation_lock.write().await;+ let written = self.backend.write(path, data, offset, flags).await?;+ let normalized = normalize_path(path);+ let key = self.file_key(&normalized);+ self.cache_delete(&key, &normalized).await;+ if offset == 0+ && matches!(flags, WriteFlag::Create | WriteFlag::Truncate)+ && self.policy.cache_file(&normalized, data.len())+ && !self.is_runtime_bypassed(&normalized).await+ {+ self.fill_file(&key, &normalized, data).await;+ }+ self.invalidate_parent_directory(path).await;+ Ok(written)+ }++ async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {+ if !self.policy.cache_directory(path) || self.is_runtime_bypassed(path).await {+ self.metrics.policy_bypass();+ return self.backend.read_dir(path).await;+ }++ let _operation_guard = self.operation_lock.read().await;+ if self.is_runtime_bypassed(path).await {+ self.metrics.policy_bypass();+ return self.backend.read_dir(path).await;+ }++ let normalized = normalize_path(path);+ let key = self.directory_key(&normalized);+ if let Some(entries) = self.probe_directory(&key, &normalized, true).await {+ return Ok(entries);+ }+ self.metrics.read_dir_miss();++ let (inflight, leader) = self.acquire_inflight(&key).await;+ if leader {+ self.metrics.inflight_leader();+ } else {+ self.metrics.inflight_follower();+ }+ let inflight_guard = inflight.lock().await;++ if !leader {+ if let Some(entries) = self.probe_directory(&key, &normalized, false).await {+ self.metrics.inflight_backend_saved();+ drop(inflight_guard);+ self.release_inflight(&key, &inflight).await;+ return Ok(entries);+ }+ }++ let entries = self.backend.read_dir(path).await;+ if let Ok(value) = &entries {+ self.metrics.backend_fallback(0);+ self.fill_directory(&key, &normalized, value).await;+ }+ drop(inflight_guard);+ self.release_inflight(&key, &inflight).await;+ entries+ }++ async fn stat(&self, path: &str) -> Result<FileInfo> {+ self.backend.stat(path).await+ }++ async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {+ let _guard = self.operation_lock.write().await;+ self.backend.rename(old_path, new_path).await?;+ self.bump_generation(old_path).await;+ self.bump_generation(new_path).await;+ self.invalidate_path_objects(old_path).await;+ self.invalidate_path_objects(new_path).await;+ self.invalidate_parent_directory(old_path).await;+ self.invalidate_parent_directory(new_path).await;+ Ok(())+ }++ async fn chmod(&self, path: &str, mode: u32) -> Result<()> {+ let _guard = self.operation_lock.write().await;+ self.backend.chmod(path, mode).await?;+ self.invalidate_path_objects(path).await;+ self.invalidate_parent_directory(path).await;+ Ok(())+ }++ async fn truncate(&self, path: &str, size: u64) -> Result<()> {+ let _guard = self.operation_lock.write().await;+ self.backend.truncate(path, size).await?;+ self.cache_delete(&self.file_key(path), path).await;+ self.invalidate_parent_directory(path).await;+ Ok(())+ }++ async fn exists(&self, path: &str) -> bool {+ self.backend.exists(path).await+ }++ async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {+ let _guard = self.operation_lock.write().await;+ self.backend.ensure_parent_dirs(path, mode).await?;+ for scope in ancestor_scopes(&parent_path(path)) {+ self.cache_delete(&self.directory_key(&scope), &scope).await;+ }+ Ok(())+ }++ async fn grep(+ &self,+ path: &str,+ pattern: &str,+ recursive: bool,+ case_insensitive: bool,+ node_limit: Option<usize>,+ exclude_path: Option<&str>,+ level_limit: Option<usize>,+ ) -> Result<GrepResult> {+ if self.policy.traversal_mode() == CacheTraversalMode::CachedTraversal+ && !self.wraps_multiwrite()+ {+ return self+ .grep_via_cache(+ path,+ pattern,+ recursive,+ case_insensitive,+ node_limit,+ exclude_path,+ level_limit,+ )+ .await;+ }++ self.backend+ .grep(+ path,+ pattern,+ recursive,+ case_insensitive,+ node_limit,+ exclude_path,+ level_limit,+ )+ .await+ }++ async fn tree_directory(+ &self,+ path: &str,+ show_hidden: bool,+ node_limit: Option<usize>,+ level_limit: Option<usize>,+ ) -> Result<Vec<TreeEntry>> {+ if self.policy.traversal_mode() == CacheTraversalMode::CachedTraversal+ && !self.wraps_multiwrite()+ {+ return self+ .tree_directory_via_cache(path, show_hidden, node_limit, level_limit)+ .await;+ }++ self.backend+ .tree_directory(path, show_hidden, node_limit, level_limit)+ .await+ }+}++fn normalize_path(path: &str) -> String {+ if path.is_empty() || path == "/" {+ "/".to_string()+ } else {+ format!("/{}", path.trim_matches('/'))+ }+}++fn parent_path(path: &str) -> String {+ let normalized = normalize_path(path);+ if normalized == "/" {+ return "/".to_string();+ }+ normalized+ .rsplit_once('/')+ .map(|(parent, _)| {+ if parent.is_empty() {+ "/".to_string()+ } else {+ parent.to_string()+ }+ })+ .unwrap_or_else(|| "/".to_string())+}++fn ancestor_scopes(path: &str) -> Vec<String> {+ let normalized = normalize_path(path);+ if normalized == "/" {+ return vec!["/".to_string()];+ }++ let mut scopes = vec!["/".to_string()];+ let mut current = String::new();+ for component in normalized.trim_start_matches('/').split('/') {+ current.push('/');+ current.push_str(component);+ scopes.push(current.clone());+ }+ scopes+}++fn is_same_or_descendant(path: &str, scope: &str) -> bool {+ scope == "/"+ || path == scope+ || path+ .strip_prefix(scope)+ .is_some_and(|suffix| suffix.starts_with('/'))+}++fn stable_hash(bytes: &[u8]) -> u64 {+ let mut hash = 0xcbf29ce484222325_u64;+ for byte in bytes {+ hash ^= u64::from(*byte);+ hash = hash.wrapping_mul(0x100000001b3);+ }+ hash+}++fn nonzero_random_u64() -> u64 {+ let value = Uuid::new_v4().as_u128() as u64;+ if value == 0 {+ 1+ } else {+ value+ }+}diff --git a/crates/ragfs/src/core/filesystem.rs b/crates/ragfs/src/core/filesystem.rsindex 121315d7c..d7fa57132 100644--- a/crates/ragfs/src/core/filesystem.rs+++ b/crates/ragfs/src/core/filesystem.rs@@ -24,7 +24,7 @@ pub(crate) fn normalize_prefix_path(path: &str) -> String {}/// Check whether `path` is under `exclude_path` (including itself).-fn is_excluded_path(path: &str, exclude_path: &str) -> bool {+pub(crate) fn is_excluded_path(path: &str, exclude_path: &str) -> bool {if exclude_path == "/" {return true;}diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rsindex a1a922f69..15015279b 100644--- a/crates/ragfs/src/core/mountable.rs+++ b/crates/ragfs/src/core/mountable.rs@@ -27,6 +27,10 @@ use super::plugin::ServicePlugin;use super::stats::{FilesystemStats, StatsCollector};use super::stats_wrapper::StatsWrappedFS;use super::types::{BackendsConfig, FileInfo, GrepResult, PluginConfig, TreeEntry, WriteFlag};+#[cfg(feature = "cache")]+use crate::cache::{+ CacheNamespace, CachePolicy, CacheProvider, CacheTraversalMode, CachedFileSystem,+};/// Information about a mounted filesystem#[derive(Clone)]@@ -67,6 +71,18 @@ pub struct MountableFS {/// multi-write mounts use it for per-backend encryption decisions.encryption_root_key: RwLock<Option<[u8; 32]>>,encryption_provider_type: RwLock<Option<u8>>,++ /// Optional cache configuration shared by mounted filesystems.+ #[cfg(feature = "cache")]+ cache: Option<MountCacheConfig>,+}++#[cfg(feature = "cache")]+#[derive(Clone)]+struct MountCacheConfig {+ provider: Arc<dyn CacheProvider>,+ namespace: CacheNamespace,+ policy: CachePolicy,}impl MountableFS {@@ -91,7 +107,11 @@ impl MountableFS {/// Try to unwrap a mounted filesystem stack to the underlying multi-write wrapper.fn as_multiwrite(fs: &Arc<dyn FileSystem>) -> Option<&MultiWriteWrappedFS> {- let any = fs.as_ref() as &dyn std::any::Any;+ Self::as_multiwrite_ref(fs.as_ref())+ }++ fn as_multiwrite_ref(fs: &dyn FileSystem) -> Option<&MultiWriteWrappedFS> {+ let any = fs as &dyn std::any::Any;if let Some(mw) = any.downcast_ref::<MultiWriteWrappedFS>() {return Some(mw);}@@ -101,9 +121,39 @@ impl MountableFS {if let Some(enc) = any.downcast_ref::<EncryptionWrappedFS>() {return Self::as_multiwrite(enc.inner_fs());}+ #[cfg(feature = "cache")]+ if let Some(cache) = any.downcast_ref::<CachedFileSystem>() {+ return Self::as_multiwrite_ref(cache.inner_fs());+ }+ #[cfg(feature = "cache")]+ if let Some(arc) = any.downcast_ref::<ArcFileSystem>() {+ return Self::as_multiwrite(&arc.0);+ }None}+ #[cfg(feature = "cache")]+ fn as_cached(fs: &Arc<dyn FileSystem>) -> Option<&CachedFileSystem> {+ let any = fs.as_ref() as &dyn std::any::Any;+ if let Some(cache) = any.downcast_ref::<CachedFileSystem>() {+ return Some(cache);+ }+ if let Some(stats) = any.downcast_ref::<StatsWrappedFS>() {+ return Self::as_cached(stats.inner_fs());+ }+ if let Some(enc) = any.downcast_ref::<EncryptionWrappedFS>() {+ return Self::as_cached(enc.inner_fs());+ }+ None+ }++ #[cfg(feature = "cache")]+ async fn invalidate_cache_after_raw_write(mount_info: &MountInfo, rel_path: &str) {+ if let Some(cache) = Self::as_cached(&mount_info.fs) {+ cache.invalidate_external_write(rel_path).await;+ }+ }+/// Query multi-write sync status for a mounted path.pub async fn system_sync_status(&self, path: &str) -> Result<Value> {let (mount_info, rel_path) = self.find_mount(path).await?;@@ -127,6 +177,31 @@ impl MountableFS {registry: Arc::new(RwLock::new(HashMap::new())),encryption_root_key: RwLock::new(None),encryption_provider_type: RwLock::new(None),+ #[cfg(feature = "cache")]+ cache: None,+ }+ }++ /// Create a new MountableFS that transparently wraps mounted backends with cache.+ ///+ /// Encrypted multi-write mounts skip the mount-level cache because their encryption boundary+ /// lives inside `MultiWriteWrappedFS`; caching outside it would store plaintext.+ #[cfg(feature = "cache")]+ pub fn with_cache(+ provider: Arc<dyn CacheProvider>,+ namespace: CacheNamespace,+ policy: CachePolicy,+ ) -> Self {+ Self {+ mounts: Arc::new(RwLock::new(Trie::new())),+ registry: Arc::new(RwLock::new(HashMap::new())),+ encryption_root_key: RwLock::new(None),+ encryption_provider_type: RwLock::new(None),+ cache: Some(MountCacheConfig {+ provider,+ namespace,+ policy,+ }),}}@@ -191,17 +266,21 @@ impl MountableFS {// Validate configurationplugin.validate(&config).await?;+ let (enc_root_key, enc_provider_type) = self.get_encryption_config().await;+ #[cfg(feature = "cache")]+ let encryption_enabled = enc_root_key.is_some() && enc_provider_type.is_some();+// Branch: multi-write vs single backendlet (inner_fs, raw_fs): (Arc<dyn FileSystem>, Option<Arc<dyn FileSystem>>) = match &config.backups{None => {- // Single backend: initialize plugin, optionally wrap with encryption.+ // Single backend: initialize plugin, optionally wrap raw storage with cache, then+ // wrap with encryption. This keeps shared cache providers ciphertext-only.// Control plugins (queuefs, serverinfofs) are never encrypted.let raw = plugin.initialize(config.clone()).await?;let raw_arc: Arc<dyn FileSystem> = Arc::from(raw);let is_control_plugin = matches!(config.name.as_str(), "queuefs" | "serverinfofs");- let (enc_root_key, enc_provider_type) = self.get_encryption_config().await;if !is_control_plugin {ensure_backend_shape(&raw_arc,@@ -212,14 +291,19 @@ impl MountableFS {).await?;}+ #[cfg(feature = "cache")]+ let storage_fs = self.maybe_wrap_cache(raw_arc.clone(), &normalized_path);+ #[cfg(not(feature = "cache"))]+ let storage_fs = raw_arc.clone();+let inner: Arc<dyn FileSystem> = if is_control_plugin {- raw_arc.clone()+ storage_fs} else {match (enc_root_key, enc_provider_type) {(Some(rk), Some(pt)) => {- Arc::new(EncryptionWrappedFS::new(raw_arc.clone(), rk, pt))+ Arc::new(EncryptionWrappedFS::new(storage_fs, rk, pt))}- _ => raw_arc.clone(),+ _ => storage_fs,}};(inner, Some(raw_arc))@@ -231,6 +315,29 @@ impl MountableFS {);let mw = self.build_multi_write_fs(&config, bc).await?;let arc: Arc<dyn FileSystem> = Arc::new(mw);+ #[cfg(feature = "cache")]+ let arc = if encryption_enabled {+ // Multi-write owns per-backend encryption internally. A mount-level cache+ // would sit outside that boundary and store plaintext, so skip it.+ warn!(+ "cache is disabled for encrypted multi-write mount '{}': mount-level cache would store plaintext",+ normalized_path+ );+ arc+ } else {+ match &self.cache {+ Some(cache) => Arc::new(CachedFileSystem::new(+ Box::new(ArcFileSystem(arc)),+ cache.provider.clone(),+ mount_namespace(&cache.namespace, &normalized_path),+ cache+ .policy+ .clone()+ .with_traversal_mode(CacheTraversalMode::Backend),+ )),+ None => arc,+ }+ };(arc, None)}};@@ -279,6 +386,31 @@ impl MountableFS {.await}+ #[cfg(feature = "cache")]+ fn maybe_wrap_cache(&self, fs: Arc<dyn FileSystem>, mount_path: &str) -> Arc<dyn FileSystem> {+ match &self.cache {+ Some(cache) => {+ let policy = if cache.policy.traversal_mode() == CacheTraversalMode::CachedTraversal+ && Self::as_multiwrite(&fs).is_some()+ {+ cache+ .policy+ .clone()+ .with_traversal_mode(CacheTraversalMode::Backend)+ } else {+ cache.policy.clone()+ };+ Arc::new(CachedFileSystem::new(+ Box::new(ArcFileSystem(fs)),+ cache.provider.clone(),+ mount_namespace(&cache.namespace, mount_path),+ policy,+ ))+ }+ None => fs,+ }+ }+/// Unmount a filesystem at the specified path////// # Arguments@@ -396,6 +528,8 @@ impl MountableFS {};Self::copy_raw_within_mount(raw_backend, &src_rel_path, &dst_rel_path).await?;+ #[cfg(feature = "cache")]+ Self::invalidate_cache_after_raw_write(&dst_mount, &dst_rel_path).await;Ok(true)}@@ -493,6 +627,58 @@ impl MountableFS {}}+#[cfg(feature = "cache")]+struct ArcFileSystem(Arc<dyn FileSystem>);++#[cfg(feature = "cache")]+#[async_trait]+impl FileSystem for ArcFileSystem {+ async fn create(&self, path: &str) -> Result<()> {+ self.0.create(path).await+ }++ async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {+ self.0.mkdir(path, mode).await+ }++ async fn remove(&self, path: &str) -> Result<()> {+ self.0.remove(path).await+ }++ async fn remove_all(&self, path: &str) -> Result<()> {+ self.0.remove_all(path).await+ }++ async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {+ self.0.read(path, offset, size).await+ }++ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {+ self.0.write(path, data, offset, flags).await+ }++ async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {+ self.0.read_dir(path).await+ }++ async fn stat(&self, path: &str) -> Result<FileInfo> {+ self.0.stat(path).await+ }++ async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {+ self.0.rename(old_path, new_path).await+ }++ async fn chmod(&self, path: &str, mode: u32) -> Result<()> {+ self.0.chmod(path, mode).await+ }+}++#[cfg(feature = "cache")]+fn mount_namespace(base: &CacheNamespace, mount_path: &str) -> CacheNamespace {+ CacheNamespace::new(format!("{}:{}", base.as_str(), mount_path))+}+impl Default for MountableFS {fn default() -> Self {Self::new()@@ -695,6 +881,7 @@ mod tests {use crate::shape::SHAPE_MANIFEST_PATH;use serde_json::Value;use std::collections::HashMap;+ use std::sync::atomic::{AtomicU64, Ordering};/// Helper to create a PluginConfig for tests with new fields defaulted.fn test_config(name: &str, mount_path: &str) -> PluginConfig {@@ -844,6 +1031,14 @@ mod tests {tree_entries: Option<Vec<TreeEntry>>,}+ struct CountingPlugin {+ reads: Arc<AtomicU64>,+ }++ struct CountingFs {+ reads: Arc<AtomicU64>,+ }+impl MockPlugin {fn new(name: &str) -> Self {Self {@@ -860,6 +1055,81 @@ mod tests {}}+ #[async_trait]+ impl ServicePlugin for CountingPlugin {+ fn name(&self) -> &str {+ "counting"+ }++ fn readme(&self) -> &str {+ "Counting plugin for cache tests"+ }++ async fn validate(&self, _config: &PluginConfig) -> Result<()> {+ Ok(())+ }++ async fn initialize(&self, _config: PluginConfig) -> Result<Box<dyn FileSystem>> {+ Ok(Box::new(CountingFs {+ reads: self.reads.clone(),+ }))+ }++ fn config_params(&self) -> &[super::super::types::ConfigParameter] {+ &[]+ }+ }++ #[async_trait]+ impl FileSystem for CountingFs {+ async fn create(&self, _path: &str) -> Result<()> {+ Ok(())+ }++ async fn mkdir(&self, _path: &str, _mode: u32) -> Result<()> {+ Ok(())+ }++ async fn remove(&self, _path: &str) -> Result<()> {+ Ok(())+ }++ async fn remove_all(&self, _path: &str) -> Result<()> {+ Ok(())+ }++ async fn read(&self, path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {+ self.reads.fetch_add(1, Ordering::Relaxed);+ Ok(format!("backend:{path}").into_bytes())+ }++ async fn write(+ &self,+ _path: &str,+ data: &[u8],+ _offset: u64,+ _flags: WriteFlag,+ ) -> Result<u64> {+ Ok(data.len() as u64)+ }++ async fn read_dir(&self, _path: &str) -> Result<Vec<FileInfo>> {+ Ok(vec![])+ }++ async fn stat(&self, path: &str) -> Result<FileInfo> {+ Ok(FileInfo::new_file(path.to_string(), 0, 0o644))+ }++ async fn rename(&self, _old_path: &str, _new_path: &str) -> Result<()> {+ Ok(())+ }++ async fn chmod(&self, _path: &str, _mode: u32) -> Result<()> {+ Ok(())+ }+ }+#[async_trait]impl ServicePlugin for MockPlugin {fn name(&self) -> &str {@@ -950,6 +1220,246 @@ mod tests {assert!(mounts.is_empty());}+ #[cfg(feature = "cache")]+ #[tokio::test]+ async fn mount_wraps_backend_with_cache_when_configured() {+ use crate::cache::{CacheNamespace, CachePolicy, MemoryCacheProvider};++ let reads = Arc::new(AtomicU64::new(0));+ let mfs = MountableFS::with_cache(+ Arc::new(MemoryCacheProvider::new()),+ CacheNamespace::new("mount-test"),+ CachePolicy::default(),+ );+ mfs.register_plugin(CountingPlugin {+ reads: reads.clone(),+ })+ .await;++ let config = PluginConfig::single_backend("counting", "/cached", HashMap::new());++ mfs.mount(config).await.unwrap();+ reads.store(0, Ordering::Relaxed);++ assert_eq!(+ mfs.read("/cached/file.txt", 0, 0).await.unwrap(),+ b"backend:/file.txt"+ );+ assert_eq!(+ mfs.read("/cached/file.txt", 0, 0).await.unwrap(),+ b"backend:/file.txt"+ );+ assert_eq!(reads.load(Ordering::Relaxed), 1);+ }++ #[cfg(feature = "cache")]+ #[tokio::test]+ async fn copy_within_mount_overwrite_invalidates_cached_destination() {+ use crate::cache::{CacheNamespace, CachePolicy, MemoryCacheProvider};+ use crate::plugins::MemFSPlugin;++ let mfs = MountableFS::with_cache(+ Arc::new(MemoryCacheProvider::new()),+ CacheNamespace::new("copy-cache-test"),+ CachePolicy::default(),+ );+ mfs.register_plugin(MemFSPlugin).await;+ mfs.mount(test_config("memfs", "/local")).await.unwrap();+ mfs.mkdir("/local/dir", 0o755).await.unwrap();+ mfs.write("/local/dir/a.md", b"new-content", 0, WriteFlag::Create)+ .await+ .unwrap();+ mfs.write("/local/dir/b.md", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();++ assert_eq!(mfs.read("/local/dir/b.md", 0, 0).await.unwrap(), b"old");+ assert_eq!(+ mfs.read_dir("/local/dir")+ .await+ .unwrap()+ .into_iter()+ .find(|entry| entry.name == "b.md")+ .unwrap()+ .size,+ 3+ );++ assert!(mfs+ .copy_within_mount("/local/dir/a.md", "/local/dir/b.md")+ .await+ .unwrap());++ let copied = mfs.read("/local/dir/b.md", 0, 0).await.unwrap();+ let copied_size = mfs+ .read_dir("/local/dir")+ .await+ .unwrap()+ .into_iter()+ .find(|entry| entry.name == "b.md")+ .unwrap()+ .size;+ assert_eq!(copied, b"new-content");+ assert_eq!(copied_size, b"new-content".len() as u64);+ }++ #[cfg(feature = "cache")]+ #[tokio::test]+ async fn encrypted_mount_caches_ciphertext_below_account_validation() {+ use crate::cache::{CacheNamespace, CachePolicy, CacheProvider, MemoryCacheProvider};+ use crate::core::{FsContextInner, FS_CTX};+ use crate::plugins::MemFSPlugin;++ let provider = Arc::new(MemoryCacheProvider::new());+ let mfs = MountableFS::with_cache(+ provider.clone(),+ CacheNamespace::new("encrypted-mount-test"),+ CachePolicy::default(),+ );+ mfs.register_plugin(MemFSPlugin).await;+ mfs.set_encryption_config(Some([9u8; 32]), Some(1)).await;+ mfs.mount(test_config("memfs", "/local")).await.unwrap();+ mfs.mkdir("/local/shared", 0o755).await.unwrap();++ let tenant_a = Arc::new(FsContextInner::new("tenant-a"));+ let tenant_b = Arc::new(FsContextInner::new("tenant-b"));+ FS_CTX+ .scope(tenant_a.clone(), async {+ mfs.write(+ "/local/shared/doc.md",+ b"tenant-a-secret",+ 0,+ WriteFlag::Create,+ )+ .await+ .unwrap();+ })+ .await;++ assert_eq!(+ FS_CTX+ .scope(tenant_a, mfs.read("/local/shared/doc.md", 0, 0))+ .await+ .unwrap(),+ b"tenant-a-secret"+ );+ assert!(+ FS_CTX+ .scope(tenant_b, mfs.read("/local/shared/doc.md", 0, 0))+ .await+ .is_err(),+ "a different account must not receive cached plaintext"+ );+ assert!(+ mfs.read("/local/shared/doc.md", 0, 0).await.is_err(),+ "missing account context must not receive cached plaintext"+ );++ let file_key = provider+ .keys()+ .await+ .into_iter()+ .find(|key| key.contains(":file:"))+ .expect("encrypted read should populate one file cache object");+ let encoded = provider+ .get(&file_key)+ .await+ .unwrap()+ .expect("file cache object should exist");+ let envelope: Value = serde_json::from_slice(&encoded).unwrap();+ let payload = envelope["payload"]["File"]+ .as_array()+ .expect("file payload should be a byte array")+ .iter()+ .map(|value| value.as_u64().unwrap() as u8)+ .collect::<Vec<_>>();+ assert!(+ payload.starts_with(b"OVE1"),+ "shared cache providers must store encrypted file envelopes"+ );+ }++ #[cfg(feature = "cache")]+ #[tokio::test]+ async fn encrypted_multiwrite_mount_does_not_install_plaintext_cache() {+ use crate::cache::{CacheNamespace, CachePolicy, MemoryCacheProvider};++ let mfs = MountableFS::with_cache(+ Arc::new(MemoryCacheProvider::new()),+ CacheNamespace::new("encrypted-multiwrite-test"),+ CachePolicy::default(),+ );+ mfs.register_plugin(MockPlugin::new("primary")).await;+ mfs.register_plugin(MockPlugin::new("backupfs")).await;+ mfs.set_encryption_config(Some([9u8; 32]), Some(1)).await;++ let mut config = multiwrite_test_config("primary", "backupfs", "/local");+ config.server_encryption_enabled = true;+ config.primary_encryption_enabled = true;+ mfs.mount(config).await.unwrap();++ let mounts = mfs.mounts.read().await;+ let mount_info = mounts.get("/local").expect("mounted entry should exist");+ let stats = (mount_info.fs.as_ref() as &dyn std::any::Any)+ .downcast_ref::<StatsWrappedFS>()+ .expect("stats wrapper should be outermost");+ assert!(+ (stats.inner_fs().as_ref() as &dyn std::any::Any)+ .downcast_ref::<MultiWriteWrappedFS>()+ .is_some(),+ "encrypted multi-write must not install a plaintext cache outside MultiWriteWrappedFS"+ );+ }++ #[cfg(feature = "cache")]+ #[tokio::test]+ async fn cached_unencrypted_multiwrite_keeps_admin_and_copy_fast_paths() {+ use crate::cache::{CacheNamespace, CachePolicy, MemoryCacheProvider};+ use crate::core::{FsContextInner, FS_CTX};+ use crate::plugins::MemFSPlugin;++ let mfs = MountableFS::with_cache(+ Arc::new(MemoryCacheProvider::new()),+ CacheNamespace::new("cached-multiwrite-test"),+ CachePolicy::default(),+ );+ mfs.register_plugin(MemFSPlugin).await;++ let mut config = multiwrite_test_config("memfs", "memfs", "/local");+ config.backups.as_mut().unwrap().items[0].name = "backup1".to_string();+ mfs.mount(config).await.unwrap();++ let ctx = Arc::new(FsContextInner::new("acct"));+ FS_CTX+ .scope(ctx.clone(), async {+ let status = mfs.system_sync_status("/local").await.unwrap();+ assert_eq!(status["path"], "/");+ assert_eq!(status["entry_count"], 0);++ let retry = mfs.system_sync_retry("/local").await.unwrap();+ assert_eq!(retry["path"], "/");+ assert_eq!(retry["retried"], 0);++ mfs.mkdir("/local/docs", 0o755).await.unwrap();+ mfs.write("/local/docs/src.md", b"copied", 0, WriteFlag::Create)+ .await+ .unwrap();++ assert!(mfs+ .copy_within_mount("/local/docs/src.md", "/local/docs/dst.md")+ .await+ .unwrap());+ assert_eq!(+ mfs.read("/local/docs/dst.md", 0, 0).await.unwrap(),+ b"copied"+ );+ })+ .await;++ mfs.unmount("/local").await.unwrap();+ assert!(mfs.list_mounts().await.is_empty());+ }+#[tokio::test]async fn test_mount_duplicate_error() {let mfs = MountableFS::new();diff --git a/crates/ragfs/src/lib.rs b/crates/ragfs/src/lib.rsindex 9d3079cc5..47106ddc6 100644--- a/crates/ragfs/src/lib.rs+++ b/crates/ragfs/src/lib.rs@@ -29,6 +29,8 @@#![warn(missing_docs)]#![warn(clippy::all)]+#[cfg(feature = "cache")]+pub mod cache;pub mod core;pub mod crypto;pub mod multibackend;diff --git a/crates/ragfs/tests/cache_wrapper.rs b/crates/ragfs/tests/cache_wrapper.rsnew file mode 100644index 000000000..d624e59ae--- /dev/null+++ b/crates/ragfs/tests/cache_wrapper.rs@@ -0,0 +1,1438 @@+use async_trait::async_trait;+use bytes::Bytes;+use ragfs::cache::{+ CacheDecision, CacheError, CacheNamespace, CachePolicy, CacheProvider, CacheResult,+ CacheTraversalMode, CachedFileSystem, MemoryCacheProvider, MemoryMockProvider,+};+use ragfs::core::{FsContextInner, GrepResult, MultiWriteWrappedFS, TreeEntry, FS_CTX};+use ragfs::plugins::MemFileSystem;+use ragfs::{FileInfo, FileSystem, Result, WriteFlag};+use std::sync::atomic::{AtomicU64, Ordering};+use std::sync::{Arc, Mutex};+use std::time::Duration;++#[derive(Clone)]+struct CountingFileSystem {+ inner: Arc<MemFileSystem>,+ reads: Arc<AtomicU64>,+ read_dirs: Arc<AtomicU64>,+ stats: Arc<AtomicU64>,+ greps: Arc<AtomicU64>,+ trees: Arc<AtomicU64>,+ read_delay: Duration,+}++struct DeleteFailingProvider {+ inner: MemoryCacheProvider,+}++struct TrackingProvider {+ inner: MemoryCacheProvider,+ gets: AtomicU64,+ batch_gets: AtomicU64,+ active_gets: AtomicU64,+ max_active_gets: AtomicU64,+ seen_get_keys: Mutex<Vec<String>>,+ seen_batch_get_keys: Mutex<Vec<Vec<String>>>,+ get_delay: Duration,+}++struct UnavailableProvider;++impl DeleteFailingProvider {+ fn new() -> Self {+ Self {+ inner: MemoryCacheProvider::new(),+ }+ }+}++impl TrackingProvider {+ fn new() -> Self {+ Self {+ inner: MemoryCacheProvider::new(),+ gets: AtomicU64::new(0),+ batch_gets: AtomicU64::new(0),+ active_gets: AtomicU64::new(0),+ max_active_gets: AtomicU64::new(0),+ seen_get_keys: Mutex::new(Vec::new()),+ seen_batch_get_keys: Mutex::new(Vec::new()),+ get_delay: Duration::ZERO,+ }+ }++ fn with_get_delay(mut self, delay: Duration) -> Self {+ self.get_delay = delay;+ self+ }++ fn reset_observed_reads(&self) {+ self.gets.store(0, Ordering::Relaxed);+ self.batch_gets.store(0, Ordering::Relaxed);+ self.active_gets.store(0, Ordering::Relaxed);+ self.max_active_gets.store(0, Ordering::Relaxed);+ self.seen_get_keys.lock().unwrap().clear();+ self.seen_batch_get_keys.lock().unwrap().clear();+ }++ fn batch_get_count(&self) -> u64 {+ self.batch_gets.load(Ordering::Relaxed)+ }++ fn observed_read_keys(&self) -> Vec<String> {+ let mut keys = self.seen_get_keys.lock().unwrap().clone();+ keys.extend(+ self.seen_batch_get_keys+ .lock()+ .unwrap()+ .iter()+ .flat_map(|batch| batch.iter().cloned()),+ );+ keys+ }++ fn max_concurrent_gets(&self) -> u64 {+ self.max_active_gets.load(Ordering::Relaxed)+ }++ fn enter_get(&self) {+ let active = self.active_gets.fetch_add(1, Ordering::Relaxed) + 1;+ let mut current = self.max_active_gets.load(Ordering::Relaxed);+ while active > current {+ match self.max_active_gets.compare_exchange_weak(+ current,+ active,+ Ordering::Relaxed,+ Ordering::Relaxed,+ ) {+ Ok(_) => break,+ Err(observed) => current = observed,+ }+ }+ }++ fn exit_get(&self) {+ self.active_gets.fetch_sub(1, Ordering::Relaxed);+ }+}++#[async_trait]+impl CacheProvider for DeleteFailingProvider {+ fn name(&self) -> &'static str {+ "delete-failing"+ }++ async fn get(&self, key: &str) -> CacheResult<Option<Bytes>> {+ self.inner.get(key).await+ }++ async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> {+ self.inner.put(key, value).await+ }++ async fn delete(&self, _key: &str) -> CacheResult<()> {+ Err(CacheError::Unavailable(+ "delete intentionally failed".to_string(),+ ))+ }+}++#[async_trait]+impl CacheProvider for TrackingProvider {+ fn name(&self) -> &'static str {+ "tracking"+ }++ fn capabilities(&self) -> ragfs::cache::ProviderCapabilities {+ self.inner.capabilities()+ }++ async fn get(&self, key: &str) -> CacheResult<Option<Bytes>> {+ self.gets.fetch_add(1, Ordering::Relaxed);+ self.seen_get_keys.lock().unwrap().push(key.to_string());+ self.enter_get();+ if !self.get_delay.is_zero() {+ tokio::time::sleep(self.get_delay).await;+ }+ let result = self.inner.get(key).await;+ self.exit_get();+ result+ }++ async fn put(&self, key: &str, value: Bytes) -> CacheResult<()> {+ self.inner.put(key, value).await+ }++ async fn delete(&self, key: &str) -> CacheResult<()> {+ self.inner.delete(key).await+ }++ async fn batch_get(&self, keys: &[String]) -> CacheResult<Vec<Option<Bytes>>> {+ self.batch_gets.fetch_add(1, Ordering::Relaxed);+ self.seen_batch_get_keys.lock().unwrap().push(keys.to_vec());+ self.inner.batch_get(keys).await+ }++ async fn batch_put(&self, entries: Vec<(String, Bytes)>) -> CacheResult<()> {+ self.inner.batch_put(entries).await+ }++ async fn invalidate(&self, keys: &[String]) -> CacheResult<()> {+ self.inner.invalidate(keys).await+ }+}++#[async_trait]+impl CacheProvider for UnavailableProvider {+ fn name(&self) -> &'static str {+ "unavailable"+ }++ async fn get(&self, _key: &str) -> CacheResult<Option<Bytes>> {+ Err(CacheError::Unavailable("provider is down".to_string()))+ }++ async fn put(&self, _key: &str, _value: Bytes) -> CacheResult<()> {+ Err(CacheError::Unavailable("provider is down".to_string()))+ }++ async fn delete(&self, _key: &str) -> CacheResult<()> {+ Err(CacheError::Unavailable("provider is down".to_string()))+ }+}++impl CountingFileSystem {+ fn new() -> Self {+ Self {+ inner: Arc::new(MemFileSystem::new()),+ reads: Arc::new(AtomicU64::new(0)),+ read_dirs: Arc::new(AtomicU64::new(0)),+ stats: Arc::new(AtomicU64::new(0)),+ greps: Arc::new(AtomicU64::new(0)),+ trees: Arc::new(AtomicU64::new(0)),+ read_delay: Duration::ZERO,+ }+ }++ fn with_read_delay(mut self, delay: Duration) -> Self {+ self.read_delay = delay;+ self+ }++ fn read_count(&self) -> u64 {+ self.reads.load(Ordering::Relaxed)+ }++ fn read_dir_count(&self) -> u64 {+ self.read_dirs.load(Ordering::Relaxed)+ }++ fn stat_count(&self) -> u64 {+ self.stats.load(Ordering::Relaxed)+ }++ fn grep_count(&self) -> u64 {+ self.greps.load(Ordering::Relaxed)+ }++ fn tree_count(&self) -> u64 {+ self.trees.load(Ordering::Relaxed)+ }+}++#[async_trait]+impl FileSystem for CountingFileSystem {+ async fn create(&self, path: &str) -> Result<()> {+ self.inner.create(path).await+ }++ async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {+ self.inner.mkdir(path, mode).await+ }++ async fn remove(&self, path: &str) -> Result<()> {+ self.inner.remove(path).await+ }++ async fn remove_all(&self, path: &str) -> Result<()> {+ self.inner.remove_all(path).await+ }++ async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {+ self.reads.fetch_add(1, Ordering::Relaxed);+ if !self.read_delay.is_zero() {+ tokio::time::sleep(self.read_delay).await;+ }+ self.inner.read(path, offset, size).await+ }++ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {+ self.inner.write(path, data, offset, flags).await+ }++ async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {+ self.read_dirs.fetch_add(1, Ordering::Relaxed);+ self.inner.read_dir(path).await+ }++ async fn stat(&self, path: &str) -> Result<FileInfo> {+ self.stats.fetch_add(1, Ordering::Relaxed);+ self.inner.stat(path).await+ }++ async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {+ self.inner.rename(old_path, new_path).await+ }++ async fn chmod(&self, path: &str, mode: u32) -> Result<()> {+ self.inner.chmod(path, mode).await+ }++ async fn truncate(&self, path: &str, size: u64) -> Result<()> {+ self.inner.truncate(path, size).await+ }++ async fn grep(+ &self,+ path: &str,+ pattern: &str,+ recursive: bool,+ case_insensitive: bool,+ node_limit: Option<usize>,+ exclude_path: Option<&str>,+ level_limit: Option<usize>,+ ) -> Result<GrepResult> {+ self.greps.fetch_add(1, Ordering::Relaxed);+ self.inner+ .grep(+ path,+ pattern,+ recursive,+ case_insensitive,+ node_limit,+ exclude_path,+ level_limit,+ )+ .await+ }++ async fn tree_directory(+ &self,+ path: &str,+ show_hidden: bool,+ node_limit: Option<usize>,+ level_limit: Option<usize>,+ ) -> Result<Vec<TreeEntry>> {+ self.trees.fetch_add(1, Ordering::Relaxed);+ self.inner+ .tree_directory(path, show_hidden, node_limit, level_limit)+ .await+ }+}++fn cached_fs(backend: CountingFileSystem) -> (Arc<CachedFileSystem>, Arc<MemoryCacheProvider>) {+ cached_fs_with_policy(backend, CachePolicy::default())+}++fn cached_fs_with_policy(+ backend: CountingFileSystem,+ policy: CachePolicy,+) -> (Arc<CachedFileSystem>, Arc<MemoryCacheProvider>) {+ let provider = Arc::new(MemoryCacheProvider::new());+ let fs = Arc::new(CachedFileSystem::new(+ Box::new(backend),+ provider.clone(),+ CacheNamespace::new("test"),+ policy,+ ));+ (fs, provider)+}++fn cached_fs_with_tracking_provider(+ backend: CountingFileSystem,+ policy: CachePolicy,+) -> (Arc<CachedFileSystem>, Arc<TrackingProvider>) {+ cached_fs_with_tracking_provider_instance(backend, policy, Arc::new(TrackingProvider::new()))+}++fn cached_fs_with_tracking_provider_instance(+ backend: CountingFileSystem,+ policy: CachePolicy,+ provider: Arc<TrackingProvider>,+) -> (Arc<CachedFileSystem>, Arc<TrackingProvider>) {+ let fs = Arc::new(CachedFileSystem::new(+ Box::new(backend),+ provider.clone(),+ CacheNamespace::new("tracking"),+ policy,+ ));+ (fs, provider)+}++#[test]+fn cache_policy_traversal_mode_defaults_to_backend() {+ assert_eq!(+ CachePolicy::default().traversal_mode(),+ CacheTraversalMode::Backend+ );+}++#[tokio::test]+async fn default_tree_directory_delegates_to_backend() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ backend+ .write("/docs/one.md", b"one", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ let entries = fs.tree_directory("/docs", false, None, None).await.unwrap();++ assert_eq!(entries.len(), 1);+ assert_eq!(probe.tree_count(), 1);+ assert_eq!(probe.read_dir_count(), 0);+}++#[tokio::test]+async fn default_grep_delegates_to_backend() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ backend+ .write("/docs/one.md", b"needle", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ let result = fs+ .grep("/docs", "needle", true, false, None, None, None)+ .await+ .unwrap();++ assert_eq!(result.count, 1);+ assert_eq!(probe.grep_count(), 1);+ assert_eq!(probe.read_dir_count(), 0);+ assert_eq!(probe.read_count(), 0);+}++#[tokio::test]+async fn cached_tree_traversal_reuses_directory_cache_after_warmup() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ backend.mkdir("/docs/sub", 0o755).await.unwrap();+ backend+ .write("/docs/one.md", b"one", 0, WriteFlag::Create)+ .await+ .unwrap();+ backend+ .write("/docs/sub/two.md", b"two", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs_with_policy(+ backend,+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ );++ let first = fs.tree_directory("/docs", false, None, None).await.unwrap();+ assert_eq!(first.len(), 3);+ assert_eq!(probe.tree_count(), 0);+ assert_eq!(probe.read_dir_count(), 2);++ let second = fs.tree_directory("/docs", false, None, None).await.unwrap();+ assert_eq!(second.len(), 3);+ assert_eq!(+ probe.read_dir_count(),+ 2,+ "second tree should reuse cached read_dir entries"+ );+}++#[tokio::test]+async fn cached_grep_traversal_reuses_directory_and_file_cache_after_warmup() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ backend.mkdir("/docs/sub", 0o755).await.unwrap();+ backend+ .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create)+ .await+ .unwrap();+ backend+ .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs_with_policy(+ backend,+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ );++ let first = fs+ .grep("/docs", "needle", true, false, None, None, None)+ .await+ .unwrap();+ assert_eq!(first.count, 2);+ assert_eq!(probe.grep_count(), 0);+ assert_eq!(probe.read_dir_count(), 2);+ assert_eq!(probe.read_count(), 2);+ assert_eq!(probe.stat_count(), 1);++ let second = fs+ .grep("/docs", "needle", true, false, None, None, None)+ .await+ .unwrap();+ assert_eq!(second.count, 2);+ assert_eq!(+ probe.read_dir_count(),+ 2,+ "second grep should reuse cached read_dir entries"+ );+ assert_eq!(+ probe.read_count(),+ 2,+ "second grep should reuse cached file contents"+ );+ assert_eq!(+ probe.stat_count(),+ 2,+ "second grep should only stat the query root, not every cached entry"+ );+}++#[tokio::test]+async fn cached_grep_batches_generation_validation_after_warmup() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ backend.mkdir("/docs/sub", 0o755).await.unwrap();+ backend+ .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create)+ .await+ .unwrap();+ backend+ .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create)+ .await+ .unwrap();+ let (fs, provider) = cached_fs_with_tracking_provider(+ backend,+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ );++ fs.grep("/docs", "needle", true, false, None, None, None)+ .await+ .unwrap();+ provider.reset_observed_reads();++ let result = fs+ .grep("/docs", "needle", true, false, None, None, None)+ .await+ .unwrap();++ assert_eq!(result.count, 2);+ assert!(+ provider.batch_get_count() > 0,+ "warm cached grep should batch generation validation reads"+ );+}++#[tokio::test]+async fn cached_grep_memoizes_generation_keys_within_one_traversal() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ backend.mkdir("/docs/sub", 0o755).await.unwrap();+ backend+ .write("/docs/a.md", b"needle\nplain", 0, WriteFlag::Create)+ .await+ .unwrap();+ backend+ .write("/docs/sub/b.md", b"other\nneedle", 0, WriteFlag::Create)+ .await+ .unwrap();+ let (fs, provider) = cached_fs_with_tracking_provider(+ backend,+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ );++ fs.grep("/docs", "needle", true, false, None, None, None)+ .await+ .unwrap();+ provider.reset_observed_reads();++ let result = fs+ .grep("/docs", "needle", true, false, None, None, None)+ .await+ .unwrap();++ assert_eq!(result.count, 2);+ let subtree_keys = provider+ .observed_read_keys()+ .into_iter()+ .filter(|key| key.contains(":subtree:"))+ .collect::<Vec<_>>();+ let unique = subtree_keys+ .iter()+ .collect::<std::collections::HashSet<_>>();+ assert_eq!(+ subtree_keys.len(),+ unique.len(),+ "one cached grep traversal should not re-read the same generation key"+ );+}++#[tokio::test]+async fn cached_grep_scans_cached_files_with_bounded_concurrency() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ for index in 0..8 {+ backend+ .write(+ &format!("/docs/{index}.md"),+ b"needle\nplain",+ 0,+ WriteFlag::Create,+ )+ .await+ .unwrap();+ }+ let provider = Arc::new(TrackingProvider::new().with_get_delay(Duration::from_millis(30)));+ let (fs, provider) = cached_fs_with_tracking_provider_instance(+ backend,+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ provider,+ );++ fs.grep("/docs", "needle", true, false, None, None, None)+ .await+ .unwrap();+ provider.reset_observed_reads();++ let result = fs+ .grep("/docs", "needle", true, false, None, None, None)+ .await+ .unwrap();++ assert_eq!(result.count, 8);+ let max_gets = provider.max_concurrent_gets();+ assert!(+ max_gets > 1,+ "warm cached grep should scan cached file reads concurrently"+ );+ assert!(+ max_gets <= 8,+ "cached grep file scan concurrency should stay bounded"+ );+}++#[tokio::test]+async fn cached_grep_traversal_matches_default_grep_semantics() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ backend.mkdir("/docs/sub", 0o755).await.unwrap();+ backend.mkdir("/docs/skip", 0o755).await.unwrap();+ backend+ .write("/docs/a.md", b"Needle\nneedle", 0, WriteFlag::Create)+ .await+ .unwrap();+ backend+ .write("/docs/sub/b.md", b"needle\nmiss", 0, WriteFlag::Create)+ .await+ .unwrap();+ backend+ .write("/docs/skip/c.md", b"needle", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let (fs, _) = cached_fs_with_policy(+ backend,+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ );++ let cached = fs+ .grep(+ "/docs",+ "needle",+ true,+ true,+ Some(2),+ Some("/docs/skip"),+ Some(1),+ )+ .await+ .unwrap();+ let direct_result = direct+ .grep(+ "/docs",+ "needle",+ true,+ true,+ Some(2),+ Some("/docs/skip"),+ Some(1),+ )+ .await+ .unwrap();++ let tuples = |result: &GrepResult| {+ result+ .matches+ .iter()+ .map(|m| (m.file.clone(), m.line, m.content.clone()))+ .collect::<Vec<_>>()+ };+ assert_eq!(tuples(&cached), tuples(&direct_result));+}++#[tokio::test]+async fn cached_grep_traversal_falls_back_for_multiwrite_backend() {+ let primary = CountingFileSystem::new();+ primary.mkdir("/docs", 0o755).await.unwrap();+ primary+ .write("/docs/a.md", b"needle", 0, WriteFlag::Create)+ .await+ .unwrap();+ let primary_probe = primary.clone();+ let multiwrite = MultiWriteWrappedFS::builder(Arc::new(primary))+ .build()+ .unwrap();+ let fs = CachedFileSystem::new(+ Box::new(multiwrite),+ Arc::new(MemoryCacheProvider::new()),+ CacheNamespace::new("grep-multiwrite"),+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ );++ let ctx = Arc::new(FsContextInner::new("acct"));+ let result = FS_CTX+ .scope(ctx, async {+ fs.grep("/docs", "needle", true, false, None, None, None)+ .await+ .unwrap()+ })+ .await;++ assert_eq!(result.count, 1);+ assert_eq!(+ primary_probe.grep_count(),+ 1,+ "multi-write backends must keep their backend grep path"+ );+ assert_eq!(+ primary_probe.read_dir_count(),+ 0,+ "cached traversal must not bypass multi-write grep semantics"+ );+}++#[tokio::test]+async fn cached_tree_traversal_matches_default_tree_semantics() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ backend.mkdir("/docs/sub", 0o755).await.unwrap();+ backend+ .write("/docs/a.md", b"a", 0, WriteFlag::Create)+ .await+ .unwrap();+ backend+ .write("/docs/.hidden.md", b"hidden", 0, WriteFlag::Create)+ .await+ .unwrap();+ backend+ .write("/docs/sub/b.md", b"b", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let (fs, _) = cached_fs_with_policy(+ backend,+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ );++ let cached = fs+ .tree_directory("/docs", false, None, Some(1))+ .await+ .unwrap();+ let direct_entries = direct+ .tree_directory("/docs", false, None, Some(1))+ .await+ .unwrap();+ assert_eq!(+ cached+ .iter()+ .map(|entry| entry.rel_path.as_str())+ .collect::<Vec<_>>(),+ direct_entries+ .iter()+ .map(|entry| entry.rel_path.as_str())+ .collect::<Vec<_>>()+ );++ let with_hidden = fs.tree_directory("/docs", true, None, None).await.unwrap();+ assert!(+ with_hidden+ .iter()+ .any(|entry| entry.rel_path == ".hidden.md"),+ "show_hidden=true should include hidden files"+ );+}++#[tokio::test]+async fn cached_tree_traversal_returns_oversized_directories_without_caching_them() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/large", 0o755).await.unwrap();+ for index in 0..4097 {+ backend+ .write(+ &format!("/large/{index:04}.txt"),+ b"x",+ 0,+ WriteFlag::Create,+ )+ .await+ .unwrap();+ }+ let probe = backend.clone();+ let (fs, _) = cached_fs_with_policy(+ backend,+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ );++ assert_eq!(+ fs.tree_directory("/large", false, None, None)+ .await+ .unwrap()+ .len(),+ 4097+ );+ assert_eq!(+ fs.tree_directory("/large", false, None, None)+ .await+ .unwrap()+ .len(),+ 4097+ );+ assert_eq!(+ probe.read_dir_count(),+ 2,+ "oversized directories should not be cached by tree traversal"+ );+}++#[tokio::test]+async fn cached_tree_traversal_falls_back_when_provider_is_unavailable() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ backend+ .write("/docs/a.md", b"a", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let fs = CachedFileSystem::new(+ Box::new(backend),+ Arc::new(UnavailableProvider),+ CacheNamespace::new("tree-unavailable"),+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ );++ assert_eq!(+ fs.tree_directory("/docs", false, None, None)+ .await+ .unwrap()+ .len(),+ 1+ );+ assert_eq!(+ fs.tree_directory("/docs", false, None, None)+ .await+ .unwrap()+ .len(),+ 1+ );+ assert_eq!(probe.read_dir_count(), 2);+ assert!(fs.metrics().snapshot().errors >= 1);+}++#[tokio::test]+async fn cached_tree_traversal_falls_back_for_multiwrite_backend() {+ let primary = CountingFileSystem::new();+ primary.mkdir("/docs", 0o755).await.unwrap();+ primary+ .write("/docs/a.md", b"a", 0, WriteFlag::Create)+ .await+ .unwrap();+ let primary_probe = primary.clone();+ let multiwrite = MultiWriteWrappedFS::builder(Arc::new(primary))+ .build()+ .unwrap();+ let fs = CachedFileSystem::new(+ Box::new(multiwrite),+ Arc::new(MemoryCacheProvider::new()),+ CacheNamespace::new("tree-multiwrite"),+ CachePolicy::default().with_traversal_mode(CacheTraversalMode::CachedTraversal),+ );++ let ctx = Arc::new(FsContextInner::new("acct"));+ let entries = FS_CTX+ .scope(ctx, async {+ fs.tree_directory("/docs", false, None, None).await.unwrap()+ })+ .await;++ assert_eq!(entries.len(), 1);+ assert_eq!(+ primary_probe.tree_count(),+ 1,+ "multi-write backends must keep their backend tree_directory path"+ );+ assert_eq!(+ primary_probe.read_dir_count(),+ 0,+ "cached traversal must not bypass multi-write tree semantics"+ );+}++#[test]+fn cache_policy_supports_explicit_permission_sensitive_prefixes() {+ let policy = CachePolicy::default().with_bypass_prefix("/private");++ assert!(!policy.cache_file("/private/secret.md", 10));+ assert!(!policy.cache_directory("/private/docs"));+ assert!(policy.cache_file("/public/.overview.md", 10));+}++#[test]+fn cache_policy_marks_high_value_objects_as_preferred() {+ let policy = CachePolicy::default();++ assert_eq!(+ policy.file_decision("/docs/.abstract.md", 128),+ CacheDecision::Prefer+ );+ assert_eq!(+ policy.file_decision("/docs/.overview.md", 128),+ CacheDecision::Prefer+ );+ assert_eq!(+ policy.file_decision("/docs/note.md", 128),+ CacheDecision::Cache+ );+ assert_eq!(policy.directory_decision("/docs"), CacheDecision::Prefer);+}++#[tokio::test]+async fn memory_provider_satisfies_the_common_contract() {+ let provider = MemoryMockProvider::new();++ provider.put("one", Bytes::from_static(b"1")).await.unwrap();+ assert!(provider.exists("one").await.unwrap());+ assert_eq!(+ provider.get("one").await.unwrap(),+ Some(Bytes::from_static(b"1"))+ );++ provider+ .batch_put(vec![+ ("two".to_string(), Bytes::from_static(b"2")),+ ("three".to_string(), Bytes::from_static(b"3")),+ ])+ .await+ .unwrap();+ assert_eq!(+ provider+ .batch_get(&["one".to_string(), "missing".to_string()])+ .await+ .unwrap(),+ vec![Some(Bytes::from_static(b"1")), None]+ );++ provider+ .invalidate(&["one".to_string(), "two".to_string()])+ .await+ .unwrap();+ assert!(!provider.exists("one").await.unwrap());+ assert!(!provider.exists("two").await.unwrap());+ provider.flush().await.unwrap();+ assert!(!provider.exists("three").await.unwrap());+ provider.close().await.unwrap();+ assert!(provider.get("three").await.is_err());+}++#[tokio::test]+async fn full_file_reads_are_read_through_cached_but_range_reads_bypass() {+ let backend = CountingFileSystem::new();+ backend+ .write("/note.md", b"hello world", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ assert_eq!(fs.read("/note.md", 0, 0).await.unwrap(), b"hello world");+ assert_eq!(fs.read("/note.md", 0, 0).await.unwrap(), b"hello world");+ assert_eq!(probe.read_count(), 1);++ assert_eq!(fs.read("/note.md", 6, 5).await.unwrap(), b"world");+ assert_eq!(probe.read_count(), 2);++ let metrics = fs.metrics().snapshot();+ assert_eq!(metrics.file_hits, 1);+ assert_eq!(metrics.file_misses, 1);+ assert_eq!(metrics.backend_fallbacks, 1);+}++#[tokio::test]+async fn read_dir_is_cached_and_parent_changes_invalidate_it() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/docs", 0o755).await.unwrap();+ backend+ .write("/docs/one.md", b"one", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ assert_eq!(fs.read_dir("/docs").await.unwrap().len(), 1);+ assert_eq!(fs.read_dir("/docs").await.unwrap().len(), 1);+ assert_eq!(probe.read_dir_count(), 1);++ fs.write("/docs/two.md", b"two", 0, WriteFlag::Create)+ .await+ .unwrap();+ let entries = fs.read_dir("/docs").await.unwrap();+ assert_eq!(entries.len(), 2);+ assert_eq!(probe.read_dir_count(), 2);++ let metrics = fs.metrics().snapshot();+ assert_eq!(metrics.read_dir_hits, 1);+ assert_eq!(metrics.read_dir_misses, 2);+ assert!(metrics.invalidations >= 1);+}++#[tokio::test]+async fn oversized_directories_bypass_directory_cache() {+ let cacheable = CountingFileSystem::new();+ cacheable.mkdir("/small", 0o755).await.unwrap();+ for index in 0..4096 {+ cacheable+ .write(+ &format!("/small/{index:04}.txt"),+ b"x",+ 0,+ WriteFlag::Create,+ )+ .await+ .unwrap();+ }+ let cacheable_probe = cacheable.clone();+ let (cacheable_fs, _) = cached_fs(cacheable);++ assert_eq!(cacheable_fs.read_dir("/small").await.unwrap().len(), 4096);+ assert_eq!(cacheable_fs.read_dir("/small").await.unwrap().len(), 4096);+ assert_eq!(cacheable_probe.read_dir_count(), 1);++ let oversized = CountingFileSystem::new();+ oversized.mkdir("/large", 0o755).await.unwrap();+ for index in 0..4097 {+ oversized+ .write(+ &format!("/large/{index:04}.txt"),+ b"x",+ 0,+ WriteFlag::Create,+ )+ .await+ .unwrap();+ }+ let oversized_probe = oversized.clone();+ let (oversized_fs, _) = cached_fs(oversized);++ assert_eq!(oversized_fs.read_dir("/large").await.unwrap().len(), 4097);+ assert_eq!(oversized_fs.read_dir("/large").await.unwrap().len(), 4097);+ assert_eq!(oversized_probe.read_dir_count(), 2);+}++#[tokio::test]+async fn all_directory_membership_mutations_invalidate_parent_entries() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/root", 0o755).await.unwrap();+ backend.mkdir("/root/old", 0o755).await.unwrap();+ backend+ .write("/root/file.txt", b"file", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);+ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);++ fs.mkdir("/root/new", 0o755).await.unwrap();+ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 3);++ fs.rename("/root/old", "/root/moved").await.unwrap();+ let names: Vec<String> = fs+ .read_dir("/root")+ .await+ .unwrap()+ .into_iter()+ .map(|entry| entry.name)+ .collect();+ assert!(names.contains(&"moved".to_string()));+ assert!(!names.contains(&"old".to_string()));++ fs.remove("/root/file.txt").await.unwrap();+ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 2);++ fs.remove_all("/root/new").await.unwrap();+ assert_eq!(fs.read_dir("/root").await.unwrap().len(), 1);+ assert_eq!(probe.read_dir_count(), 5);+}++#[tokio::test]+async fn writes_and_deletes_never_leave_stale_file_cache_entries() {+ let backend = CountingFileSystem::new();+ backend+ .write("/value.txt", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"old");+ fs.write("/value.txt", b"new", 0, WriteFlag::Truncate)+ .await+ .unwrap();+ assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"new");++ fs.remove("/value.txt").await.unwrap();+ assert!(fs.read("/value.txt", 0, 0).await.is_err());+ assert_eq!(probe.read_count(), 2);+}++#[tokio::test]+async fn full_writes_populate_cache_before_returning() {+ let backend = CountingFileSystem::new();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ fs.write("/fresh.md", b"fresh", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(fs.read("/fresh.md", 0, 0).await.unwrap(), b"fresh");+ assert_eq!(probe.read_count(), 0);+}++#[tokio::test]+async fn partial_writes_and_truncate_invalidate_cached_file_contents() {+ let backend = CountingFileSystem::new();+ backend+ .write("/partial.txt", b"hello", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hello");+ fs.write("/partial.txt", b"X", 1, WriteFlag::None)+ .await+ .unwrap();+ assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hXllo");++ fs.truncate("/partial.txt", 2).await.unwrap();+ assert_eq!(fs.read("/partial.txt", 0, 0).await.unwrap(), b"hX");+ assert_eq!(probe.read_count(), 3);+}++#[tokio::test]+async fn file_rename_invalidates_old_and_new_paths() {+ let backend = CountingFileSystem::new();+ backend+ .write("/old.txt", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ backend+ .write("/new.txt", b"historical", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let (fs, _) = cached_fs(backend);++ assert_eq!(fs.read("/old.txt", 0, 0).await.unwrap(), b"old");+ assert_eq!(fs.read("/new.txt", 0, 0).await.unwrap(), b"historical");+ direct.remove("/new.txt").await.unwrap();++ fs.rename("/old.txt", "/new.txt").await.unwrap();++ assert!(fs.read("/old.txt", 0, 0).await.is_err());+ assert_eq!(fs.read("/new.txt", 0, 0).await.unwrap(), b"old");+}++#[tokio::test]+async fn remove_all_generation_rejects_residual_descendant_cache_entries() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/tree", 0o755).await.unwrap();+ backend+ .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ assert_eq!(fs.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old");+ fs.remove_all("/tree").await.unwrap();++ direct.mkdir("/tree", 0o755).await.unwrap();+ direct+ .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create)+ .await+ .unwrap();++ assert_eq!(fs.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"new");+ assert_eq!(probe.read_count(), 2);+}++#[tokio::test]+async fn shared_provider_generation_bump_invalidates_other_wrappers() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/tree", 0o755).await.unwrap();+ backend+ .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let probe = backend.clone();+ let provider = Arc::new(MemoryCacheProvider::new());++ let first = CachedFileSystem::new(+ Box::new(backend.clone()),+ provider.clone(),+ CacheNamespace::new("shared"),+ CachePolicy::default(),+ );+ let second = CachedFileSystem::new(+ Box::new(backend),+ provider,+ CacheNamespace::new("shared"),+ CachePolicy::default(),+ );++ assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old");+ second.remove_all("/tree").await.unwrap();++ direct.mkdir("/tree", 0o755).await.unwrap();+ direct+ .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create)+ .await+ .unwrap();++ assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"new");+ assert_eq!(probe.read_count(), 2);+}++#[tokio::test]+async fn provider_generation_eviction_after_restart_cannot_revive_old_descendants() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/tree", 0o755).await.unwrap();+ backend+ .write("/tree/leaf.txt", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let probe = backend.clone();+ let provider = Arc::new(MemoryCacheProvider::new());++ let first = CachedFileSystem::new(+ Box::new(backend.clone()),+ provider.clone(),+ CacheNamespace::new("restart"),+ CachePolicy::default(),+ );+ assert_eq!(first.read("/tree/leaf.txt", 0, 0).await.unwrap(), b"old");+ first.remove_all("/tree").await.unwrap();++ direct.mkdir("/tree", 0o755).await.unwrap();+ direct+ .write("/tree/leaf.txt", b"new", 0, WriteFlag::Create)+ .await+ .unwrap();++ for key in provider.keys().await {+ if key.contains(":subtree:") {+ provider.delete(&key).await.unwrap();+ }+ }+ drop(first);++ let restarted = CachedFileSystem::new(+ Box::new(backend),+ provider,+ CacheNamespace::new("restart"),+ CachePolicy::default(),+ );+ assert_eq!(+ restarted.read("/tree/leaf.txt", 0, 0).await.unwrap(),+ b"new"+ );+ assert_eq!(probe.read_count(), 2);+}++#[tokio::test]+async fn directory_rename_invalidates_old_and_historical_new_subtrees() {+ let backend = CountingFileSystem::new();+ backend.mkdir("/old", 0o755).await.unwrap();+ backend+ .write("/old/leaf.txt", b"moved", 0, WriteFlag::Create)+ .await+ .unwrap();+ let direct = backend.clone();+ let (fs, _) = cached_fs(backend);++ assert_eq!(fs.read("/old/leaf.txt", 0, 0).await.unwrap(), b"moved");+ fs.rename("/old", "/new").await.unwrap();+ assert!(fs.read("/old/leaf.txt", 0, 0).await.is_err());+ assert_eq!(fs.read("/new/leaf.txt", 0, 0).await.unwrap(), b"moved");++ fs.remove_all("/new").await.unwrap();+ direct.mkdir("/new", 0o755).await.unwrap();+ direct+ .write("/new/leaf.txt", b"replacement", 0, WriteFlag::Create)+ .await+ .unwrap();+ assert_eq!(+ fs.read("/new/leaf.txt", 0, 0).await.unwrap(),+ b"replacement"+ );+}++#[tokio::test]+async fn concurrent_misses_share_one_backend_read() {+ let backend = CountingFileSystem::new().with_read_delay(Duration::from_millis(30));+ backend+ .write("/hot.md", b"hot", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ let mut tasks = Vec::new();+ for _ in 0..12 {+ let fs = fs.clone();+ tasks.push(tokio::spawn(async move {+ fs.read("/hot.md", 0, 0).await.unwrap()+ }));+ }+ for task in tasks {+ assert_eq!(task.await.unwrap(), b"hot");+ }++ assert_eq!(probe.read_count(), 1);+ let metrics = fs.metrics().snapshot();+ assert_eq!(metrics.inflight_leaders, 1);+ assert_eq!(metrics.inflight_followers, 11);+ assert_eq!(metrics.inflight_backend_saved, 11);+}++#[tokio::test]+async fn cache_policy_bypasses_lock_and_control_files() {+ let backend = CountingFileSystem::new();+ backend+ .write("/state.lock", b"first", 0, WriteFlag::Create)+ .await+ .unwrap();+ backend.mkdir("/queue", 0o755).await.unwrap();+ backend+ .write("/queue/peek", b"live", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let (fs, _) = cached_fs(backend);++ fs.read("/state.lock", 0, 0).await.unwrap();+ fs.read("/state.lock", 0, 0).await.unwrap();+ fs.read("/queue/peek", 0, 0).await.unwrap();+ fs.read("/queue/peek", 0, 0).await.unwrap();++ assert_eq!(probe.read_count(), 4);+ assert_eq!(fs.metrics().snapshot().policy_bypasses, 4);+}++#[tokio::test]+async fn failed_invalidation_bypasses_cache_instead_of_serving_stale_data() {+ let backend = CountingFileSystem::new();+ backend+ .write("/value.txt", b"old", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let fs = CachedFileSystem::new(+ Box::new(backend),+ Arc::new(DeleteFailingProvider::new()),+ CacheNamespace::new("delete-failure"),+ CachePolicy::default(),+ );++ assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"old");+ fs.write("/value.txt", b"new", 0, WriteFlag::Truncate)+ .await+ .unwrap();+ assert_eq!(fs.read("/value.txt", 0, 0).await.unwrap(), b"new");+ assert_eq!(probe.read_count(), 2);++ let metrics = fs.metrics().snapshot();+ assert!(metrics.errors >= 1);+ assert!(metrics.policy_bypasses >= 1);+}++#[tokio::test]+async fn unavailable_provider_falls_back_to_backend_and_enters_bypass() {+ let backend = CountingFileSystem::new();+ backend+ .write("/available.txt", b"backend", 0, WriteFlag::Create)+ .await+ .unwrap();+ let probe = backend.clone();+ let fs = CachedFileSystem::new(+ Box::new(backend),+ Arc::new(UnavailableProvider),+ CacheNamespace::new("unavailable"),+ CachePolicy::default(),+ );++ assert_eq!(fs.read("/available.txt", 0, 0).await.unwrap(), b"backend");+ assert_eq!(fs.read("/available.txt", 0, 0).await.unwrap(), b"backend");+ assert_eq!(probe.read_count(), 2);++ let metrics = fs.metrics().snapshot();+ assert!(metrics.errors >= 2);+ assert!(metrics.policy_bypasses >= 1);+}++#[tokio::test]+async fn metrics_cover_operations_bytes_latency_and_errors() {+ let backend = CountingFileSystem::new();+ backend+ .write("/metrics.txt", b"metrics", 0, WriteFlag::Create)+ .await+ .unwrap();+ let (fs, _) = cached_fs(backend);++ assert_eq!(fs.read("/metrics.txt", 0, 0).await.unwrap(), b"metrics");+ assert_eq!(fs.read("/metrics.txt", 0, 0).await.unwrap(), b"metrics");+ fs.write("/metrics.txt", b"updated", 0, WriteFlag::Truncate)+ .await+ .unwrap();++ let metrics = fs.metrics().snapshot();+ assert_eq!(metrics.file_hits, 1);+ assert_eq!(metrics.file_misses, 1);+ assert_eq!(metrics.backend_fallbacks, 1);+ assert!(metrics.puts >= 1);+ assert!(metrics.deletes >= 1);+ assert!(metrics.invalidations >= 1);+ assert_eq!(metrics.backend_bytes, 7);+ assert_eq!(metrics.cache_bytes, 7);+ assert!(metrics.get_latency_ns > 0);+ assert!(metrics.put_latency_ns > 0);+ assert!(metrics.delete_latency_ns > 0);+ assert_eq!(metrics.errors, 0);+}diff --git a/docker/mooncake-test/Dockerfile b/docker/mooncake-test/Dockerfilenew file mode 100644index 000000000..9ffa8b857--- /dev/null+++ b/docker/mooncake-test/Dockerfile@@ -0,0 +1,92 @@+FROM public.ecr.aws/ubuntu/ubuntu:22.04++ARG DEBIAN_FRONTEND=noninteractive+ARG MOONCAKE_COMMIT=1352bbec43081e461356aaecf6c70cddd826b455+ARG RUST_TOOLCHAIN=1.91.1++ENV PATH=/root/.cargo/bin:${PATH} \+ CARGO_NET_GIT_FETCH_WITH_CLI=true++RUN apt-get update && apt-get install -y --no-install-recommends \+ build-essential \+ ca-certificates \+ clang \+ cmake \+ curl \+ git \+ libasio-dev \+ libboost-all-dev \+ libcurl4-openssl-dev \+ libgflags-dev \+ libgoogle-glog-dev \+ libgrpc++-dev \+ libgrpc-dev \+ libgtest-dev \+ libhiredis-dev \+ libibverbs-dev \+ libjemalloc-dev \+ libjsoncpp-dev \+ libmsgpack-dev \+ libnuma-dev \+ libprotobuf-dev \+ libpython3-dev \+ libssl-dev \+ libunwind-dev \+ liburing-dev \+ libxxhash-dev \+ libyaml-cpp-dev \+ libzstd-dev \+ ninja-build \+ pkg-config \+ protobuf-compiler-grpc \+ python3 \+ python3-pip \+ && rm -rf /var/lib/apt/lists/*++RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \+ | sh -s -- -y --profile minimal --default-toolchain "${RUST_TOOLCHAIN}"++RUN git clone --filter=blob:none https://github.com/kvcache-ai/Mooncake.git /opt/Mooncake \+ && cd /opt/Mooncake \+ && git checkout "${MOONCAKE_COMMIT}" \+ && git submodule update --init --recursive++RUN cmake -S /opt/Mooncake/extern/yalantinglibs \+ -B /opt/Mooncake/extern/yalantinglibs/build \+ -G Ninja \+ -DBUILD_EXAMPLES=OFF \+ -DBUILD_BENCHMARK=OFF \+ -DBUILD_UNIT_TESTS=OFF \+ && cmake --build /opt/Mooncake/extern/yalantinglibs/build --parallel 2 \+ && cmake --install /opt/Mooncake/extern/yalantinglibs/build++RUN cmake -S /opt/Mooncake -B /opt/Mooncake/build -G Ninja \+ -DWITH_STORE=ON \+ -DWITH_STORE_RUST=ON \+ -DWITH_P2P_STORE=OFF \+ -DUSE_TCP=ON \+ -DUSE_HTTP=ON \+ -DUSE_ETCD=OFF \+ -DSTORE_USE_ETCD=OFF \+ -DUSE_REDIS=OFF \+ -DSTORE_USE_REDIS=OFF \+ -DUSE_CUDA=OFF \+ -DUSE_MNNVL=OFF \+ -DUSE_UB=OFF \+ -DUSE_CXL=OFF \+ -DBUILD_UNIT_TESTS=OFF \+ -DBUILD_EXAMPLES=OFF \+ -DBUILD_BENCHMARK=OFF \+ -DWITH_METRICS=OFF \+ -DCMAKE_BUILD_TYPE=Release \+ && cmake --build /opt/Mooncake/build \+ --target build_mooncake_store_rust mooncake_master \+ --parallel 2++RUN python3 -m pip install --no-cache-dir aiohttp++COPY run-smoke.sh /usr/local/bin/run-openviking-mooncake-smoke+RUN chmod +x /usr/local/bin/run-openviking-mooncake-smoke++WORKDIR /workspace/OpenViking+ENTRYPOINT ["/usr/local/bin/run-openviking-mooncake-smoke"]diff --git a/docker/mooncake-test/run-smoke.sh b/docker/mooncake-test/run-smoke.shnew file mode 100644index 000000000..656a1f2cf--- /dev/null+++ b/docker/mooncake-test/run-smoke.sh@@ -0,0 +1,81 @@+#!/usr/bin/env bash+set -euo pipefail++MOONCAKE_ROOT=/opt/Mooncake+MOONCAKE_BUILD_DIR=${MOONCAKE_ROOT}/build++export MOONCAKE_BUILD_DIR+export MOONCAKE_STORE_LIB_DIR=${MOONCAKE_BUILD_DIR}/mooncake-store/src+export MOONCAKE_STORE_INCLUDE_DIR=${MOONCAKE_ROOT}/mooncake-store/include+export LD_LIBRARY_PATH="${MOONCAKE_BUILD_DIR}/mooncake-asio:\+${MOONCAKE_BUILD_DIR}/mooncake-common:\+${MOONCAKE_BUILD_DIR}/mooncake-common/src:\+${MOONCAKE_BUILD_DIR}/mooncake-common/etcd:\+${MOONCAKE_BUILD_DIR}/mooncake-store/src:\+${MOONCAKE_BUILD_DIR}/mooncake-store/src/cachelib_memory_allocator:\+${MOONCAKE_BUILD_DIR}/mooncake-transfer-engine/src:\+${MOONCAKE_BUILD_DIR}/mooncake-transfer-engine/src/common/base:\+${LD_LIBRARY_PATH:-}"++cleanup() {+ local exit_code=$?+ if [[ -n "${MASTER_PID:-}" ]]; then+ kill "${MASTER_PID}" 2>/dev/null || true+ fi+ if [[ -n "${METADATA_PID:-}" ]]; then+ kill "${METADATA_PID}" 2>/dev/null || true+ fi+ if [[ ${exit_code} -ne 0 ]]; then+ echo "=== Mooncake metadata log ==="+ cat /tmp/mooncake-metadata.log 2>/dev/null || true+ echo "=== Mooncake master log ==="+ cat /tmp/mooncake-master.log 2>/dev/null || true+ fi+ exit "${exit_code}"+}+trap cleanup EXIT++cd "${MOONCAKE_ROOT}/mooncake-transfer-engine/example/http-metadata-server-python"+python3 bootstrap_server.py >/tmp/mooncake-metadata.log 2>&1 &+METADATA_PID=$!++"${MOONCAKE_BUILD_DIR}/mooncake-store/src/mooncake_master" \+ --eviction_high_watermark_ratio=0.95 \+ --cluster_id=openviking_smoke \+ --port=50051 \+ >/tmp/mooncake-master.log 2>&1 &+MASTER_PID=$!++for _ in $(seq 1 60); do+ if curl -fsS http://127.0.0.1:8080/health >/dev/null 2>&1 \+ && kill -0 "${MASTER_PID}" 2>/dev/null; then+ break+ fi+ sleep 1+done++if ! kill -0 "${METADATA_PID}" 2>/dev/null || ! kill -0 "${MASTER_PID}" 2>/dev/null; then+ echo "Mooncake services failed to start" >&2+ exit 1+fi++echo "=== Official Mooncake Rust smoke ==="+cd "${MOONCAKE_ROOT}/mooncake-store/rust"+MC_RUST_STORE_RUN_INTEGRATION=true \+MC_METADATA_SERVER=http://127.0.0.1:8080/metadata \+MC_RUST_STORE_MASTER_ADDR=127.0.0.1:50051 \+MC_RUST_STORE_LOCAL_HOSTNAME=127.0.0.1 \+MC_RUST_STORE_PROTOCOL=tcp \+MC_RUST_STORE_DEVICE_NAME= \+cargo test --test minimal_smoke -- --nocapture++echo "=== OpenViking MooncakeProvider smoke ==="+cd /workspace/OpenViking+OPENVIKING_RUN_MOONCAKE_INTEGRATION=true \+MOONCAKE_LOCAL_HOSTNAME=127.0.0.1 \+MOONCAKE_METADATA_SERVER=http://127.0.0.1:8080/metadata \+MOONCAKE_MASTER_SERVER_ADDR=127.0.0.1:50051 \+MOONCAKE_PROTOCOL=tcp \+cargo test --locked -p ragfs-cache-mooncake \+ --features mooncake-native \+ --test native_smoke -- --nocapturediff --git a/docs/en/guides/13-ragfs-cache.md b/docs/en/guides/13-ragfs-cache.mdnew file mode 100644index 000000000..6ce0795bd--- /dev/null+++ b/docs/en/guides/13-ragfs-cache.md@@ -0,0 +1,375 @@+# RAGFS Cache++RAGFS cache is an optional read-cache layer for OpenViking. It speeds up full file reads and directory reads. It is only an acceleration layer, not the source of truth; backend filesystem data remains authoritative.++Assumptions:++- Only one OpenViking / RAGFS process writes to the same namespace.+- File and directory changes go through RAGFS.+- The backend is not modified externally by bypassing RAGFS.+- After a cache Provider successfully writes or deletes one key, later reads of that key do not return the old value.++## Quick Start++For first-time setup, complete the base configuration first:++```bash+openviking-server init+openviking-server doctor+```++Then enable the cache under `storage.agfs.cache` in `~/.openviking/ov.conf`. The following Redis example is a good quick validation setup:++```json+{+ "storage": {+ "workspace": "./data",+ "agfs": {+ "backend": "local",+ "cache": {+ "enabled": true,+ "provider": "redis",+ "namespace": "openviking",+ "max_file_size_bytes": 1048576,+ "bypass_prefixes": ["/queue", "/tmp"],+ "redis": {+ "mode": "standalone",+ "endpoints": ["redis://127.0.0.1:6379"],+ "pool_size": 32,+ "connect_timeout_ms": 1000,+ "command_timeout_ms": 20,+ "key_prefix": "ragfs-cache",+ "default_ttl_seconds": 3600,+ "read_from_replica": false+ }+ }+ }+ }+}+```++Start Redis and OpenViking:++```bash+redis-server+openviking-server --config ~/.openviking/ov.conf+```++If the configuration file is at the default path `~/.openviking/ov.conf`, you can also run:++```bash+openviking-server+```++Available Providers:++| Provider | Best for | Notes |+|----------|----------|-------|+| `memory` | Local validation and tests | In-process cache; lost after restart |+| `redis` | Fast rollout on standard networks | Currently supports standalone; read from primary only |+| `yuanrong` | Near-compute cache, shared memory, or heterogeneous multi-tier cache | Requires Yuanrong worker and native feature |+| `mooncake` | Remote memory pool, RDMA/TCP data plane | Requires Mooncake services and native feature |++If the runtime package was not compiled with the selected Provider, startup returns an error similar to "requires the ... feature".++## Native Provider Builds++The standard OpenViking wheel is suitable for the `memory` and `redis`+Providers. The `yuanrong` and `mooncake` Providers depend on platform-specific+native SDKs and must be built for the target deployment environment.++Install the wheel builder first:++```bash+python -m pip install "maturin[patchelf]"+```++### Yuanrong++Install the Yuanrong DataSystem C++ SDK and export its header and library+locations:++```bash+export YUANRONG_SDK_INCLUDE=/path/to/yuanrong/include+export YUANRONG_SDK_LIB_DIR=/path/to/yuanrong/lib+# Optional; defaults to "datasystem".+export YUANRONG_SDK_LIB_NAME=datasystem+export LD_LIBRARY_PATH="$YUANRONG_SDK_LIB_DIR:${LD_LIBRARY_PATH:-}"+```++Build and install the wheel:++```bash+maturin build --release \+ --manifest-path crates/ragfs-python-native/Cargo.toml \+ --features yuanrong-native++python -m pip install --force-reinstall target/wheels/ragfs_python-*.whl+```++The Yuanrong worker configured by `storage.agfs.cache.yuanrong` must be+available when OpenViking starts.++### Mooncake++Check out the Mooncake revision used by+`crates/ragfs-cache-mooncake/Cargo.toml`, then build Mooncake Store with Rust+support:++```bash+cmake -S /path/to/Mooncake -B /path/to/Mooncake/build \+ -DWITH_STORE=ON \+ -DWITH_STORE_RUST=ON \+ -DCMAKE_BUILD_TYPE=Release++cmake --build /path/to/Mooncake/build \+ --target build_mooncake_store_rust mooncake_master -j+```++Export the paths required by the official Mooncake Rust binding:++```bash+export MOONCAKE_BUILD_DIR=/path/to/Mooncake/build+export MOONCAKE_STORE_LIB_DIR="$MOONCAKE_BUILD_DIR/mooncake-store/src"+export MOONCAKE_STORE_INCLUDE_DIR=/path/to/Mooncake/mooncake-store/include+export LD_LIBRARY_PATH="$MOONCAKE_BUILD_DIR/mooncake-common:\+$MOONCAKE_BUILD_DIR/mooncake-common/src:\+$MOONCAKE_BUILD_DIR/mooncake-store/src:\+$MOONCAKE_BUILD_DIR/mooncake-store/src/cachelib_memory_allocator:\+$MOONCAKE_BUILD_DIR/mooncake-transfer-engine/src:\+$MOONCAKE_BUILD_DIR/mooncake-transfer-engine/src/common/base:\+${LD_LIBRARY_PATH:-}"+```++Build and install the wheel:++```bash+maturin build --release \+ --manifest-path crates/ragfs-python-native/Cargo.toml \+ --features mooncake-native++python -m pip install --force-reinstall target/wheels/ragfs_python-*.whl+```++The Mooncake metadata service and Master configured by+`storage.agfs.cache.mooncake` must be available when OpenViking starts.+Native wheels are platform-specific and should be built on a system compatible+with the target deployment environment.++For production wheels, use a Mooncake revision whose Rust `build.rs` links+`libasan` only when ASan is explicitly enabled. Verify that the release wheel+does not contain or depend on `libasan`:++```bash+rm -rf /tmp/ragfs-python-wheel+python -m zipfile -e target/wheels/ragfs_python-*.whl /tmp/ragfs-python-wheel+readelf -d /tmp/ragfs-python-wheel/ragfs_python/ragfs_python.abi3.so \+ | grep libasan+find /tmp/ragfs-python-wheel -name 'libasan*'+```++Both checks should produce no output.++## Configuration++`storage.agfs.cache` supports these common options:++| Option | Type | Default | Description |+|--------|------|---------|-------------|+| `enabled` | bool | `false` | Enable the RAGFS cache |+| `provider` | str | `"memory"` | `memory`, `redis`, `yuanrong`, or `mooncake` |+| `namespace` | str | `"openviking"` | Cache namespace for isolating deployments or tenants |+| `max_file_size_bytes` | int | `1048576` | Maximum full-file object size admitted to cache |+| `bypass_prefixes` | list[str] | `[]` | Path prefixes that always bypass cache |++Redis configuration:++| Option | Default | Description |+|--------|---------|-------------|+| `mode` | `"standalone"` | Redis deployment mode |+| `endpoints` | `["redis://127.0.0.1:6379"]` | Redis connection URLs |+| `username` | `""` | Redis ACL username |+| `password_env` | `""` | Environment variable that stores the Redis password |+| `pool_size` | `32` | Command concurrency |+| `connect_timeout_ms` | `1000` | Connection timeout |+| `command_timeout_ms` | `20` | Command timeout |+| `key_prefix` | `"ragfs-cache"` | Redis-side key prefix |+| `default_ttl_seconds` | `3600` | Default TTL; `0` means no TTL |+| `read_from_replica` | `false` | Must be `false` in standalone mode |++Yuanrong configuration:++```json+{+ "storage": {+ "agfs": {+ "cache": {+ "enabled": true,+ "provider": "yuanrong",+ "yuanrong": {+ "host": "127.0.0.1",+ "port": 31501,+ "connect_timeout_ms": 5000,+ "request_timeout_ms": 5000,+ "sdk_concurrency": 4+ }+ }+ }+ }+}+```++Mooncake configuration:++```json+{+ "storage": {+ "agfs": {+ "cache": {+ "enabled": true,+ "provider": "mooncake",+ "mooncake": {+ "local_hostname": "127.0.0.1",+ "metadata_server": "http://127.0.0.1:8080/metadata",+ "master_server_addr": "127.0.0.1:50051",+ "protocol": "tcp",+ "device_name": "",+ "global_segment_size": 536870912,+ "local_buffer_size": 134217728,+ "replica_num": 2,+ "sdk_concurrency": 4,+ "operation_timeout_ms": 5000+ }+ }+ }+ }+}+```++## Architecture++RAGFS splits caching into two layers:++- `CachedFileSystem`: implements filesystem semantics, including cache hit/miss handling, backend fallback, cache fill, invalidation, generation checks, and metrics.+- `CacheProvider`: only stores cache objects through `get`, `put`, `delete`, batch reads/writes, and close operations.++Call flow:++```text+OpenViking+ -> RAGFS / MountableFS+ -> CachedFileSystem+ |-> CacheProvider -> Memory / Redis / Yuanrong / Mooncake+ `-> Backend FileSystem+```++With this boundary, file, directory, rename, recursive delete, and write-after-invalidation logic live only in the common layer. A Provider does not need to understand path semantics; it only needs to store stable key-value objects.++## Cache Objects++RAGFS mainly caches three object types.++### File Cache++File keys use a stable namespace and path hash:++```text+ragfs:v1:{namespace}:file:{hash(path)}+```++The file value is a `CacheEnvelope` containing file content, object kind, path, and generation snapshots. After a full-read cache hit, RAGFS validates the envelope and generation before returning the content.++The default policy prefers summary files such as `.abstract.md` and `.overview.md`. Files larger than `max_file_size_bytes` are not admitted to cache. Non-full range reads also bypass the cache.++### Directory Cache++Directory key:++```text+ragfs:v1:{namespace}:dir:{hash(path)}+```++The directory cache stores raw backend `read_dir` entries, not permission-filtered final results. Permission, role, and agent-context filtering still happens in the OpenViking upper layer at request time.++This lets one directory cache object serve `ls`, `tree`, `glob`, the file-collection phase of `grep`, and path collection before delete or move operations.++### Subtree Generation++Subtree generation key:++```text+ragfs:v1:{namespace}:subtree:{hash(scope)}+```++`remove_all` and directory `rename` can leave descendant keys behind in the Provider. RAGFS bumps the subtree generation so old envelopes fail their generation snapshot check. Later real reads fall back to the backend and rebuild the cache.++## Consistency and Invalidation++In the single-writer scenario, RAGFS does not need a distributed write lock. The important part is maintaining three invalidation classes according to filesystem semantics:++- File changes: delete or update `file_key(path)` and delete `dir_key(parent)`.+- Directory changes: delete the directory's own `dir_key` and the parent directory's `dir_key`.+- Subtree changes: bump `subtree` generation for recursive delete and directory rename.++Typical write order:++```text+Acquire the in-process operation lock+-> Apply backend change+-> Update or delete related cache keys+-> Bump subtree generation when needed+-> Return result+```++If a Provider fails, RAGFS treats the backend as authoritative and puts the affected path into short-term bypass, avoiding reads from potentially stale cache.++## Request Coalescing++When multiple requests read the same uncached small file or directory at the same time, `CachedFileSystem` uses an in-process inflight table to coalesce them:++```text+The first miss becomes the leader and performs backend fallback and cache fill.+Later requests for the same key become followers and wait for the leader result.+The inflight entry is removed after the request completes.+```++This only reduces duplicate backend access within one OpenViking process. It does not change the Provider consistency boundary.++## Cache Policy++RAGFS automatically bypasses paths that are not suitable for caching:++- Lock files: `.path.ovlock`, `*.lock`, `*.lck`+- Control files: `enqueue`, `dequeue`, `peek`, `ack`+- Transient state: `heartbeat`, `lease`, `cursor`, `offset`, `pid`+- Path prefixes configured through `bypass_prefixes`++Add permission-sensitive directories to `bypass_prefixes`. If raw directory entries themselves depend on the caller's permissions, that directory should not be cached.++## Failure and Observability++The cache layer must not affect filesystem correctness:++- `get` failure: fall back to backend.+- `put` failure: record the error and put the path into bypass.+- `delete` failure: record the error and put the path or scope into bypass.+- Provider unavailable: do not return old cache; use backend results as authoritative.++Recommended signals to watch:++- cache hit / miss / bypass+- stale generation+- provider get / put / delete latency+- cache set / delete failures+- inflight leader / follower / backend saved+- backend fallback bytes++## Recommended Rollout++1. Use `memory` locally to validate the configuration shape.+2. Use `redis` to validate real remote-cache benefits.+3. Move to `yuanrong` or `mooncake` for high-performance environments.+4. Cache summary files and raw `read_dir` first, then expand to more regular small files.+5. Add lock, control-plane, and permission-sensitive paths to `bypass_prefixes`.++In short: RAGFS cache is responsible for correct invalidation according to filesystem semantics, while the Provider is responsible for where cache objects live. As long as the backend remains the source of truth, every cache hit must pass envelope and generation validation before it is returned.diff --git a/docs/superpowers/specs/2026-06-12-tos-s3fs-backend-test-design.md b/docs/superpowers/specs/2026-06-12-tos-s3fs-backend-test-design.mdnew file mode 100644index 000000000..66c281e9a--- /dev/null+++ b/docs/superpowers/specs/2026-06-12-tos-s3fs-backend-test-design.md@@ -0,0 +1,141 @@+# TOS S3FS Backend Test Design++## Goal++Verify that OpenViking can use a real ByteDance TOS bucket as the AGFS backend through RAGFS `s3fs`.++The test should prove three things:++1. User-facing config `storage.agfs.backend = "s3"` is accepted for TOS.+2. RAGFS binding operations work against the real TOS bucket.+3. OpenViking service filesystem APIs work on top of that same TOS-backed AGFS.++## Assumptions++- The target object store is TOS using the S3-compatible endpoint.+- The bucket, region, access key, and secret key are provided outside the repo.+- Test data must live under an isolated prefix such as `openviking-e2e/RUN_ID`.+- The test may create and delete objects only under that isolated prefix.+- VectorDB and model-provider behavior are out of scope unless required to start the service.++## Selected Approach++Use a two-layer end-to-end test:++1. Run the existing AGFS binding S3 test suite with a TOS `ov.conf`.+2. Start OpenViking with the same config and run a small filesystem smoke test through the HTTP or CLI surface.++This is preferred over only validating config because it exercises real network I/O, directory markers, reads, listings, and deletion. It is also safer than broad ingestion tests because it avoids semantic processing, embedding calls, and large object creation.++## Alternatives Considered++### Config-only validation++This checks that OpenViking maps `backend = "s3"` to the Rust `s3fs` plugin. It is fast and safe, but it does not prove credentials, endpoint style, directory markers, or TOS object operations work.++### Binding-only test++This runs `tests/agfs/test_fs_binding_s3.py` with a real TOS config. It verifies core filesystem behavior and is the minimum useful live test. It does not verify the OpenViking server wiring above VikingFS.++### Full ingestion test++This would add resources and wait for indexing. It gives broader application confidence, but it introduces model provider, parser, queue, and vector-index variables. That extra surface is not needed to answer whether backendfs works as TOS-backed `s3fs`.++## TOS Configuration++Use a dedicated config file outside committed source, for example `/private/tmp/ov-tos-s3fs-e2e.conf`.++Recommended `storage.agfs` shape:++```json+{+ "backend": "s3",+ "timeout": 10,+ "s3": {+ "bucket": "tos-bucket-name",+ "region": "cn-beijing",+ "access_key": "local-tos-access-key-id",+ "secret_key": "local-tos-secret-access-key",+ "endpoint": "https://tos-s3-cn-beijing.volces.com",+ "prefix": "openviking-e2e/RUN_ID",+ "use_ssl": true,+ "use_path_style": false,+ "directory_marker_mode": "nonempty",+ "normalize_encoding_chars": "?#%+@"+ }+}+```++TOS-specific choices:++- `use_path_style: false` uses virtual-host-style requests, which matches the existing TOS example.+- `directory_marker_mode: "nonempty"` avoids relying on zero-byte directory markers.+- `bucket`, `access_key`, and `secret_key` are local-only values supplied by the operator before the run.+- `prefix` must be unique per run so cleanup is bounded and test objects never mix with production data.++## Test Flow++### 1. Binding smoke++Run the existing live S3 binding test:++```bash+OPENVIKING_CONFIG=/private/tmp/ov-tos-s3fs-e2e.conf pytest tests/agfs/test_fs_binding_s3.py -q+```++Expected result:++- The test is not skipped.+- File write, stat, ls, read, rm pass.+- Directory mkdir, nested write, tree, recursive rm pass.+- Binary write/read pass.++### 2. Service smoke++Start OpenViking with the same config, then perform filesystem operations through the public service surface:++1. Write a small text file under `viking://temp/tos-s3fs-e2e-RUN_ID.txt`.+2. `stat` the URI and verify it is a file.+3. `ls viking://temp/` and verify the file appears.+4. `read` the URI and verify exact content.+5. Delete the URI.+6. Confirm the URI is gone.++Expected result:++- OpenViking starts successfully with the TOS-backed AGFS config.+- Filesystem operations work through the server layer, not only the raw binding layer.++### 3. Optional TOS object check++If an S3-compatible client is available, list the isolated prefix after the write step and after cleanup.++Expected result:++- During the test, objects appear only under `openviking-e2e/RUN_ID`.+- After cleanup, no test file objects remain under that run prefix.++## Safety And Cleanup++- Never run the test with an empty, shared, or production prefix.+- Generate a new `RUN_ID` for each live run.+- Cleanup should delete only objects under the generated run prefix.+- Credentials must not be committed to the repo or printed in logs.+- If cleanup fails, report the exact prefix that needs manual cleanup.++## Success Criteria++The TOS `s3fs` backend test is successful when:++1. `tests/agfs/test_fs_binding_s3.py` passes against the real TOS config.+2. OpenViking starts with the same config.+3. A service-level filesystem write/read/list/delete smoke test passes.+4. Any created objects are removed, or the remaining isolated prefix is reported.++## Out Of Scope++- Adding new automated tests.+- Changing RAGFS or OpenViking runtime code.+- Benchmarking S3FS performance.+- Testing multi-write backup behavior.+- Testing RAGFS cache providers.diff --git a/docs/superpowers/specs/2026-06-16-ragfs-cache-grep-traversal-design.md b/docs/superpowers/specs/2026-06-16-ragfs-cache-grep-traversal-design.mdnew file mode 100644index 000000000..c5c911f3f--- /dev/null+++ b/docs/superpowers/specs/2026-06-16-ragfs-cache-grep-traversal-design.md@@ -0,0 +1,180 @@+# RAGFS Cache-Aware Grep Traversal 设计++## 背景++`CachedFileSystem::grep()` 当前直接委托给被包装的 backend:++```rust+self.backend.grep(...).await+```++这样保留了 backend 的定制能力,但也意味着 cache wrapper 的 `read_dir()` 和 `read()` 缓存不会参与 grep。默认 `FileSystem::grep()` 本身会递归调用 `self.read_dir()` 和 `self.read()`;只有当 `self` 是 `CachedFileSystem` 时,grep 才能复用 cache 层。++这个差异对不同 backend 很重要:++- S3FS/MinIO override 了 `grep()`,先做 flat listing,再并发读取对象。这是冷缓存下的重要 fast path。+- `EncryptionWrappedFS::grep()` 必须在 wrapper 层做递归 traversal,再通过 `self.read()` 解密,不能委托到底层 ciphertext backend。+- `MultiWriteWrappedFS::grep()` 会过滤内部元数据,并补充 redirect file 的搜索语义。+- Python `glob()` 没有 Rust 独立接口,它复用 `VikingFS.tree()` 后在 Python 侧匹配 `rel_path`;因此 glob 已经可以通过 tree cached traversal 间接受益。++因此 grep cache 不能简单替换默认行为。它应该是显式 opt-in,并且默认保留 backend fast path。++## 目标++- 让 grep 在显式配置下可以复用 cache 层的目录 entries 缓存和文件内容缓存。+- 默认保持当前 backend grep 行为,避免影响 S3FS/MinIO 冷缓存性能。+- 让 tree、glob、grep 使用同一个 traversal strategy 配置,减少配置矩阵。+- 不缓存 grep 结果对象,只复用已有 `read_dir()` 和 `read()` 缓存。+- 保持 `recursive`、`case_insensitive`、`node_limit`、`exclude_path`、`level_limit` 的现有语义。++## 非目标++- 不设计 grep result cache。pattern、case、exclude、limit、level 等维度会让 key 和失效复杂化,第一版不做。+- 不改变 `find()` 或 semantic search 路径。+- 不改变 Python `glob()` 的实现方式。+- 不让 cache-aware grep 成为默认行为。+- 不为 multi-write 重写一套 cache-aware redirect grep 语义。++## 建议方案++把当前 tree 专用的 `CacheTreeMode` 泛化为 traversal 级别配置:++```rust+pub enum CacheTraversalMode {+ Backend,+ CachedTraversal,+}+```++`CachePolicy` 保存 `traversal_mode`,默认是 `Backend`。现有 tree 逻辑从 `tree_mode()` 迁移到 `traversal_mode()`。++### Backend Mode++`CachedFileSystem::tree_directory()` 和 `CachedFileSystem::grep()` 都继续委托给 backend:++```rust+self.backend.tree_directory(...).await+self.backend.grep(...).await+```++这个 mode 保留 S3FS flat listing、S3FS 并发 grep、multi-write redirect 处理、encryption wrapper 解密 grep,以及其他 backend override。++### CachedTraversal Mode++`CachedFileSystem::tree_directory()` 继续使用现有 `tree_directory_via_cache()`。++`CachedFileSystem::grep()` 新增 wrapper 私有 helper,例如 `grep_via_cache()`。它复刻默认 `FileSystem::grep()` 的逻辑,但递归过程中调用 cache wrapper 自己的方法:++- `self.stat()` 用于判断文件/目录,保持当前行为:stat 仍直接透传 backend。+- `self.read_dir()` 用于目录展开,因此可复用目录 entries 缓存。+- `self.read(path, 0, 0)` 用于文件内容读取,因此可复用完整文件缓存。+- regex 编译继续使用现有 `compile_grep_regex()`。+- 路径过滤继续使用现有 `normalize_prefix_path()`、`is_excluded_path()`、`relative_match_file()`、`relative_depth()`。++不向 provider 写入 grep result cache key。++## 数据流++Backend mode:++```text+VikingFS.grep()+ -> ragfs grep()+ -> CachedFileSystem::grep()+ -> backend.grep()+```++CachedTraversal mode:++```text+VikingFS.grep()+ -> ragfs grep()+ -> CachedFileSystem::grep()+ -> CachedFileSystem::grep_via_cache()+ -> stat() 判定节点类型+ -> read_dir() 展开目录,复用目录缓存+ -> read(path, 0, 0) 读取文件,复用文件缓存+ -> regex line match+```++Python `glob()` 不新增配置路径:++```text+VikingFS.glob()+ -> VikingFS.tree()+ -> ragfs tree_directory()+ -> traversal_mode 决定 backend tree 或 cached traversal+```++因此 tree、glob、grep 都由同一个 traversal mode 控制。++## 配置++建议使用一个统一配置项,而不是 grep/tree 分开配置:++```toml+[storage.cache]+enabled = true+traversal_mode = "backend" # 默认值+# traversal_mode = "cached_traversal"+```++如果现有临时实现还没有暴露 `tree_mode` 到 `ov.conf`,则直接新增 `traversal_mode` 即可。如果已经暴露过 `tree_mode`,建议短期兼容读取旧字段,但内部统一映射到 `CacheTraversalMode`。++未知值应该在配置校验阶段报错。++## 安全规则++1. 默认 `Backend`。启用 cache 不应该静默改变 grep/tree 的后端执行路径。+2. multi-write 下强制 fallback 到 `Backend` traversal,避免绕过 `MultiWriteWrappedFS::grep()` 和 `tree_directory()` 的 redirect/metadata 语义。+3. encrypted multi-write 已经禁用 mount-level cache,保持现状。+4. 非 multi-write 的 encrypted mount 不能让 cached grep 对 ciphertext 做 regex。若 cache wrapper 位于 encryption wrapper 下方,则该 mount 必须 fallback 到 `Backend` traversal,继续使用 `EncryptionWrappedFS::grep()` 的明文语义;只有当 traversal 位于解密层之上时,才允许 cached grep。+5. provider 不可用时沿用 `read_dir()`/`read()` 的 fallback 行为。只要 backend 可读,cached traversal grep 不应因为 provider 故障失败。+6. 大目录保护继续生效:`entries.len() > max_cached_dir_entries` 时,grep 仍返回正确结果,但该目录 entries 不缓存。+7. `node_limit` 表示 match 数量上限,不是扫描文件数上限;cached grep 必须与默认 grep 保持一致。++## 性能预期++cached traversal grep 的收益主要来自 warm cache:++- 目录 entries 已缓存时,递归展开可以减少 backend `read_dir()` round trip。+- 文件内容已缓存时,重复 grep 可以减少 backend `read()` 或 object get。+- 对 S3FS/MinIO 冷缓存,cached traversal 可能比 backend grep 慢,因为它放弃了 flat listing + 并发读取的专用 fast path。+- 对单层超大目录,超过 `max_cached_dir_entries` 后目录 entries 不缓存,收益有限。+- 对 localfs,收益可能很小,因为本地读取成本低,而 cache provider 增加额外访问成本。++benchmark 必须报告:++- cache enabled/disabled+- `traversal_mode`+- cold cache/warm cache+- backend 类型+- 数据集目录形态++## 测试++围绕 `CachedFileSystem` 增加聚焦测试:++- 默认 traversal mode 是 `Backend`,`grep()` 继续委托 backend。+- `CachedTraversal` 下,第一次 grep 会调用 backend `read_dir()`/`read()` 并填充缓存。+- 目录和文件预热后,第二次 grep 命中 cache,避免重复 backend `read_dir()`/`read()`。+- `recursive=false`、`case_insensitive`、`node_limit`、`exclude_path`、`level_limit` 语义与默认 `FileSystem::grep()` 一致。+- oversized directory 仍能 grep 出正确结果,但目录 entries 不缓存。+- provider unavailable 时 fallback 到 backend,grep 仍返回正确结果。+- multi-write wrapper 即使配置 `CachedTraversal`,grep/tree 也 fallback 到 backend。++如有配置解析层测试,还应覆盖:++- 缺省 `traversal_mode` 等价于 `backend`。+- `cached_traversal` 能正确映射到 `CacheTraversalMode::CachedTraversal`。+- 未知值报错。++## 推进步骤++1. 将 `CacheTreeMode` 重命名或兼容迁移为 `CacheTraversalMode`。+2. 将 `CachePolicy::tree_mode()` 替换为 `traversal_mode()`,默认仍是 `Backend`。+3. 调整 tree 逻辑使用统一 traversal mode。+4. 为 `CachedFileSystem::grep()` 增加 cached traversal 分支和私有 helper。+5. 保留 multi-write fallback 到 backend traversal。+6. 增加 grep cached traversal 单测和配置解析测试。+7. 更新 cache 文档或 benchmark 说明,明确 grep 只有在 `cached_traversal` 下才可能复用 cache。diff --git a/docs/zh/guides/13-ragfs-cache.md b/docs/zh/guides/13-ragfs-cache.mdnew file mode 100644index 000000000..4450c32ee--- /dev/null+++ b/docs/zh/guides/13-ragfs-cache.md@@ -0,0 +1,370 @@+# RAGFS 缓存++RAGFS 缓存是 OpenViking 的可选读缓存层,用于加速文件全量读取和目录读取。它只作为加速层,不作为事实数据源;数据仍以 backend filesystem 为准。++适用前提:++- 只有一个 OpenViking / RAGFS 进程写入同一 namespace。+- 文件和目录变更都经过 RAGFS。+- backend 不被外部绕过 RAGFS 直接修改。+- 缓存 Provider 的同一 key 写入或删除成功后,后续读取不会返回旧值。++## 快速开始++首次配置仍建议先完成基础配置:++```bash+openviking-server init+openviking-server doctor+```++然后在 `~/.openviking/ov.conf` 的 `storage.agfs.cache` 中启用缓存。下面是 Redis 示例,适合快速验证:++```json+{+ "storage": {+ "workspace": "./data",+ "agfs": {+ "backend": "local",+ "cache": {+ "enabled": true,+ "provider": "redis",+ "namespace": "openviking",+ "max_file_size_bytes": 1048576,+ "bypass_prefixes": ["/queue", "/tmp"],+ "redis": {+ "mode": "standalone",+ "endpoints": ["redis://127.0.0.1:6379"],+ "pool_size": 32,+ "connect_timeout_ms": 1000,+ "command_timeout_ms": 20,+ "key_prefix": "ragfs-cache",+ "default_ttl_seconds": 3600,+ "read_from_replica": false+ }+ }+ }+ }+}+```++启动 Redis 和 OpenViking:++```bash+redis-server+openviking-server --config ~/.openviking/ov.conf+```++如果配置文件在默认路径 `~/.openviking/ov.conf`,也可以直接运行:++```bash+openviking-server+```++可用 Provider:++| Provider | 适用场景 | 备注 |+|----------|----------|------|+| `memory` | 本地验证、测试 | 进程内缓存,重启后丢失 |+| `redis` | 快速落地、普通网络环境 | 当前支持 standalone;建议只从 primary 读取 |+| `yuanrong` | 近计算缓存、共享内存或异构多级缓存 | 需要 Yuanrong worker 和 native feature |+| `mooncake` | 远程内存池、RDMA/TCP 数据面 | 需要 Mooncake 服务和 native feature |++如果运行包没有编译对应 Provider,启动时会返回类似 “requires the ... feature” 的错误。++## 原生 Provider 构建++标准 OpenViking wheel 适用于 `memory` 和 `redis` Provider。`yuanrong` 和+`mooncake` Provider 依赖平台相关的原生 SDK,需要针对目标部署环境单独构建。++先安装 wheel 构建工具:++```bash+python -m pip install "maturin[patchelf]"+```++### Yuanrong++安装 Yuanrong DataSystem C++ SDK,并导出头文件和库目录:++```bash+export YUANRONG_SDK_INCLUDE=/path/to/yuanrong/include+export YUANRONG_SDK_LIB_DIR=/path/to/yuanrong/lib+# 可选,默认值为 "datasystem"。+export YUANRONG_SDK_LIB_NAME=datasystem+export LD_LIBRARY_PATH="$YUANRONG_SDK_LIB_DIR:${LD_LIBRARY_PATH:-}"+```++构建并安装 wheel:++```bash+maturin build --release \+ --manifest-path crates/ragfs-python-native/Cargo.toml \+ --features yuanrong-native++python -m pip install --force-reinstall target/wheels/ragfs_python-*.whl+```++OpenViking 启动时,`storage.agfs.cache.yuanrong` 配置的 Yuanrong worker+必须可用。++### Mooncake++检出 `crates/ragfs-cache-mooncake/Cargo.toml` 使用的 Mooncake revision,+然后构建启用 Rust 支持的 Mooncake Store:++```bash+cmake -S /path/to/Mooncake -B /path/to/Mooncake/build \+ -DWITH_STORE=ON \+ -DWITH_STORE_RUST=ON \+ -DCMAKE_BUILD_TYPE=Release++cmake --build /path/to/Mooncake/build \+ --target build_mooncake_store_rust mooncake_master -j+```++导出 Mooncake 官方 Rust binding 所需路径:++```bash+export MOONCAKE_BUILD_DIR=/path/to/Mooncake/build+export MOONCAKE_STORE_LIB_DIR="$MOONCAKE_BUILD_DIR/mooncake-store/src"+export MOONCAKE_STORE_INCLUDE_DIR=/path/to/Mooncake/mooncake-store/include+export LD_LIBRARY_PATH="$MOONCAKE_BUILD_DIR/mooncake-common:\+$MOONCAKE_BUILD_DIR/mooncake-common/src:\+$MOONCAKE_BUILD_DIR/mooncake-store/src:\+$MOONCAKE_BUILD_DIR/mooncake-store/src/cachelib_memory_allocator:\+$MOONCAKE_BUILD_DIR/mooncake-transfer-engine/src:\+$MOONCAKE_BUILD_DIR/mooncake-transfer-engine/src/common/base:\+${LD_LIBRARY_PATH:-}"+```++构建并安装 wheel:++```bash+maturin build --release \+ --manifest-path crates/ragfs-python-native/Cargo.toml \+ --features mooncake-native++python -m pip install --force-reinstall target/wheels/ragfs_python-*.whl+```++OpenViking 启动时,`storage.agfs.cache.mooncake` 配置的 Mooncake metadata+service 和 Master 必须可用。原生 wheel 与平台相关,应在与目标部署环境兼容的+系统中构建。++生产 wheel 应使用仅在显式启用 ASan 时才链接 `libasan` 的 Mooncake revision。+构建后检查 release wheel 不包含且不依赖 `libasan`:++```bash+rm -rf /tmp/ragfs-python-wheel+python -m zipfile -e target/wheels/ragfs_python-*.whl /tmp/ragfs-python-wheel+readelf -d /tmp/ragfs-python-wheel/ragfs_python/ragfs_python.abi3.so \+ | grep libasan+find /tmp/ragfs-python-wheel -name 'libasan*'+```++两项检查都应无输出。++## 配置项++`storage.agfs.cache` 支持以下通用配置:++| 参数 | 类型 | 默认值 | 说明 |+|------|------|--------|------|+| `enabled` | bool | `false` | 是否启用 RAGFS 缓存 |+| `provider` | str | `"memory"` | `memory`、`redis`、`yuanrong` 或 `mooncake` |+| `namespace` | str | `"openviking"` | 缓存命名空间,用于隔离不同部署或租户 |+| `max_file_size_bytes` | int | `1048576` | 允许进入缓存的最大完整文件大小 |+| `bypass_prefixes` | list[str] | `[]` | 强制绕过缓存的路径前缀 |++Redis 配置:++| 参数 | 默认值 | 说明 |+|------|--------|------|+| `mode` | `"standalone"` | Redis 部署模式 |+| `endpoints` | `["redis://127.0.0.1:6379"]` | Redis 连接地址 |+| `username` | `""` | Redis ACL 用户名 |+| `password_env` | `""` | 存放 Redis 密码的环境变量名 |+| `pool_size` | `32` | 命令并发数 |+| `connect_timeout_ms` | `1000` | 连接超时 |+| `command_timeout_ms` | `20` | 命令超时 |+| `key_prefix` | `"ragfs-cache"` | Redis 侧 key 前缀 |+| `default_ttl_seconds` | `3600` | 默认 TTL;`0` 表示不设置 TTL |+| `read_from_replica` | `false` | standalone 模式下必须为 `false` |++Yuanrong 配置:++```json+{+ "storage": {+ "agfs": {+ "cache": {+ "enabled": true,+ "provider": "yuanrong",+ "yuanrong": {+ "host": "127.0.0.1",+ "port": 31501,+ "connect_timeout_ms": 5000,+ "request_timeout_ms": 5000,+ "sdk_concurrency": 4+ }+ }+ }+ }+}+```++Mooncake 配置:++```json+{+ "storage": {+ "agfs": {+ "cache": {+ "enabled": true,+ "provider": "mooncake",+ "mooncake": {+ "local_hostname": "127.0.0.1",+ "metadata_server": "http://127.0.0.1:8080/metadata",+ "master_server_addr": "127.0.0.1:50051",+ "protocol": "tcp",+ "device_name": "",+ "global_segment_size": 536870912,+ "local_buffer_size": 134217728,+ "replica_num": 2,+ "sdk_concurrency": 4,+ "operation_timeout_ms": 5000+ }+ }+ }+ }+}+```++## 整体架构++RAGFS 将缓存拆成两层:++- `CachedFileSystem`:实现文件系统语义,包括 cache hit/miss、backend 回源、回填、失效、generation 校验和指标。+- `CacheProvider`:只负责缓存对象的 `get`、`put`、`delete`、批量读写和关闭。++调用关系:++```text+OpenViking+ -> RAGFS / MountableFS+ -> CachedFileSystem+ |-> CacheProvider -> Memory / Redis / Yuanrong / Mooncake+ `-> Backend FileSystem+```++这种边界让文件、目录、rename、递归删除和写后失效逻辑只在公共层实现。Provider 不需要理解路径语义,只需要稳定存取 key-value 对象。++## 缓存对象++RAGFS 主要缓存三类对象。++### 文件缓存++文件 key 使用稳定命名空间和路径 hash:++```text+ragfs:v1:{namespace}:file:{hash(path)}+```++文件 value 是 `CacheEnvelope`,包含文件内容、对象类型、路径和 generation 快照。全量读取命中后,RAGFS 会先校验 envelope 和 generation,再返回内容。++默认策略会优先缓存 `.abstract.md` 和 `.overview.md` 这类摘要文件;超过 `max_file_size_bytes` 的文件不会进入缓存。非全量 range read 也会绕过缓存。++### 目录缓存++目录 key:++```text+ragfs:v1:{namespace}:dir:{hash(path)}+```++目录缓存保存 backend 原始 `read_dir` entries,而不是权限过滤后的最终结果。权限、角色和 agent context 仍在 OpenViking 上层实时处理。++这样同一份目录缓存可以服务 `ls`、`tree`、`glob`、`grep` 的文件收集阶段,以及删除或移动前的路径收集。++### 子树 Generation++子树 generation key:++```text+ragfs:v1:{namespace}:subtree:{hash(scope)}+```++`remove_all` 和目录 `rename` 可能让 Provider 中残留子孙 key。RAGFS 通过 bump subtree generation,让旧 envelope 的 generation 快照失效,后续真实读取会回源并重建缓存。++## 一致性与失效++单写者场景下,RAGFS 不需要分布式写锁。关键是按文件系统语义维护三类失效:++- 文件变更:删除或更新 `file_key(path)`,删除 `dir_key(parent)`。+- 目录变更:删除目录自身和父目录的 `dir_key`。+- 子树变更:对递归删除和目录 rename bump `subtree` generation。++典型写入顺序:++```text+获取进程内操作锁+-> 执行 backend 变更+-> 更新或删除相关 cache key+-> 必要时 bump subtree generation+-> 返回结果+```++如果 Provider 失败,RAGFS 会以 backend 为准,并让受影响路径进入短期 bypass,避免继续读取可能陈旧的缓存。++## 请求合并++当多个请求同时读取同一个未缓存的小文件或目录时,`CachedFileSystem` 会用进程内 inflight 表合并请求:++```text+第一个 miss 请求成为 leader,负责回源和回填。+后续相同 key 的请求成为 follower,等待 leader 结果。+请求完成后删除 inflight 条目。+```++这只减少同一 OpenViking 进程内的重复 backend 访问,不改变 Provider 的一致性边界。++## 缓存策略++RAGFS 会自动绕过不适合缓存的路径:++- 锁文件:`.path.ovlock`、`*.lock`、`*.lck`+- 控制文件:`enqueue`、`dequeue`、`peek`、`ack`+- 瞬时状态:`heartbeat`、`lease`、`cursor`、`offset`、`pid`+- 用户通过 `bypass_prefixes` 指定的路径前缀++权限敏感目录建议加入 `bypass_prefixes`。如果目录原始 entries 本身就依赖调用者权限,不应缓存该目录。++## 故障与观测++缓存层不能影响文件系统正确性:++- `get` 失败:回源 backend。+- `put` 失败:记录错误,路径进入 bypass。+- `delete` 失败:记录错误,路径或 scope 进入 bypass。+- Provider 不可用:不返回旧缓存,以 backend 结果为准。++建议重点观察:++- cache hit / miss / bypass+- stale generation+- provider get / put / delete 延迟+- cache set / delete 失败+- inflight leader / follower / backend saved+- backend fallback 字节数++## 推荐使用顺序++1. 本地用 `memory` 验证配置形态。+2. 用 `redis` 验证真实远程缓存收益。+3. 对高性能环境再接入 `yuanrong` 或 `mooncake`。+4. 先缓存摘要文件和 raw `read_dir`,再扩展到更多普通小文件。+5. 将锁、控制面和权限敏感路径加入 `bypass_prefixes`。++一句话总结:RAGFS 缓存负责“按文件系统语义正确失效”,Provider 负责“把缓存对象放在哪里”。只要 backend 是事实来源,缓存命中就必须先通过 envelope 和 generation 校验。diff --git a/openviking/utils/agfs_utils.py b/openviking/utils/agfs_utils.pyindex 92ca27cb5..f2d568a96 100644--- a/openviking/utils/agfs_utils.py+++ b/openviking/utils/agfs_utils.py@@ -10,6 +10,8 @@ from dataclasses import dataclassfrom pathlib import Pathfrom typing import Any, Callable, Dict+from openviking_cli.utils.config.config_loader import resolve_config_path+from openviking_cli.utils.config.consts import DEFAULT_OV_CONF, OPENVIKING_CONFIG_ENVfrom openviking_cli.utils.logger import get_loggerlogger = get_logger(__name__)@@ -29,7 +31,9 @@ class RagfsBindingConfig:def to_binding_dict(self) -> Dict[str, Any]:"""Convert the runtime config into the sectioned dict consumed by `RAGFSBindingClient`."""- binding_config: Dict[str, Any] = {}+ binding_config: Dict[str, Any] = {+ "cache": self.agfs.cache.model_dump(mode="json"),+ }if self.root_key is not None:if len(self.root_key) != 32:@@ -355,7 +359,11 @@ def create_agfs_client(config: RagfsBindingConfig) -> Any:)# Construction-time decides whether the stack includes the encryption layer.- client = RAGFSBindingClient(config=config.to_binding_dict())+ config_path = resolve_config_path(None, OPENVIKING_CONFIG_ENV, DEFAULT_OV_CONF)+ client = RAGFSBindingClient(+ str(config_path) if config_path else None,+ config=config.to_binding_dict(),+ )# Automatically mount backend for binding clientmount_agfs_backend(client, config)diff --git a/openviking_cli/utils/config/agfs_config.py b/openviking_cli/utils/config/agfs_config.pyindex e956ddd10..3f6a5b190 100644--- a/openviking_cli/utils/config/agfs_config.py+++ b/openviking_cli/utils/config/agfs_config.py@@ -153,6 +153,168 @@ class QueueFSConfig(BaseModel):return self+class AGFSCacheProvider(str, Enum):+ """Cache providers supported by RAGFS."""++ MEMORY = "memory"+ YUANRONG = "yuanrong"+ MOONCAKE = "mooncake"+ REDIS = "redis"+++class AGFSCacheTraversalMode(str, Enum):+ """Traversal strategy for cache-aware recursive RAGFS APIs."""++ BACKEND = "backend"+ CACHED_TRAVERSAL = "cached_traversal"+++class YuanrongCacheConfig(BaseModel):+ """Configuration for Yuanrong cache provider."""++ host: str = Field(default="127.0.0.1", description="Yuanrong worker host")+ port: int = Field(default=31501, description="Yuanrong worker port")+ connect_timeout_ms: int = Field(default=5000, description="Yuanrong connect timeout")+ request_timeout_ms: int = Field(default=5000, description="Yuanrong request timeout")+ sdk_concurrency: int = Field(default=4, description="Yuanrong SDK concurrency")++ model_config = {"extra": "forbid"}++ @model_validator(mode="after")+ def validate_config(self):+ if not self.host.strip():+ raise ValueError("yuanrong host must not be empty")+ if self.port <= 0 or self.port > 65535:+ raise ValueError("yuanrong port must be between 1 and 65535")+ if self.connect_timeout_ms <= 0:+ raise ValueError("yuanrong connect_timeout_ms must be > 0")+ if self.request_timeout_ms <= 0:+ raise ValueError("yuanrong request_timeout_ms must be > 0")+ if self.sdk_concurrency <= 0:+ raise ValueError("yuanrong sdk_concurrency must be > 0")+ return self+++class MooncakeCacheConfig(BaseModel):+ """Configuration for Mooncake cache provider."""++ local_hostname: str = Field(default="127.0.0.1", description="Mooncake local hostname")+ metadata_server: str = Field(+ default="http://127.0.0.1:8080/metadata",+ description="Mooncake metadata server",+ )+ master_server_addr: str = Field(+ default="127.0.0.1:50051",+ description="Mooncake master server address",+ )+ protocol: str = Field(default="tcp", description="Mooncake transfer protocol")+ device_name: str = Field(default="", description="Mooncake transport device name")+ global_segment_size: int = Field(default=512 << 20, description="Mooncake global segment size")+ local_buffer_size: int = Field(default=128 << 20, description="Mooncake local buffer size")+ replica_num: int = Field(default=2, description="Mooncake replica count")+ sdk_concurrency: int = Field(default=4, description="Mooncake SDK concurrency")+ operation_timeout_ms: int = Field(default=5000, description="Mooncake operation timeout")++ model_config = {"extra": "forbid"}++ @model_validator(mode="after")+ def validate_config(self):+ for name in ("local_hostname", "metadata_server", "master_server_addr", "protocol"):+ if not getattr(self, name).strip():+ raise ValueError(f"mooncake {name} must not be empty")+ if self.protocol not in {"tcp", "rdma", "ascend", "cxl", "nvlink", "barex"}:+ raise ValueError("mooncake protocol is unsupported")+ if self.global_segment_size <= 0:+ raise ValueError("mooncake global_segment_size must be > 0")+ if self.local_buffer_size <= 0:+ raise ValueError("mooncake local_buffer_size must be > 0")+ if self.replica_num <= 0:+ raise ValueError("mooncake replica_num must be > 0")+ if self.sdk_concurrency <= 0:+ raise ValueError("mooncake sdk_concurrency must be > 0")+ if self.operation_timeout_ms <= 0:+ raise ValueError("mooncake operation_timeout_ms must be > 0")+ return self+++class RedisCacheConfig(BaseModel):+ """Configuration for Redis cache provider."""++ mode: str = Field(default="standalone", description="Redis deployment mode")+ endpoints: list[str] = Field(+ default_factory=lambda: ["redis://127.0.0.1:6379"],+ description="Redis endpoint URLs",+ )+ username: str = Field(default="", description="Redis ACL username")+ password_env: str = Field(default="", description="Environment variable containing password")+ pool_size: int = Field(default=32, description="Redis command concurrency")+ connect_timeout_ms: int = Field(default=1000, description="Redis connect timeout")+ command_timeout_ms: int = Field(default=20, description="Redis command timeout")+ key_prefix: str = Field(default="ragfs-cache", description="Redis cache key prefix")+ default_ttl_seconds: int = Field(default=3600, description="Redis default cache TTL")+ read_from_replica: bool = Field(default=False, description="Read from Redis replicas")++ model_config = {"extra": "forbid"}++ @model_validator(mode="after")+ def validate_config(self):+ if self.mode != "standalone":+ raise ValueError("redis mode must be standalone")+ if not self.endpoints:+ raise ValueError("redis endpoints must not be empty")+ if any(not endpoint.strip() for endpoint in self.endpoints):+ raise ValueError("redis endpoints must not contain empty values")+ if self.pool_size <= 0:+ raise ValueError("redis pool_size must be > 0")+ if self.connect_timeout_ms <= 0:+ raise ValueError("redis connect_timeout_ms must be > 0")+ if self.command_timeout_ms <= 0:+ raise ValueError("redis command_timeout_ms must be > 0")+ if not self.key_prefix.strip():+ raise ValueError("redis key_prefix must not be empty")+ if self.default_ttl_seconds < 0:+ raise ValueError("redis default_ttl_seconds must be >= 0")+ if self.read_from_replica:+ raise ValueError("redis read_from_replica is not supported in standalone mode")+ return self+++class AGFSCacheConfig(BaseModel):+ """Configuration for optional RAGFS cache layer."""++ enabled: bool = Field(default=False, description="Enable RAGFS cache")+ provider: AGFSCacheProvider = Field(+ default=AGFSCacheProvider.MEMORY,+ description="RAGFS cache provider",+ )+ namespace: str = Field(default="openviking", description="RAGFS cache namespace")+ max_file_size_bytes: int = Field(+ default=1024 * 1024,+ description="Maximum full-file object size admitted to cache",+ )+ traversal_mode: AGFSCacheTraversalMode = Field(+ default=AGFSCacheTraversalMode.BACKEND,+ description="Traversal strategy for tree, glob, and grep",+ )+ bypass_prefixes: list[str] = Field(+ default_factory=list,+ description="Path prefixes that bypass cache",+ )+ yuanrong: YuanrongCacheConfig = Field(default_factory=YuanrongCacheConfig)+ mooncake: MooncakeCacheConfig = Field(default_factory=MooncakeCacheConfig)+ redis: RedisCacheConfig = Field(default_factory=RedisCacheConfig)++ model_config = {"extra": "forbid"}++ @model_validator(mode="after")+ def validate_config(self):+ if not self.namespace.strip():+ raise ValueError("cache namespace must not be empty")+ if self.max_file_size_bytes <= 0:+ raise ValueError("cache max_file_size_bytes must be > 0")+ return self++class AGFSConfig(BaseModel):"""Configuration for RAGFS (Rust-based AGFS)."""@@ -214,6 +376,11 @@ class AGFSConfig(BaseModel):description="QueueFS configuration.",)+ cache: AGFSCacheConfig = Field(+ default_factory=AGFSCacheConfig,+ description="RAGFS cache configuration.",+ )+retry_times: Any = Field(default=None,exclude=True,diff --git a/tests/misc/test_config_validation.py b/tests/misc/test_config_validation.pyindex 40880a1f7..263993152 100644--- a/tests/misc/test_config_validation.py+++ b/tests/misc/test_config_validation.py@@ -14,6 +14,7 @@ from openviking.utils.agfs_utils import (create_agfs_client,mount_agfs_backend,)+from openviking_cli.utils.config.consts import OPENVIKING_CONFIG_ENVfrom openviking_cli.utils.config.agfs_config import AGFSConfig, S3Configfrom openviking_cli.utils.config.embedding_config import EmbeddingConfig, EmbeddingModelConfigfrom openviking_cli.utils.config.vectordb_config import VectorDBBackendConfig, VolcengineConfig@@ -186,6 +187,93 @@ def test_agfs_queuefs_validation_rejects_invalid_shapes(queuefs, match):AGFSConfig(path="/tmp/ov-test", backend="local", queuefs=queuefs)+def test_agfs_cache_defaults_to_disabled_memory_provider():+ config = AGFSConfig(path="/tmp/ov-test", backend="local")++ assert config.cache.enabled is False+ assert config.cache.provider == "memory"+ assert config.cache.namespace == "openviking"+ assert config.cache.traversal_mode == "backend"+++def test_agfs_cache_accepts_yuanrong_provider_config():+ config = AGFSConfig(+ path="/tmp/ov-test",+ backend="local",+ cache={+ "enabled": True,+ "provider": "yuanrong",+ "namespace": "ov-test",+ "max_file_size_bytes": 4096,+ "traversal_mode": "cached_traversal",+ "bypass_prefixes": ["/queue"],+ "yuanrong": {+ "host": "10.0.0.1",+ "port": 31501,+ "connect_timeout_ms": 1000,+ "request_timeout_ms": 2000,+ "sdk_concurrency": 2,+ },+ },+ )++ assert config.cache.enabled is True+ assert config.cache.provider == "yuanrong"+ assert config.cache.namespace == "ov-test"+ assert config.cache.max_file_size_bytes == 4096+ assert config.cache.traversal_mode == "cached_traversal"+ assert config.cache.bypass_prefixes == ["/queue"]+ assert config.cache.yuanrong.host == "10.0.0.1"+ assert config.cache.yuanrong.sdk_concurrency == 2+++def test_agfs_cache_accepts_redis_provider_config():+ config = AGFSConfig(+ path="/tmp/ov-test",+ backend="local",+ cache={+ "enabled": True,+ "provider": "redis",+ "namespace": "ov-test",+ "redis": {+ "mode": "standalone",+ "endpoints": ["redis://127.0.0.1:6379"],+ "pool_size": 8,+ "connect_timeout_ms": 1000,+ "command_timeout_ms": 20,+ "key_prefix": "ragfs-cache",+ "default_ttl_seconds": 3600,+ "read_from_replica": False,+ },+ },+ )++ assert config.cache.enabled is True+ assert config.cache.provider == "redis"+ assert config.cache.redis.mode == "standalone"+ assert config.cache.redis.endpoints == ["redis://127.0.0.1:6379"]+ assert config.cache.redis.pool_size == 8+ assert config.cache.redis.default_ttl_seconds == 3600+++def test_agfs_cache_rejects_invalid_provider():+ with pytest.raises(ValueError, match="provider"):+ AGFSConfig(+ path="/tmp/ov-test",+ backend="local",+ cache={"provider": "bogus"},+ )+++def test_agfs_cache_rejects_invalid_traversal_mode():+ with pytest.raises(ValueError, match="traversal_mode"):+ AGFSConfig(+ path="/tmp/ov-test",+ backend="local",+ cache={"traversal_mode": "bogus"},+ )++@pytest.mark.parametrize(("queuefs", "queue_db_path", "expected"),[@@ -419,7 +507,8 @@ class _FailingMountClient(_FakeMountClient):class _FakeBindingClient:- def __init__(self, *, config):+ def __init__(self, config_arg=None, *, config=None):+ self.config_arg = config_argself.config = configself.mount_calls = []@@ -475,7 +564,15 @@ def test_mount_agfs_backend_raises_mount_error(tmp_path):def test_ragfs_binding_config_builds_single_binding_dict_for_local_backend(tmp_path):- agfs_config = AGFSConfig(path=str(tmp_path), backend="local")+ agfs_config = AGFSConfig(+ path=str(tmp_path),+ backend="local",+ cache={+ "enabled": True,+ "provider": "memory",+ "namespace": "runtime-cache",+ },+ )config = RagfsBindingConfig(agfs=agfs_config,@@ -487,16 +584,20 @@ def test_ragfs_binding_config_builds_single_binding_dict_for_local_backend(tmp_p"encryption": {"root_key": b"\x01" * 32,"provider_type": 7,- }+ },+ "cache": agfs_config.cache.model_dump(mode="json"),}def test_create_agfs_client_uses_single_binding_config_object(monkeypatch, tmp_path):- agfs_config = AGFSConfig(path=str(tmp_path), backend="memory")- fake_client = _FakeBindingClient(config={})+ agfs_config = AGFSConfig(+ path=str(tmp_path),+ backend="memory",+ cache={"enabled": True, "provider": "memory", "namespace": "runtime-cache"},+ )def _fake_get_binding_client():- return (lambda *, config: fake_client.__class__(config=config), None)+ return (_FakeBindingClient, None)monkeypatch.setattr("openviking.pyagfs.get_binding_client", _fake_get_binding_client)@@ -504,10 +605,31 @@ def test_create_agfs_client_uses_single_binding_config_object(monkeypatch, tmp_pclient = create_agfs_client(config)assert isinstance(client, _FakeBindingClient)- assert client.config == {}+ assert client.config["cache"]["enabled"] is True+ assert client.config["cache"]["namespace"] == "runtime-cache"assert any(call[0] == "memfs" for call in client.mount_calls)+def test_create_agfs_client_passes_resolved_ov_conf_path(monkeypatch, tmp_path):+ config_path = tmp_path / "ov.conf"+ config_path.write_text('{"storage": {"agfs": {"cache": {"enabled": false}}}}')+ monkeypatch.setenv(OPENVIKING_CONFIG_ENV, str(config_path))++ class FakeRAGFSBindingClient(_FakeBindingClient):+ pass++ monkeypatch.setattr(+ "openviking.pyagfs.get_binding_client",+ lambda: (FakeRAGFSBindingClient, None),+ )++ config = RagfsBindingConfig(agfs=AGFSConfig(path=str(tmp_path), backend="memory"))+ client = create_agfs_client(config)++ assert isinstance(client, FakeRAGFSBindingClient)+ assert client.config_arg == str(config_path)++def test_vectordb_validation():"""Test VectorDB config validation"""print("\n" + "=" * 60)diff --git a/tests/misc/test_ragfs_python_manifest_isolation.py b/tests/misc/test_ragfs_python_manifest_isolation.pynew file mode 100644index 000000000..6b9152946--- /dev/null+++ b/tests/misc/test_ragfs_python_manifest_isolation.py@@ -0,0 +1,68 @@+import re+from pathlib import Path+++ROOT = Path(__file__).resolve().parents[2]+++def _read(path: Path) -> str:+ return path.read_text(encoding="utf-8")+++def _array_items(text: str, key: str) -> set[str]:+ match = re.search(rf"{key}\s*=\s*\[(.*?)\]", text, flags=re.DOTALL)+ assert match is not None, f"{key} array not found"+ return set(re.findall(r'"([^"]+)"', match.group(1)))+++def _section(text: str, name: str) -> str:+ match = re.search(rf"^\[{re.escape(name)}\]\s*$(.*?)(?=^\[|\Z)", text, flags=re.DOTALL | re.MULTILINE)+ assert match is not None, f"[{name}] section not found"+ return match.group(1)+++def test_default_workspace_excludes_native_cache_providers():+ manifest = _read(ROOT / "Cargo.toml")+ workspace = _section(manifest, "workspace")+ members = _array_items(workspace, "members")+ excludes = _array_items(workspace, "exclude")++ assert "crates/ragfs-cache-redis" in members+ assert "crates/ragfs-cache-mooncake" not in members+ assert "crates/ragfs-cache-yuanrong" not in members+ assert "crates/ragfs-cache-yuanrong-sys" not in members++ assert "crates/ragfs-cache-mooncake" in excludes+ assert "crates/ragfs-cache-yuanrong" in excludes+ assert "crates/ragfs-cache-yuanrong-sys" in excludes+ assert "crates/ragfs-python-native" in excludes+++def test_default_ragfs_python_manifest_depends_only_on_redis_provider():+ manifest = _read(ROOT / "crates/ragfs-python/Cargo.toml")+ features = _section(manifest, "features")+ dependencies = _section(manifest, "dependencies")++ assert "cache-redis" in features+ assert "mooncake-native" not in features+ assert "yuanrong-native" not in features++ assert "ragfs-cache-redis" in dependencies+ assert "ragfs-cache-mooncake" not in dependencies+ assert "ragfs-cache-yuanrong" not in dependencies+++def test_native_ragfs_python_manifest_is_explicit_provider_entrypoint():+ manifest = _read(ROOT / "crates/ragfs-python-native/Cargo.toml")+ lib = _section(manifest, "lib")+ features = _section(manifest, "features")+ dependencies = _section(manifest, "dependencies")++ assert 'path = "../ragfs-python/src/lib.rs"' in lib+ assert "cache-redis" in features+ assert "mooncake-native" in features+ assert "yuanrong-native" in features++ assert "ragfs-cache-redis" in dependencies+ assert "ragfs-cache-mooncake" in dependencies+ assert "ragfs-cache-yuanrong" in dependenciesdiff --git a/tests/unit/service/test_core_encryption_startup.py b/tests/unit/service/test_core_encryption_startup.pyindex dfe1d7c4d..75d3013df 100644--- a/tests/unit/service/test_core_encryption_startup.py+++ b/tests/unit/service/test_core_encryption_startup.py@@ -12,11 +12,21 @@ from openviking.service.core import OpenVikingServicefrom openviking.utils.agfs_utils import RagfsBindingConfig+class _FakeCacheConfig:+ def model_dump(self, mode: str) -> dict:+ assert mode == "json"+ return {"enabled": False, "provider": "memory"}++class _FakeConfig:"""Minimal config object exposing the service-facing to_dict API."""storage = SimpleNamespace(- agfs=SimpleNamespace(path="/tmp/ov-test", backend="local"),+ agfs=SimpleNamespace(+ path="/tmp/ov-test",+ backend="local",+ cache=_FakeCacheConfig(),+ ),skip_process_lock=False,)@@ -58,6 +68,10 @@ async def test_build_ragfs_binding_config_works_inside_running_event_loop(monkeyassert isinstance(ragfs_config, RagfsBindingConfig)assert ragfs_config.agfs is service._config.storage.agfsassert ragfs_config.to_binding_dict() == {+ "cache": {+ "enabled": False,+ "provider": "memory",+ },"encryption": {"root_key": b"k" * 32,"provider_type": 1,