Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: add `ov observer filesystem`, and optimize mkdir frequency (#2045)

MaojiaShenga1c9e5080bb8
diff --git a/crates/ov_cli/src/commands/observer.rs b/crates/ov_cli/src/commands/observer.rs
index d07c2ebe1..41485b35f 100644
--- a/crates/ov_cli/src/commands/observer.rs
+++ b/crates/ov_cli/src/commands/observer.rs
@@ -38,6 +38,16 @@ pub async fn retrieval(
Ok(())
}
+pub async fn filesystem(
+ client: &HttpClient,
+ output_format: OutputFormat,
+ compact: bool,
+) -> Result<()> {
+ let response: serde_json::Value = client.get("/api/v1/observer/filesystem", &[]).await?;
+ output_success(&response, output_format, compact);
+ Ok(())
+}
+
pub async fn system(client: &HttpClient, output_format: OutputFormat, compact: bool) -> Result<()> {
let response: serde_json::Value = client.get("/api/v1/observer/system", &[]).await?;
output_success(&response, output_format, compact);
diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rs
index dfeb1332d..a123ccad1 100644
--- a/crates/ov_cli/src/handlers.rs
+++ b/crates/ov_cli/src/handlers.rs
@@ -278,6 +278,9 @@ pub async fn handle_observer(cmd: ObserverCommands, ctx: CliContext) -> Result<(
ObserverCommands::Retrieval => {
commands::observer::retrieval(&client, ctx.output_format, ctx.compact).await
}
+ ObserverCommands::Filesystem => {
+ commands::observer::filesystem(&client, ctx.output_format, ctx.compact).await
+ }
ObserverCommands::System => {
commands::observer::system(&client, ctx.output_format, ctx.compact).await
}
diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs
index fd1227539..4cd061121 100644
--- a/crates/ov_cli/src/main.rs
+++ b/crates/ov_cli/src/main.rs
@@ -681,6 +681,8 @@ enum ObserverCommands {
Transaction,
/// Get retrieval quality metrics
Retrieval,
+ /// Get filesystem operation metrics
+ Filesystem,
/// Get overall system status
System,
}
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs
index 9859de0be..e5fa3da34 100644
--- a/crates/ragfs-python/src/lib.rs
+++ b/crates/ragfs-python/src/lib.rs
@@ -12,7 +12,8 @@ use std::sync::Arc;
use std::time::UNIX_EPOCH;
use ragfs::core::{
- ConfigValue, FileInfo, FileSystem, GrepResult, MountableFS, PluginConfig, WriteFlag,
+ ConfigValue, FileInfo, FileSystem, FilesystemStats, FsOperation, GrepResult, MountableFS,
+ OperationStats, PluginConfig, WriteFlag,
};
#[cfg(feature = "s3")]
use ragfs::plugins::S3FSPlugin;
@@ -119,6 +120,32 @@ fn grep_result_to_py_dict(py: Python<'_>, result: &GrepResult) -> PyResult<Py<Py
Ok(dict.into())
}
+/// Convert OperationStats to a Python dict.
+fn operation_stats_to_py_dict(py: Python<'_>, stats: &OperationStats) -> PyResult<Py<PyDict>> {
+ let dict = PyDict::new(py);
+ dict.set_item("count", stats.count)?;
+ dict.set_item("total_time_us", stats.total_time_us)?;
+ dict.set_item("min_time_us", stats.min_time_us)?;
+ dict.set_item("max_time_us", stats.max_time_us)?;
+ dict.set_item("avg_time_us", stats.avg_time_us())?;
+ Ok(dict.into())
+}
+
+/// Convert FilesystemStats to a Python dict.
+fn filesystem_stats_to_py_dict(py: Python<'_>, stats: &FilesystemStats) -> PyResult<Py<PyDict>> {
+ let dict = PyDict::new(py);
+ let ops_dict = PyDict::new(py);
+
+ for op in FsOperation::all() {
+ let op_stats = stats.get(*op);
+ let op_dict = operation_stats_to_py_dict(py, op_stats)?;
+ ops_dict.set_item(op.as_str(), op_dict)?;
+ }
+
+ dict.set_item("operations", ops_dict)?;
+ Ok(dict.into())
+}
+
/// Format unix timestamp as RFC3339 string (simplified, UTC)
fn format_rfc3339(secs: u64) -> String {
let s = secs;
@@ -349,6 +376,23 @@ impl RAGFSBindingClient {
Ok(m)
}
+ /// Ensure all parent directories exist for the given path.
+ #[pyo3(signature = (path, mode="755"))]
+ fn ensure_parent_dirs(&self, path: String, mode: &str) -> PyResult<HashMap<String, String>> {
+ let mode_int = u32::from_str_radix(mode, 8)
+ .map_err(|e| PyRuntimeError::new_err(format!("Invalid mode '{}': {}", mode, e)))?;
+
+ let fs = self.fs.clone();
+ Python::attach(|py| {
+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.ensure_parent_dirs(&path, mode_int).await }))
+ })
+ .map_err(to_py_err)?;
+
+ let mut m = HashMap::new();
+ m.insert("message".to_string(), "parent directories ensured".to_string());
+ Ok(m)
+ }
+
/// Remove a file or directory.
#[pyo3(signature = (path, recursive=false))]
fn rm(&self, path: String, recursive: bool) -> PyResult<HashMap<String, String>> {
@@ -611,6 +655,66 @@ impl RAGFSBindingClient {
"digest not yet implemented in ragfs-python",
))
}
+
+ /// Get filesystem statistics.
+ ///
+ /// Args:
+ /// path: Optional mount path to get stats for. If None, get stats for all mounts.
+ ///
+ /// Returns:
+ /// Statistics data as a dict.
+ #[pyo3(signature = (path=None))]
+ fn get_stats(&self, path: Option<String>) -> PyResult<Py<PyAny>> {
+ let fs = self.fs.clone();
+
+ Python::attach(|py| {
+ if let Some(mount_path) = path {
+ // Get stats for a specific mount
+ let fs2 = fs.clone();
+ let mount_path_clone = mount_path.clone();
+ let stats = py_detach_blocking(py, move || {
+ self.rt.block_on(async move { fs.get_mount_stats(&mount_path_clone).await })
+ })
+ .map_err(to_py_err)?;
+
+ let mounts = py_detach_blocking(py, move || {
+ self.rt.block_on(async move { fs2.list_mounts().await })
+ });
+
+ let plugin_name = mounts
+ .into_iter()
+ .find(|(p, _)| p == &mount_path)
+ .map(|(_, plugin)| plugin)
+ .unwrap_or_default();
+
+ let result = PyDict::new(py);
+ result.set_item("path", mount_path)?;
+ result.set_item("plugin", plugin_name)?;
+ let stats_dict = filesystem_stats_to_py_dict(py, &stats)?;
+ result.set_item("stats", stats_dict)?;
+ Ok(result.into())
+ } else {
+ // Get stats for all mounts
+ let all_stats = py_detach_blocking(py, move || {
+ self.rt.block_on(async move { fs.get_all_stats().await })
+ });
+
+ let mounts_list = PyList::empty(py);
+ for (path, (plugin, stats)) in all_stats {
+ let mount_dict = PyDict::new(py);
+ mount_dict.set_item("path", path)?;
+ mount_dict.set_item("plugin", plugin)?;
+ let stats_dict = filesystem_stats_to_py_dict(py, &stats)?;
+ mount_dict.set_item("stats", stats_dict)?;
+ mounts_list.append(mount_dict)?;
+ }
+
+ let result = PyDict::new(py);
+ result.set_item("mounts", mounts_list)?;
+ Ok(result.into())
+ }
+ })
+ }
}
/// Python module definition
diff --git a/crates/ragfs/src/core/filesystem.rs b/crates/ragfs/src/core/filesystem.rs
index 7dee204b6..7b07b9e8c 100644
--- a/crates/ragfs/src/core/filesystem.rs
+++ b/crates/ragfs/src/core/filesystem.rs
@@ -7,7 +7,7 @@
use async_trait::async_trait;
use regex::Regex;
-use super::errors::Result;
+use super::errors::{Error, Result};
use super::types::{FileInfo, GrepResult, WriteFlag};
/// Normalize a path for prefix comparisons.
@@ -396,6 +396,44 @@ pub trait FileSystem: Send + Sync {
Ok(())
}
+
+ /// Ensure all parent directories exist for the given path
+ ///
+ /// # Arguments
+ /// * `path` - The path for which to ensure parent directories exist
+ /// * `mode` - Unix-style permissions for created directories (default: 0o755)
+ ///
+ /// # Returns
+ /// Ok(()) if successful
+ async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {
+ // Skip if path is root or has no parent
+ if path == "/" || !path.contains('/') {
+ return Ok(());
+ }
+
+ // Get parent path by removing the last component
+ let parts: Vec<&str> = path.trim_start_matches('/').split('/').collect();
+ if parts.len() <= 1 {
+ return Ok(());
+ }
+
+ let parent_parts = &parts[..parts.len() - 1];
+ let mut current = String::new();
+
+ for part in parent_parts {
+ current.push('/');
+ current.push_str(part);
+
+ // Try to create directory, ignore if already exists
+ match self.mkdir(&current, mode).await {
+ Ok(_) => {}
+ Err(Error::AlreadyExists(_)) => {}
+ Err(e) => return Err(e),
+ }
+ }
+
+ Ok(())
+ }
}
#[cfg(test)]
diff --git a/crates/ragfs/src/core/mod.rs b/crates/ragfs/src/core/mod.rs
index 58ce85cfb..5718cc308 100644
--- a/crates/ragfs/src/core/mod.rs
+++ b/crates/ragfs/src/core/mod.rs
@@ -11,6 +11,8 @@ pub mod errors;
pub mod filesystem;
pub mod mountable;
pub mod plugin;
+pub mod stats;
+pub mod stats_wrapper;
pub mod types;
// Re-export commonly used types
@@ -18,4 +20,6 @@ pub use errors::{Error, Result};
pub use filesystem::FileSystem;
pub use mountable::MountableFS;
pub use plugin::{HealthStatus, PluginRegistry, ServicePlugin};
+pub use stats::{FilesystemStats, FsOperation, OperationStats, OperationTimer, StatsCollector};
+pub use stats_wrapper::StatsWrappedFS;
pub use types::{ConfigParameter, ConfigValue, FileInfo, GrepResult, PluginConfig, WriteFlag};
diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs
index 1bcc6ad2e..ca613d8aa 100644
--- a/crates/ragfs/src/core/mountable.rs
+++ b/crates/ragfs/src/core/mountable.rs
@@ -13,6 +13,8 @@ use tokio::sync::RwLock;
use super::errors::{Error, Result};
use super::filesystem::FileSystem;
use super::plugin::ServicePlugin;
+use super::stats::{FilesystemStats, StatsCollector};
+use super::stats_wrapper::StatsWrappedFS;
use super::types::{FileInfo, GrepResult, PluginConfig, WriteFlag};
/// Information about a mounted filesystem
@@ -21,9 +23,12 @@ struct MountInfo {
/// The mount path (e.g., "/memfs")
path: String,
- /// The filesystem instance
+ /// The filesystem instance (wrapped with statistics)
fs: Arc<dyn FileSystem>,
+ /// The statistics collector for this mount
+ stats: Arc<StatsCollector>,
+
/// The plugin that created this filesystem
plugin_name: String,
}
@@ -97,10 +102,15 @@ impl MountableFS {
// Initialize filesystem
let fs = plugin.initialize(config.clone()).await?;
+ // Wrap with statistics
+ let wrapped_fs = StatsWrappedFS::new(fs);
+ let stats_collector = wrapped_fs.stats_collector();
+
// Add to mounts
let mount_info = MountInfo {
path: normalized_path.clone(),
- fs: Arc::from(fs),
+ fs: Arc::from(Box::new(wrapped_fs) as Box<dyn FileSystem>),
+ stats: stats_collector,
plugin_name: config.name.clone(),
};
@@ -140,6 +150,40 @@ impl MountableFS {
.collect()
}
+ /// Get statistics for a specific mount point
+ ///
+ /// # Arguments
+ /// * `path` - The mount path
+ ///
+ /// # Returns
+ /// The filesystem statistics for the mount
+ pub async fn get_mount_stats(&self, path: &str) -> Result<FilesystemStats> {
+ let normalized_path = normalize_path(path);
+ let mounts = self.mounts.read().await;
+
+ let mount_info = mounts
+ .get(&normalized_path)
+ .ok_or_else(|| Error::MountPointNotFound(normalized_path.clone()))?;
+
+ Ok(mount_info.stats.snapshot().await)
+ }
+
+ /// Get aggregated statistics for all mount points
+ ///
+ /// # Returns
+ /// A map of mount path to filesystem statistics
+ pub async fn get_all_stats(&self) -> HashMap<String, (String, FilesystemStats)> {
+ let mounts = self.mounts.read().await;
+ let mut result = HashMap::new();
+
+ for (path, info) in mounts.iter() {
+ let stats = info.stats.snapshot().await;
+ result.insert(path.clone(), (info.plugin_name.clone(), stats));
+ }
+
+ result
+ }
+
/// Find the mount point for a given path
///
/// # Arguments
@@ -280,6 +324,11 @@ impl FileSystem for MountableFS {
mount_info.fs.truncate(&rel_path, size).await
}
+ async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {
+ let (mount_info, rel_path) = self.find_mount(path).await?;
+ mount_info.fs.ensure_parent_dirs(&rel_path, mode).await
+ }
+
async fn grep(
&self,
path: &str,
diff --git a/crates/ragfs/src/core/stats.rs b/crates/ragfs/src/core/stats.rs
new file mode 100644
index 000000000..7ef4b8973
--- /dev/null
+++ b/crates/ragfs/src/core/stats.rs
@@ -0,0 +1,228 @@
+//! Filesystem operation statistics
+//!
+//! This module provides functionality to track filesystem operations,
+//! including operation counts and latency statistics.
+
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::sync::RwLock;
+
+/// Type of filesystem operation
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+pub enum FsOperation {
+ /// Create a file
+ Create,
+ /// Create a directory
+ Mkdir,
+ /// Remove a file
+ Remove,
+ /// Remove recursively
+ RemoveAll,
+ /// Read a file
+ Read,
+ /// Write a file
+ Write,
+ /// List directory contents
+ ReadDir,
+ /// Get file metadata
+ Stat,
+ /// Rename a file/directory
+ Rename,
+ /// Change file permissions
+ Chmod,
+ /// Truncate a file
+ Truncate,
+ /// Check if a path exists
+ Exists,
+ /// Grep operation
+ Grep,
+ /// Ensure parent directories exist
+ EnsureParentDirs,
+}
+
+impl FsOperation {
+ /// Get all operation types
+ pub fn all() -> &'static [FsOperation] {
+ &[
+ FsOperation::Create,
+ FsOperation::Mkdir,
+ FsOperation::Remove,
+ FsOperation::RemoveAll,
+ FsOperation::Read,
+ FsOperation::Write,
+ FsOperation::ReadDir,
+ FsOperation::Stat,
+ FsOperation::Rename,
+ FsOperation::Chmod,
+ FsOperation::Truncate,
+ FsOperation::Exists,
+ FsOperation::Grep,
+ FsOperation::EnsureParentDirs,
+ ]
+ }
+
+ /// Get the operation name as string
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ FsOperation::Create => "create",
+ FsOperation::Mkdir => "mkdir",
+ FsOperation::Remove => "remove",
+ FsOperation::RemoveAll => "remove_all",
+ FsOperation::Read => "read",
+ FsOperation::Write => "write",
+ FsOperation::ReadDir => "read_dir",
+ FsOperation::Stat => "stat",
+ FsOperation::Rename => "rename",
+ FsOperation::Chmod => "chmod",
+ FsOperation::Truncate => "truncate",
+ FsOperation::Exists => "exists",
+ FsOperation::Grep => "grep",
+ FsOperation::EnsureParentDirs => "ensure_parent_dirs",
+ }
+ }
+}
+
+/// Statistics for a single operation type
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct OperationStats {
+ /// Number of times the operation was called
+ pub count: u64,
+ /// Total time spent on this operation (microseconds)
+ pub total_time_us: u64,
+ /// Minimum time spent (microseconds)
+ pub min_time_us: u64,
+ /// Maximum time spent (microseconds)
+ pub max_time_us: u64,
+}
+
+impl Default for OperationStats {
+ fn default() -> Self {
+ Self {
+ count: 0,
+ total_time_us: 0,
+ min_time_us: u64::MAX,
+ max_time_us: 0,
+ }
+ }
+}
+
+impl OperationStats {
+ /// Record an operation duration
+ pub fn record(&mut self, duration: Duration) {
+ let us = duration.as_micros() as u64;
+ self.count += 1;
+ self.total_time_us += us;
+ if us < self.min_time_us {
+ self.min_time_us = us;
+ }
+ if us > self.max_time_us {
+ self.max_time_us = us;
+ }
+ }
+
+ /// Get average time per operation (microseconds)
+ pub fn avg_time_us(&self) -> f64 {
+ if self.count == 0 {
+ 0.0
+ } else {
+ self.total_time_us as f64 / self.count as f64
+ }
+ }
+}
+
+/// Complete filesystem statistics
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct FilesystemStats {
+ /// Per-operation statistics
+ pub operations: HashMap<FsOperation, OperationStats>,
+}
+
+impl Default for FilesystemStats {
+ fn default() -> Self {
+ let mut operations = HashMap::new();
+ for op in FsOperation::all() {
+ operations.insert(*op, OperationStats::default());
+ }
+ Self { operations }
+ }
+}
+
+impl FilesystemStats {
+ /// Get statistics for a specific operation
+ pub fn get(&self, op: FsOperation) -> &OperationStats {
+ self.operations.get(&op).unwrap()
+ }
+
+ /// Get total number of operations
+ pub fn total_operations(&self) -> u64 {
+ self.operations.values().map(|s| s.count).sum()
+ }
+
+ /// Reset all statistics
+ pub fn reset(&mut self) {
+ *self = Self::default();
+ }
+}
+
+/// Thread-safe statistics collector
+pub struct StatsCollector {
+ stats: Arc<RwLock<FilesystemStats>>,
+}
+
+impl Default for StatsCollector {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl StatsCollector {
+ /// Create a new statistics collector
+ pub fn new() -> Self {
+ Self {
+ stats: Arc::new(RwLock::new(FilesystemStats::default())),
+ }
+ }
+
+ /// Record an operation
+ pub async fn record(&self, op: FsOperation, duration: Duration) {
+ let mut stats = self.stats.write().await;
+ stats.operations.entry(op).or_default().record(duration);
+ }
+
+ /// Get a snapshot of current statistics
+ pub async fn snapshot(&self) -> FilesystemStats {
+ self.stats.read().await.clone()
+ }
+
+ /// Reset all statistics
+ pub async fn reset(&self) {
+ let mut stats = self.stats.write().await;
+ stats.reset();
+ }
+}
+
+/// Timer for measuring operation duration
+pub struct OperationTimer {
+ op: FsOperation,
+ start: Instant,
+ collector: Arc<StatsCollector>,
+}
+
+impl OperationTimer {
+ /// Create a new timer for an operation
+ pub fn start(op: FsOperation, collector: Arc<StatsCollector>) -> Self {
+ Self {
+ op,
+ start: Instant::now(),
+ collector,
+ }
+ }
+
+ /// Finish the timer and record the duration
+ pub async fn finish(self) {
+ let duration = self.start.elapsed();
+ self.collector.record(self.op, duration).await;
+ }
+}
diff --git a/crates/ragfs/src/core/stats_wrapper.rs b/crates/ragfs/src/core/stats_wrapper.rs
new file mode 100644
index 000000000..dd28e06fa
--- /dev/null
+++ b/crates/ragfs/src/core/stats_wrapper.rs
@@ -0,0 +1,165 @@
+//! Statistics wrapper for FileSystem
+//!
+//! This module provides a wrapper around any FileSystem implementation
+//! that automatically collects operation statistics.
+
+use async_trait::async_trait;
+use std::sync::Arc;
+
+use super::{
+ FileInfo, FileSystem, FsOperation, GrepResult, OperationTimer, Result, StatsCollector,
+ WriteFlag,
+};
+
+/// A wrapper around FileSystem that automatically collects operation statistics
+pub struct StatsWrappedFS {
+ inner: Box<dyn FileSystem>,
+ stats: Arc<StatsCollector>,
+}
+
+impl StatsWrappedFS {
+ /// Create a new statistics-wrapped filesystem
+ pub fn new(inner: Box<dyn FileSystem>) -> Self {
+ Self {
+ inner,
+ stats: Arc::new(StatsCollector::new()),
+ }
+ }
+
+ /// Create a new statistics-wrapped filesystem with an existing collector
+ pub fn with_collector(inner: Box<dyn FileSystem>, stats: Arc<StatsCollector>) -> Self {
+ Self { inner, stats }
+ }
+
+ /// Get a reference to the statistics collector
+ pub fn stats_collector(&self) -> Arc<StatsCollector> {
+ Arc::clone(&self.stats)
+ }
+
+ /// Get the inner filesystem (for testing)
+ #[cfg(test)]
+ pub fn into_inner(self) -> Box<dyn FileSystem> {
+ self.inner
+ }
+}
+
+#[async_trait]
+impl FileSystem for StatsWrappedFS {
+ async fn create(&self, path: &str) -> Result<()> {
+ let timer = OperationTimer::start(FsOperation::Create, Arc::clone(&self.stats));
+ let result = self.inner.create(path).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn mkdir(&self, path: &str, mode: u32) -> Result<()> {
+ let timer = OperationTimer::start(FsOperation::Mkdir, Arc::clone(&self.stats));
+ let result = self.inner.mkdir(path, mode).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn remove(&self, path: &str) -> Result<()> {
+ let timer = OperationTimer::start(FsOperation::Remove, Arc::clone(&self.stats));
+ let result = self.inner.remove(path).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn remove_all(&self, path: &str) -> Result<()> {
+ let timer = OperationTimer::start(FsOperation::RemoveAll, Arc::clone(&self.stats));
+ let result = self.inner.remove_all(path).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn read(&self, path: &str, offset: u64, size: u64) -> Result<Vec<u8>> {
+ let timer = OperationTimer::start(FsOperation::Read, Arc::clone(&self.stats));
+ let result = self.inner.read(path, offset, size).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn write(&self, path: &str, data: &[u8], offset: u64, flags: WriteFlag) -> Result<u64> {
+ let timer = OperationTimer::start(FsOperation::Write, Arc::clone(&self.stats));
+ let result = self.inner.write(path, data, offset, flags).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {
+ let timer = OperationTimer::start(FsOperation::ReadDir, Arc::clone(&self.stats));
+ let result = self.inner.read_dir(path).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn stat(&self, path: &str) -> Result<FileInfo> {
+ let timer = OperationTimer::start(FsOperation::Stat, Arc::clone(&self.stats));
+ let result = self.inner.stat(path).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {
+ let timer = OperationTimer::start(FsOperation::Rename, Arc::clone(&self.stats));
+ let result = self.inner.rename(old_path, new_path).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn chmod(&self, path: &str, mode: u32) -> Result<()> {
+ let timer = OperationTimer::start(FsOperation::Chmod, Arc::clone(&self.stats));
+ let result = self.inner.chmod(path, mode).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn truncate(&self, path: &str, size: u64) -> Result<()> {
+ let timer = OperationTimer::start(FsOperation::Truncate, Arc::clone(&self.stats));
+ let result = self.inner.truncate(path, size).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {
+ let timer = OperationTimer::start(FsOperation::EnsureParentDirs, Arc::clone(&self.stats));
+ let result = self.inner.ensure_parent_dirs(path, mode).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn exists(&self, path: &str) -> bool {
+ let timer = OperationTimer::start(FsOperation::Exists, Arc::clone(&self.stats));
+ let result = self.inner.exists(path).await;
+ timer.finish().await;
+ result
+ }
+
+ async fn grep(
+ &self,
+ path: &str,
+ pattern: &str,
+ recursive: bool,
+ case_insensitive: bool,
+ node_limit: Option<usize>,
+ exclude_path: Option<&str>,
+ level_limit: Option<usize>,
+ ) -> Result<GrepResult> {
+ let timer = OperationTimer::start(FsOperation::Grep, Arc::clone(&self.stats));
+ let result = self
+ .inner
+ .grep(
+ path,
+ pattern,
+ recursive,
+ case_insensitive,
+ node_limit,
+ exclude_path,
+ level_limit,
+ )
+ .await;
+ timer.finish().await;
+ result
+ }
+}
diff --git a/crates/ragfs/src/lib.rs b/crates/ragfs/src/lib.rs
index fa3464ad9..8c345276a 100644
--- a/crates/ragfs/src/lib.rs
+++ b/crates/ragfs/src/lib.rs
@@ -38,8 +38,9 @@ pub mod server;
// Re-export core types for convenience
pub use core::{
- ConfigParameter, ConfigValue, Error, FileInfo, FileSystem, HealthStatus, MountableFS,
- PluginConfig, PluginRegistry, Result, ServicePlugin, WriteFlag,
+ ConfigParameter, ConfigValue, Error, FileInfo, FileSystem, FilesystemStats, FsOperation,
+ HealthStatus, MountableFS, OperationStats, OperationTimer, PluginConfig, PluginRegistry,
+ Result, ServicePlugin, StatsCollector, StatsWrappedFS, WriteFlag,
};
/// Version of RAGFS
diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rs
index 1eb570c7e..4c98f9b4c 100644
--- a/crates/ragfs/src/plugins/localfs/mod.rs
+++ b/crates/ragfs/src/plugins/localfs/mod.rs
@@ -787,6 +787,20 @@ impl FileSystem for LocalFileSystem {
Ok(())
}
+ async fn ensure_parent_dirs(&self, path: &str, _mode: u32) -> Result<()> {
+ let local_path = self.resolve_path(path);
+
+ // Get parent directory
+ if let Some(parent) = local_path.parent() {
+ // Create all parent directories if they don't exist
+ tokio::fs::create_dir_all(parent)
+ .await
+ .map_err(|e| Error::plugin(format!("failed to create parent directories: {}", e)))?;
+ }
+
+ Ok(())
+ }
+
async fn grep(
&self,
path: &str,
diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rs
index b1aef5bf1..e802a2134 100644
--- a/crates/ragfs/src/plugins/s3fs/mod.rs
+++ b/crates/ragfs/src/plugins/s3fs/mod.rs
@@ -460,6 +460,11 @@ impl FileSystem for S3FileSystem {
Ok(())
}
+ async fn ensure_parent_dirs(&self, _path: &str, _mode: u32) -> Result<()> {
+ // S3 doesn't require directories to exist - no-op
+ Ok(())
+ }
+
async fn truncate(&self, path: &str, size: u64) -> Result<()> {
let normalized = Self::normalize_path(path);
let key = self.client.build_key(&normalized);
diff --git a/crates/ragfs/src/server/handlers.rs b/crates/ragfs/src/server/handlers.rs
index 4806681be..2d7b67555 100644
--- a/crates/ragfs/src/server/handlers.rs
+++ b/crates/ragfs/src/server/handlers.rs
@@ -11,7 +11,7 @@ use axum::{
use serde::{Deserialize, Serialize};
use std::sync::Arc;
-use crate::core::{FileSystem, GrepResult, MountableFS, PluginConfig, WriteFlag};
+use crate::core::{FileSystem, FilesystemStats, GrepResult, MountableFS, PluginConfig, WriteFlag};
/// Shared application state
#[derive(Clone)]
@@ -71,6 +71,33 @@ pub struct FileQuery {
pub struct DirQuery {
/// Directory path
pub path: String,
+ /// Optional permission mode (octal string, e.g. "755")
+ pub mode: Option<String>,
+}
+
+/// Query parameters for statistics operations
+#[derive(Debug, Deserialize)]
+pub struct StatsQuery {
+ /// Mount path (optional, if not provided returns all stats)
+ pub path: Option<String>,
+}
+
+/// Statistics response for a single mount
+#[derive(Debug, Serialize)]
+pub struct MountStats {
+ /// Mount path
+ pub path: String,
+ /// Plugin name
+ pub plugin: String,
+ /// Filesystem statistics
+ pub stats: FilesystemStats,
+}
+
+/// Statistics response for all mounts
+#[derive(Debug, Serialize)]
+pub struct AllStatsResponse {
+ /// Statistics for all mounts
+ pub mounts: Vec<MountStats>,
}
/// Request body for mount operation
@@ -273,6 +300,28 @@ pub async fn create_directory(
}
}
+/// POST /api/v1/directories/ensure-parent - Ensure parent directories exist
+pub async fn ensure_parent_dirs(
+ State(state): State<AppState>,
+ Query(query): Query<DirQuery>,
+) -> Response {
+ let mode = query.mode.and_then(|m| u32::from_str_radix(&m, 8).ok()).unwrap_or(0o755);
+ match state.fs.ensure_parent_dirs(&query.path, mode).await {
+ Ok(_) => (
+ StatusCode::OK,
+ Json(ApiResponse::success(serde_json::json!({
+ "path": query.path
+ }))),
+ )
+ .into_response(),
+ Err(e) => (
+ StatusCode::INTERNAL_SERVER_ERROR,
+ Json(ApiResponse::<()>::error(e.to_string())),
+ )
+ .into_response(),
+ }
+}
+
// ============================================================================
// Mount Management Handlers
// ============================================================================
@@ -413,6 +462,50 @@ pub async fn health_check() -> Response {
(StatusCode::OK, Json(ApiResponse::success(response))).into_response()
}
+// ============================================================================
+// Statistics Handlers
+// ============================================================================
+
+/// GET /api/v1/stats - Get filesystem statistics
+pub async fn get_stats(State(state): State<AppState>, Query(query): Query<StatsQuery>) -> Response {
+ if let Some(path) = query.path {
+ // Get stats for a specific mount
+ match state.fs.get_mount_stats(&path).await {
+ Ok(stats) => {
+ let mounts = state.fs.list_mounts().await;
+ let plugin_name = mounts
+ .into_iter()
+ .find(|(p, _)| p == &path)
+ .map(|(_, plugin)| plugin)
+ .unwrap_or_default();
+
+ let mount_stats = MountStats {
+ path,
+ plugin: plugin_name,
+ stats,
+ };
+
+ (StatusCode::OK, Json(ApiResponse::success(mount_stats))).into_response()
+ }
+ Err(e) => (
+ StatusCode::NOT_FOUND,
+ Json(ApiResponse::<()>::error(e.to_string())),
+ )
+ .into_response(),
+ }
+ } else {
+ // Get stats for all mounts
+ let all_stats = state.fs.get_all_stats().await;
+ let mounts: Vec<MountStats> = all_stats
+ .into_iter()
+ .map(|(path, (plugin, stats))| MountStats { path, plugin, stats })
+ .collect();
+
+ let response = AllStatsResponse { mounts };
+ (StatusCode::OK, Json(ApiResponse::success(response))).into_response()
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/ragfs/src/server/router.rs b/crates/ragfs/src/server/router.rs
index 314eec935..44c5fc00f 100644
--- a/crates/ragfs/src/server/router.rs
+++ b/crates/ragfs/src/server/router.rs
@@ -13,8 +13,9 @@ use tower_http::{
use tracing::Level;
use super::handlers::{
- create_directory, create_file, delete_file, grep_content, health_check, list_directory,
- list_mounts, mount_filesystem, read_file, stat_file, unmount_filesystem, write_file, AppState,
+ create_directory, create_file, delete_file, ensure_parent_dirs, get_stats, grep_content,
+ health_check, list_directory, list_mounts, mount_filesystem, read_file, stat_file,
+ unmount_filesystem, write_file, AppState,
};
/// Create the main application router
@@ -29,12 +30,15 @@ pub fn create_router(state: AppState, enable_cors: bool) -> Router {
// Directory operations
.route("/directories", get(list_directory))
.route("/directories", post(create_directory))
+ .route("/directories/ensure-parent", post(ensure_parent_dirs))
// Mount management
.route("/mounts", get(list_mounts))
.route("/mount", post(mount_filesystem))
.route("/unmount", post(unmount_filesystem))
// Search operations
.route("/grep", post(grep_content))
+ // Statistics
+ .route("/stats", get(get_stats))
// Health check
.route("/health", get(health_check));
diff --git a/openviking/pyagfs/client.py b/openviking/pyagfs/client.py
index 645368068..d9c647ed1 100644
--- a/openviking/pyagfs/client.py
+++ b/openviking/pyagfs/client.py
@@ -292,6 +292,19 @@ class AGFSClient:
except Exception as e:
self._handle_request_error(e)
+ def ensure_parent_dirs(self, path: str, mode: str = "755") -> Dict[str, Any]:
+ """Ensure all parent directories exist for the given path"""
+ try:
+ response = self.session.post(
+ f"{self.api_base}/directories/ensure-parent",
+ params={"path": path, "mode": mode},
+ timeout=self.timeout,
+ )
+ response.raise_for_status()
+ return response.json()
+ except Exception as e:
+ self._handle_request_error(e)
+
def rm(self, path: str, recursive: bool = False, force: bool = True) -> Dict[str, Any]:
"""Remove a file or directory.
@@ -826,6 +839,30 @@ class AGFSClient:
except Exception as e:
self._handle_request_error(e)
+ def get_stats(self, path: Optional[str] = None) -> Dict[str, Any]:
+ """Get filesystem statistics
+
+ Args:
+ path: Optional mount path to get stats for. If None, get stats for all mounts.
+
+ Returns:
+ Statistics data
+ """
+ try:
+ params = {}
+ if path is not None:
+ params["path"] = path
+ response = self.session.get(
+ f"{self.api_base}/stats",
+ params=params,
+ timeout=self.timeout,
+ )
+ response.raise_for_status()
+ data = response.json()
+ return data.get("data", {})
+ except Exception as e:
+ self._handle_request_error(e)
+
class FileHandle:
"""A file handle for stateful file operations
diff --git a/openviking/server/routers/observer.py b/openviking/server/routers/observer.py
index e2c626c25..f88a0804c 100644
--- a/openviking/server/routers/observer.py
+++ b/openviking/server/routers/observer.py
@@ -92,6 +92,16 @@ async def observer_retrieval(
return Response(status="ok", result=_component_to_dict(component))
[email protected]("/filesystem")
+async def observer_filesystem(
+ _ctx: RequestContext = Depends(get_request_context),
+):
+ """Get filesystem operation metrics."""
+ service = get_service()
+ component = service.debug.observer.filesystem
+ return Response(status="ok", result=_component_to_dict(component))
+
+
@router.get("/system")
async def observer_system(
ctx: RequestContext = Depends(get_request_context),
diff --git a/openviking/service/core.py b/openviking/service/core.py
index a39f1f42e..2792143f1 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -361,6 +361,7 @@ class OpenVikingService:
self._debug_service.set_dependencies(
vikingdb=self._vikingdb_manager,
config=self._config,
+ agfs_client=self._agfs_client,
)
self._initialized = True
diff --git a/openviking/service/debug_service.py b/openviking/service/debug_service.py
index 955f97d61..d0a8f8625 100644
--- a/openviking/service/debug_service.py
+++ b/openviking/service/debug_service.py
@@ -5,11 +5,12 @@ Debug Service - provides system status query and health check.
"""
from dataclasses import dataclass
-from typing import Dict, List, Optional
+from typing import Any, Dict, List, Optional
from openviking.server.identity import RequestContext
from openviking.storage import VikingDBManager
from openviking.storage.observers import (
+ FilesystemObserver,
LockObserver,
ModelsObserver,
QueueObserver,
@@ -19,6 +20,9 @@ from openviking.storage.observers import (
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.transaction import get_lock_manager
from openviking_cli.utils.config import OpenVikingConfig
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
@dataclass
@@ -62,18 +66,23 @@ class ObserverService:
self,
vikingdb: Optional[VikingDBManager] = None,
config: Optional[OpenVikingConfig] = None,
+ agfs_client: Optional[Any] = None,
):
self._vikingdb = vikingdb
self._config = config
+ self._agfs_client = agfs_client
def set_dependencies(
self,
vikingdb: VikingDBManager,
config: OpenVikingConfig,
+ agfs_client: Optional[Any] = None,
) -> None:
"""Set dependencies after initialization."""
self._vikingdb = vikingdb
self._config = config
+ if agfs_client is not None:
+ self._agfs_client = agfs_client
@property
def _dependencies_ready(self) -> bool:
@@ -185,6 +194,43 @@ class ObserverService:
status=observer.get_status_table(),
)
+ @property
+ def filesystem(self) -> ComponentStatus:
+ """Get filesystem operation status."""
+ observer = FilesystemObserver()
+ return ComponentStatus(
+ name="filesystem",
+ is_healthy=observer.is_healthy(),
+ has_errors=observer.has_errors(),
+ status=observer.get_status_table(),
+ )
+
+ async def get_filesystem_stats(self, mount_path: Optional[str] = None) -> dict:
+ """
+ Get filesystem statistics from RAGFS.
+
+ Args:
+ mount_path: Optional specific mount path.
+
+ Returns:
+ Statistics data.
+ """
+ try:
+ if self._agfs_client is None:
+ logger.debug("RAGFS client not available, returning empty stats")
+ return {}
+
+ # Call get_stats on the RAGFS client
+ import asyncio
+ stats = await asyncio.to_thread(
+ self._agfs_client.get_stats,
+ mount_path
+ )
+ return stats
+ except Exception as e:
+ logger.error(f"Error getting filesystem stats: {e}")
+ return {}
+
def system(self, ctx: Optional[RequestContext] = None) -> SystemStatus:
"""Get system overall status."""
components = {
@@ -193,6 +239,7 @@ class ObserverService:
"models": self.models,
"lock": self.lock,
"retrieval": self.retrieval,
+ "filesystem": self.filesystem,
}
errors = [f"{c.name} has errors" for c in components.values() if c.has_errors]
return SystemStatus(
@@ -215,16 +262,18 @@ class DebugService:
self,
vikingdb: Optional[VikingDBManager] = None,
config: Optional[OpenVikingConfig] = None,
+ agfs_client: Optional[Any] = None,
):
- self._observer = ObserverService(vikingdb, config)
+ self._observer = ObserverService(vikingdb, config, agfs_client)
def set_dependencies(
self,
vikingdb: VikingDBManager,
config: OpenVikingConfig,
+ agfs_client: Optional[Any] = None,
) -> None:
"""Set dependencies after initialization."""
- self._observer.set_dependencies(vikingdb, config)
+ self._observer.set_dependencies(vikingdb, config, agfs_client)
@property
def observer(self) -> ObserverService:
diff --git a/openviking/storage/observers/__init__.py b/openviking/storage/observers/__init__.py
index 94448c0d7..b95548026 100644
--- a/openviking/storage/observers/__init__.py
+++ b/openviking/storage/observers/__init__.py
@@ -1,6 +1,7 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
from .base_observer import BaseObserver
+from .filesystem_observer import FilesystemObserver
from .lock_observer import LockObserver
from .models_observer import ModelsObserver
from .queue_observer import QueueObserver
@@ -9,6 +10,7 @@ from .vikingdb_observer import VikingDBObserver
__all__ = [
"BaseObserver",
+ "FilesystemObserver",
"LockObserver",
"ModelsObserver",
"QueueObserver",
diff --git a/openviking/storage/observers/filesystem_observer.py b/openviking/storage/observers/filesystem_observer.py
new file mode 100644
index 000000000..2ca97ec84
--- /dev/null
+++ b/openviking/storage/observers/filesystem_observer.py
@@ -0,0 +1,161 @@
+"""
+FilesystemObserver: Filesystem observability tool.
+
+Provides methods to observe and report filesystem operation statistics.
+"""
+
+from typing import Dict, Optional
+
+from openviking.storage.observers.base_observer import BaseObserver
+from openviking_cli.utils import run_async
+from openviking_cli.utils.logger import get_logger
+
+logger = get_logger(__name__)
+
+
+class FilesystemObserver(BaseObserver):
+ """
+ FilesystemObserver: System observability tool for filesystem operations.
+
+ Reads accumulated statistics from RAGFS and formats them for display.
+ """
+
+ def __init__(self, mount_path: Optional[str] = None):
+ """
+ Initialize the filesystem observer.
+
+ Args:
+ mount_path: Optional specific mount path to observe. If None,
+ observe all mounts.
+ """
+ self.mount_path = mount_path
+
+ @staticmethod
+ def _get_collector():
+ """
+ Get the stats collector from the service.
+
+ Lazy import to avoid circular dependencies.
+ """
+ from openviking.server.dependencies import get_service
+ service = get_service()
+ return service.debug.observer
+
+ def _format_stats_table(self, stats_data: dict) -> str:
+ """
+ Format filesystem statistics as a string table.
+
+ Args:
+ stats_data: The statistics data from RAGFS.
+
+ Returns:
+ Formatted table string.
+ """
+ from tabulate import tabulate
+
+ if not stats_data:
+ return "No filesystem statistics available."
+
+ # Handle both single mount and all mounts cases
+ if "path" in stats_data:
+ # Single mount
+ mounts = [stats_data]
+ else:
+ # All mounts
+ mounts = stats_data.get("mounts", [])
+
+ if not mounts:
+ return "No filesystem statistics available."
+
+ result = []
+ # Sort mounts by path for consistent output
+ mounts = sorted(mounts, key=lambda m: m.get("path", ""))
+ for mount in mounts:
+ mount_path = mount.get("path", "unknown")
+ plugin_name = mount.get("plugin", "unknown")
+ stats = mount.get("stats", {})
+ operations = stats.get("operations", {})
+
+ result.append(f"Mount: {mount_path} (plugin: {plugin_name})")
+ result.append("=" * 60)
+
+ # Prepare operation statistics
+ table_data = []
+ total_ops = 0
+ total_time_us = 0
+
+ for op_name, op_stats in operations.items():
+ if op_stats:
+ count = op_stats.get("count", 0)
+ total_ops += count
+ total_time_us += op_stats.get("total_time_us", 0)
+
+ avg_us = op_stats.get("avg_time_us", 0.0)
+ min_us = op_stats.get("min_time_us", 0)
+ max_us = op_stats.get("max_time_us", 0)
+
+ if count > 0:
+ table_data.append({
+ "Operation": op_name,
+ "Count": count,
+ "Avg (ms)": f"{avg_us / 1000:.3f}",
+ "Min (ms)": f"{min_us / 1000:.3f}",
+ "Max (ms)": f"{max_us / 1000:.3f}",
+ })
+
+ if table_data:
+ result.append(tabulate(table_data, headers="keys", tablefmt="pretty"))
+ result.append("")
+
+ # Summary
+ avg_total_us = total_time_us / total_ops if total_ops > 0 else 0
+ summary = [
+ {"Metric": "Total Operations", "Value": total_ops},
+ {"Metric": "Total Time (s)", "Value": f"{total_time_us / 1000000:.3f}"},
+ {"Metric": "Overall Avg (ms)", "Value": f"{avg_total_us / 1000:.3f}"},
+ ]
+ result.append(tabulate(summary, headers="keys", tablefmt="pretty"))
+ else:
+ result.append("No operation statistics recorded yet.")
+
+ result.append("\n")
+
+ return "\n".join(result)
+
+ def get_status_table(self) -> str:
+ """
+ Format filesystem statistics as a string table.
+
+ Returns:
+ Formatted table string.
+ """
+ try:
+ observer_service = self._get_collector()
+ stats_data = run_async(observer_service.get_filesystem_stats(self.mount_path))
+ return self._format_stats_table(stats_data)
+ except Exception as e:
+ logger.error(f"Error getting filesystem stats: {e}")
+ return f"Error retrieving filesystem statistics: {e}"
+
+ def __str__(self) -> str:
+ return self.get_status_table()
+
+ def is_healthy(self) -> bool:
+ """
+ Check if the filesystem is healthy.
+
+ Returns:
+ True if filesystem is healthy, False otherwise.
+ """
+ # For now, always return True unless we have specific error conditions
+ return True
+
+ def has_errors(self) -> bool:
+ """
+ Check if there are filesystem errors.
+
+ Returns:
+ True if there are errors, False otherwise.
+ """
+ # For now, always return False unless we have specific error conditions
+ return False
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 3b5c8c21c..36f31acbe 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -2005,19 +2005,11 @@ class VikingFS:
async def _ensure_parent_dirs(self, path: str) -> None:
"""Recursively create all parent directories."""
- # Remove leading slash if present, then split
- parts = path.lstrip("/").split("/")
- # If it's a file path (not just a directory), we need to create parent directories
- # We create directories up to the last component (which might be a file)
- for i in range(1, len(parts)):
- parent = "/" + "/".join(parts[:i])
- try:
- await self._run_in_threadpool(self.agfs.mkdir, parent)
- except Exception as e:
- # Log the error but continue, as parent might already exist
- # or we might be creating it in the next iteration
- if "exist" not in str(e).lower() and "already" not in str(e).lower():
- logger.debug(f"Failed to create parent directory {parent}: {e}")
+ try:
+ await self._run_in_threadpool(self.agfs.ensure_parent_dirs, path)
+ except Exception as e:
+ # Log the error but continue
+ logger.debug(f"Failed to ensure parent directories for {path}: {e}")
# ========== Relation Table Internal Methods ==========
diff --git a/tests/misc/test_mkdir.py b/tests/misc/test_mkdir.py
index d62e0efff..58af5e23f 100644
--- a/tests/misc/test_mkdir.py
+++ b/tests/misc/test_mkdir.py
@@ -48,7 +48,7 @@ class TestMkdir:
"""mkdir(exist_ok=True) should return early if directory exists."""
fs = _make_viking_fs()
fs._ensure_parent_dirs = AsyncMock()
- fs.stat = AsyncMock(return_value={"type": "directory"})
+ fs.stat = AsyncMock(return_value={"isDir": True})
await fs.mkdir("viking://resources/existing_dir", exist_ok=True)