Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(storage): optimize tree func (#2372)

baojun-zhang073e929a1a8c
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs
index e5fa3da34..b7abdb3e4 100644
--- a/crates/ragfs-python/src/lib.rs
+++ b/crates/ragfs-python/src/lib.rs
@@ -13,7 +13,7 @@ use std::time::UNIX_EPOCH;
use ragfs::core::{
ConfigValue, FileInfo, FileSystem, FilesystemStats, FsOperation, GrepResult, MountableFS,
- OperationStats, PluginConfig, WriteFlag,
+ OperationStats, PluginConfig, TreeEntry, WriteFlag,
};
#[cfg(feature = "s3")]
use ragfs::plugins::S3FSPlugin;
@@ -100,6 +100,62 @@ fn file_info_to_py_dict(py: Python<'_>, info: &FileInfo) -> PyResult<Py<PyDict>>
Ok(dict.into())
}
+/// Convert TreeEntry to a Python dict:
+/// {"path": str, "rel_path": str, "info": dict, "extra": dict}
+fn tree_entry_to_py_dict(py: Python<'_>, entry: &TreeEntry) -> PyResult<Py<PyDict>> {
+ let dict = PyDict::new(py);
+ dict.set_item("path", &entry.path)?;
+ dict.set_item("rel_path", &entry.rel_path)?;
+ dict.set_item("info", file_info_to_py_dict(py, &entry.info)?)?;
+
+ let extra_dict = PyDict::new(py);
+ for (k, v) in &entry.extra {
+ let py_val: Py<PyAny> = serde_json_to_py(py, v)?;
+ extra_dict.set_item(k, py_val)?;
+ }
+ dict.set_item("extra", extra_dict)?;
+
+ Ok(dict.into())
+}
+
+/// Convert a serde_json::Value to a Python object.
+fn serde_json_to_py(py: Python<'_>, val: &serde_json::Value) -> PyResult<Py<PyAny>> {
+ match val {
+ serde_json::Value::Null => Ok(py.None()),
+ serde_json::Value::Bool(b) => {
+ let val: bool = *b;
+ let bound = val
+ .into_pyobject(py)
+ .map_err(|e| PyRuntimeError::new_err(e.to_string()))?;
+ Ok(bound.as_any().clone().unbind())
+ }
+ serde_json::Value::Number(n) => {
+ if let Some(i) = n.as_i64() {
+ Ok(i.into_pyobject(py)?.into_any().unbind())
+ } else if let Some(f) = n.as_f64() {
+ Ok(f.into_pyobject(py)?.into_any().unbind())
+ } else {
+ Ok(py.None())
+ }
+ }
+ serde_json::Value::String(s) => Ok(s.into_pyobject(py)?.into_any().unbind()),
+ serde_json::Value::Array(arr) => {
+ let list = PyList::empty(py);
+ for item in arr {
+ list.append(serde_json_to_py(py, item)?)?;
+ }
+ Ok(list.into())
+ }
+ serde_json::Value::Object(obj) => {
+ let d = PyDict::new(py);
+ for (k, v) in obj {
+ d.set_item(k, serde_json_to_py(py, v)?)?;
+ }
+ Ok(d.into())
+ }
+ }
+}
+
/// Convert GrepResult to a Python dict matching the Go binding JSON format:
/// {"matches": [{"file": str, "line": int, "content": str}, ...], "count": int}
fn grep_result_to_py_dict(py: Python<'_>, result: &GrepResult) -> PyResult<Py<PyDict>> {
@@ -647,6 +703,46 @@ impl RAGFSBindingClient {
})
}
+ /// Recursively traverse a directory tree.
+ ///
+ /// Args:
+ /// path: The root path of the traversal
+ /// show_hidden: Whether to include hidden files (default: False)
+ /// node_limit: Maximum number of nodes to return (default: None, no limit)
+ /// level_limit: Maximum depth relative to query root (default: None, no limit)
+ ///
+ /// Returns:
+ /// A list of dicts, each with keys: path, rel_path, info, extra
+ #[pyo3(signature = (path, show_hidden=false, node_limit=None, level_limit=None))]
+ fn tree_directory(
+ &self,
+ path: String,
+ show_hidden: bool,
+ node_limit: Option<i32>,
+ level_limit: Option<i32>,
+ ) -> PyResult<Py<PyAny>> {
+ let fs = self.fs.clone();
+ let limit = node_limit.map(|n| if n < 0 { 0 } else { n as usize });
+ let level_limit_usize = level_limit.map(|n| if n < 0 { 0 } else { n as usize });
+
+ let entries = self
+ .rt
+ .block_on(async move {
+ fs.tree_directory(&path, show_hidden, limit, level_limit_usize)
+ .await
+ })
+ .map_err(to_py_err)?;
+
+ Python::attach(|py| {
+ let list = PyList::empty(py);
+ for entry in &entries {
+ let dict = tree_entry_to_py_dict(py, entry)?;
+ list.append(dict)?;
+ }
+ Ok(list.into())
+ })
+ }
+
/// Calculate file digest (not yet implemented in ragfs).
#[pyo3(signature = (path, algorithm="xxh3"))]
fn digest(&self, path: String, algorithm: &str) -> PyResult<HashMap<String, String>> {
diff --git a/crates/ragfs/src/core/filesystem.rs b/crates/ragfs/src/core/filesystem.rs
index 7b07b9e8c..cdd0e8fd8 100644
--- a/crates/ragfs/src/core/filesystem.rs
+++ b/crates/ragfs/src/core/filesystem.rs
@@ -8,13 +8,13 @@ use async_trait::async_trait;
use regex::Regex;
use super::errors::{Error, Result};
-use super::types::{FileInfo, GrepResult, WriteFlag};
+use super::types::{FileInfo, GrepResult, TreeEntry, WriteFlag};
/// Normalize a path for prefix comparisons.
///
/// - Keeps "/" as-is.
/// - Strips trailing slashes for non-root paths (so "/a" and "/a/" behave the same).
-fn normalize_prefix_path(path: &str) -> String {
+pub(crate) fn normalize_prefix_path(path: &str) -> String {
if path == "/" {
"/".to_string()
} else {
@@ -39,7 +39,7 @@ fn is_excluded_path(path: &str, exclude_path: &str) -> bool {
/// Contract:
/// - "." means query root itself.
/// - otherwise returns a relative path without leading "/".
-fn relative_match_file(query_root: &str, path: &str) -> String {
+pub(crate) fn relative_match_file(query_root: &str, path: &str) -> String {
let base = normalize_prefix_path(query_root);
if path == base {
return ".".to_string();
@@ -66,7 +66,7 @@ fn relative_match_file(query_root: &str, path: &str) -> String {
///
/// - "." and "" => 0
/// - "a/b" => 2
-fn relative_depth(rel: &str) -> usize {
+pub(crate) fn relative_depth(rel: &str) -> usize {
if rel.is_empty() || rel == "." {
0
} else {
@@ -397,6 +397,119 @@ pub trait FileSystem: Send + Sync {
Ok(())
}
+ /// Recursively traverse a directory tree, returning a flat list of TreeEntry nodes.
+ ///
+ /// This is the public entry point. The default implementation uses recursive
+ /// `read_dir()` calls. Plugins (e.g. s3fs) may override this for better performance.
+ ///
+ /// # Arguments
+ /// * `path` - The root path of the traversal
+ /// * `show_hidden` - Whether to include hidden files (names starting with `.`)
+ /// * `node_limit` - Maximum number of nodes to return (None means no limit)
+ /// * `level_limit` - Maximum depth relative to query root (None means no limit)
+ ///
+ /// # Returns
+ /// A flat `Vec<TreeEntry>` in DFS order (directories before their children).
+ async fn tree_directory(
+ &self,
+ path: &str,
+ show_hidden: bool,
+ node_limit: Option<usize>,
+ level_limit: Option<usize>,
+ ) -> Result<Vec<TreeEntry>> {
+ let normalized_path = normalize_prefix_path(path);
+ let mut result = Vec::new();
+
+ self.tree_directory_internal(
+ normalized_path.as_str(),
+ normalized_path.as_str(),
+ show_hidden,
+ node_limit,
+ level_limit,
+ &mut result,
+ )
+ .await?;
+
+ Ok(result)
+ }
+
+ /// Internal recursive helper for tree_directory.
+ ///
+ /// # Arguments
+ /// * `base_path` - The original query root (used for rel_path computation)
+ /// * `current_path` - The current directory being traversed
+ /// * `show_hidden` - Whether to include hidden files
+ /// * `node_limit` - Maximum number of nodes to return
+ /// * `level_limit` - Maximum depth relative to base_path
+ /// * `result` - Accumulator for the flat result list
+ async fn tree_directory_internal(
+ &self,
+ base_path: &str,
+ current_path: &str,
+ show_hidden: bool,
+ node_limit: Option<usize>,
+ level_limit: Option<usize>,
+ result: &mut Vec<TreeEntry>,
+ ) -> Result<()> {
+ if node_limit.is_some_and(|limit| result.len() >= limit) {
+ return Ok(());
+ }
+
+ if let Some(limit) = level_limit {
+ let current_rel = relative_match_file(base_path, current_path);
+ let current_depth = relative_depth(&current_rel);
+ // `current_depth >= limit` is a pre-expansion guard: stop before
+ // reading the directory. Equivalent to S3FS's post-generation
+ // `depth > limit` filter — both ensure entries at depth > limit
+ // are excluded.
+ if current_depth >= limit {
+ return Ok(());
+ }
+ }
+
+ let entries = self.read_dir(current_path).await?;
+
+ for entry in entries {
+ if node_limit.is_some_and(|limit| result.len() >= limit) {
+ break;
+ }
+
+ let is_hidden_file = !entry.is_dir && entry.name.starts_with('.');
+ if is_hidden_file && !show_hidden {
+ continue;
+ }
+
+ let entry_path = if current_path == "/" {
+ format!("/{}", entry.name)
+ } else {
+ format!("{}/{}", current_path, entry.name)
+ };
+
+ let rel_path = relative_match_file(base_path, &entry_path);
+
+ result.push(TreeEntry {
+ path: entry_path.clone(),
+ rel_path,
+ info: entry.clone(),
+ extra: std::collections::HashMap::new(),
+ });
+
+ if entry.is_dir {
+ self.tree_directory_internal(
+ base_path,
+ &entry_path,
+ show_hidden,
+ node_limit,
+ level_limit,
+ result,
+ )
+ .await?;
+ }
+ }
+
+ Ok(())
+ }
+
/// Ensure all parent directories exist for the given path
///
/// # Arguments
@@ -652,4 +765,246 @@ mod tests {
assert_eq!(out.count, 1);
assert_eq!(out.matches[0].file, "a.txt");
}
+
+ // ── tree_directory default implementation tests ──
+
+ fn tree_names(entries: &[TreeEntry]) -> Vec<String> {
+ entries.iter().map(|e| e.info.name.clone()).collect()
+ }
+
+ fn tree_paths(entries: &[TreeEntry]) -> Vec<String> {
+ entries.iter().map(|e| e.path.clone()).collect()
+ }
+
+ fn tree_rel_paths(entries: &[TreeEntry]) -> Vec<String> {
+ entries.iter().map(|e| e.rel_path.clone()).collect()
+ }
+
+ #[tokio::test]
+ async fn test_tree_empty_dir() {
+ let fs = TreeFS::default().with_dir_entries("/root", vec![]);
+
+ let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+
+ assert!(entries.is_empty(), "empty dir returns empty vec");
+ }
+
+ #[tokio::test]
+ async fn test_tree_single_layer_files() {
+ let fs = TreeFS::default().with_dir_entries(
+ "/root",
+ vec![("a.txt", false), ("b.txt", false), ("c.txt", false)],
+ );
+
+ let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+
+ assert_eq!(entries.len(), 3);
+ assert_eq!(tree_names(&entries), vec!["a.txt", "b.txt", "c.txt"]);
+ assert_eq!(tree_rel_paths(&entries), vec!["a.txt", "b.txt", "c.txt"]);
+ }
+
+ #[tokio::test]
+ async fn test_tree_nested_dfs_order() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/root", vec![("a.txt", false), ("sub", true)])
+ .with_dir_entries("/root/sub", vec![("b.txt", false)]);
+
+ let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+
+ assert_eq!(entries.len(), 3);
+ assert_eq!(tree_names(&entries), vec!["a.txt", "sub", "b.txt"]);
+ assert_eq!(tree_rel_paths(&entries), vec!["a.txt", "sub", "sub/b.txt"]);
+ assert_eq!(
+ tree_paths(&entries),
+ vec!["/root/a.txt", "/root/sub", "/root/sub/b.txt"]
+ );
+ assert_eq!(entries[0].info.mode, 0o644);
+ assert!(!entries[0].info.is_dir);
+ assert_eq!(entries[1].info.mode, 0o755);
+ assert!(entries[1].info.is_dir);
+ assert!(entries.iter().all(|entry| entry.extra.is_empty()));
+ }
+
+ #[tokio::test]
+ async fn test_tree_node_limit() {
+ let fs = TreeFS::default().with_dir_entries(
+ "/root",
+ vec![
+ ("a.txt", false),
+ ("b.txt", false),
+ ("c.txt", false),
+ ("d.txt", false),
+ ("e.txt", false),
+ ],
+ );
+
+ let entries = fs
+ .tree_directory("/root", false, Some(3), None)
+ .await
+ .unwrap();
+
+ assert_eq!(entries.len(), 3);
+ assert_eq!(tree_names(&entries), vec!["a.txt", "b.txt", "c.txt"]);
+ }
+
+ #[tokio::test]
+ async fn test_tree_node_limit_zero() {
+ let fs = TreeFS::default().with_dir_entries("/root", vec![("a.txt", false)]);
+
+ let entries = fs
+ .tree_directory("/root", false, Some(0), None)
+ .await
+ .unwrap();
+
+ assert!(entries.is_empty(), "node_limit=0 returns empty");
+ }
+
+ #[tokio::test]
+ async fn test_tree_level_limit_zero() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/root", vec![("a.txt", false), ("sub", true)])
+ .with_dir_entries("/root/sub", vec![("b.txt", false)]);
+
+ let entries = fs
+ .tree_directory("/root", false, None, Some(0))
+ .await
+ .unwrap();
+
+ assert!(
+ entries.is_empty(),
+ "level_limit=0 returns empty (root not expanded)"
+ );
+ }
+
+ #[tokio::test]
+ async fn test_tree_level_limit_one() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/root", vec![("a.txt", false), ("sub", true)])
+ .with_dir_entries("/root/sub", vec![("b.txt", false)]);
+
+ let entries = fs
+ .tree_directory("/root", false, None, Some(1))
+ .await
+ .unwrap();
+
+ assert_eq!(entries.len(), 2);
+ assert_eq!(tree_names(&entries), vec!["a.txt", "sub"]);
+ }
+
+ #[tokio::test]
+ async fn test_tree_level_limit_none() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/root", vec![("a.txt", false), ("sub", true)])
+ .with_dir_entries("/root/sub", vec![("b.txt", false), ("deep", true)])
+ .with_dir_entries("/root/sub/deep", vec![("c.txt", false)]);
+
+ let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+
+ assert_eq!(entries.len(), 5);
+ assert!(tree_paths(&entries).contains(&"/root/sub/deep/c.txt".to_string()));
+ }
+
+ #[tokio::test]
+ async fn test_tree_show_hidden_false() {
+ let fs = TreeFS::default().with_dir_entries(
+ "/root",
+ vec![("a.txt", false), (".hidden", false), ("visible", true)],
+ );
+
+ let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+
+ let names = tree_names(&entries);
+ assert!(names.contains(&"a.txt".to_string()));
+ assert!(names.contains(&"visible".to_string()));
+ assert!(!names.contains(&".hidden".to_string()));
+ }
+
+ #[tokio::test]
+ async fn test_tree_show_hidden_false_dir_preserved() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/root", vec![(".hidden_dir", true)])
+ .with_dir_entries("/root/.hidden_dir", vec![("secret.txt", false)]);
+
+ let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+
+ assert_eq!(entries.len(), 2);
+ assert_eq!(tree_names(&entries), vec![".hidden_dir", "secret.txt"]);
+ }
+
+ #[tokio::test]
+ async fn test_tree_show_hidden_true() {
+ let fs = TreeFS::default().with_dir_entries(
+ "/root",
+ vec![("a.txt", false), (".hidden", false), (".hidden_dir", true)],
+ );
+
+ let entries = fs.tree_directory("/root", true, None, None).await.unwrap();
+
+ let names = tree_names(&entries);
+ assert!(names.contains(&"a.txt".to_string()));
+ assert!(names.contains(&".hidden".to_string()));
+ assert!(names.contains(&".hidden_dir".to_string()));
+ }
+
+ #[tokio::test]
+ async fn test_tree_root_path() {
+ let fs = TreeFS::default().with_dir_entries("/", vec![("a.txt", false), ("sub", true)]);
+
+ let entries = fs.tree_directory("/", false, None, None).await.unwrap();
+
+ assert_eq!(entries.len(), 2);
+ assert_eq!(entries[0].path, "/a.txt");
+ assert_eq!(entries[0].rel_path, "a.txt");
+ }
+
+ #[tokio::test]
+ async fn test_tree_deep_nesting() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/a", vec![("b", true)])
+ .with_dir_entries("/a/b", vec![("c", true)])
+ .with_dir_entries("/a/b/c", vec![("d", true)])
+ .with_dir_entries("/a/b/c/d", vec![("e", true)])
+ .with_dir_entries("/a/b/c/d/e", vec![("f.txt", false)]);
+
+ let entries = fs.tree_directory("/a", false, None, None).await.unwrap();
+
+ assert_eq!(entries.len(), 5);
+ assert_eq!(
+ tree_paths(&entries),
+ vec![
+ "/a/b",
+ "/a/b/c",
+ "/a/b/c/d",
+ "/a/b/c/d/e",
+ "/a/b/c/d/e/f.txt"
+ ]
+ );
+ }
+
+ #[tokio::test]
+ async fn test_tree_mixed_scenario() {
+ let fs = TreeFS::default()
+ .with_dir_entries(
+ "/root",
+ vec![
+ ("a.txt", false),
+ (".hidden_file", false),
+ ("sub", true),
+ (".hidden_dir", true),
+ ],
+ )
+ .with_dir_entries("/root/sub", vec![("b.txt", false)])
+ .with_dir_entries("/root/.hidden_dir", vec![("secret.txt", false)]);
+
+ let entries = fs.tree_directory("/root", false, None, None).await.unwrap();
+
+ assert_eq!(entries.len(), 5);
+ let names = tree_names(&entries);
+ assert!(names.contains(&"a.txt".to_string()));
+ assert!(names.contains(&"sub".to_string()));
+ assert!(names.contains(&"b.txt".to_string()));
+ assert!(names.contains(&".hidden_dir".to_string()));
+ assert!(names.contains(&"secret.txt".to_string()));
+ assert!(!names.contains(&".hidden_file".to_string()));
+ }
}
diff --git a/crates/ragfs/src/core/mod.rs b/crates/ragfs/src/core/mod.rs
index 58387dce4..19d9d49ef 100644
--- a/crates/ragfs/src/core/mod.rs
+++ b/crates/ragfs/src/core/mod.rs
@@ -22,4 +22,7 @@ pub use mountable::MountableFS;
pub use plugin::{HealthStatus, PluginRegistry, ServicePlugin};
pub use stats::{FilesystemStats, FsOperation, OperationStats, OperationTimer, StatsCollector};
pub use stats_wrapper::StatsWrappedFS;
-pub use types::{ConfigParameter, ConfigValue, FileInfo, GrepMatch, GrepResult, PluginConfig, WriteFlag};
+pub use types::{
+ ConfigParameter, ConfigValue, FileInfo, GrepMatch, GrepResult, PluginConfig, TreeEntry,
+ WriteFlag,
+};
diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs
index ca613d8aa..1262365f8 100644
--- a/crates/ragfs/src/core/mountable.rs
+++ b/crates/ragfs/src/core/mountable.rs
@@ -15,7 +15,7 @@ use super::filesystem::FileSystem;
use super::plugin::ServicePlugin;
use super::stats::{FilesystemStats, StatsCollector};
use super::stats_wrapper::StatsWrappedFS;
-use super::types::{FileInfo, GrepResult, PluginConfig, WriteFlag};
+use super::types::{FileInfo, GrepResult, PluginConfig, TreeEntry, WriteFlag};
/// Information about a mounted filesystem
#[derive(Clone)]
@@ -373,6 +373,39 @@ impl FileSystem for MountableFS {
)
.await
}
+
+ async fn tree_directory(
+ &self,
+ path: &str,
+ show_hidden: bool,
+ node_limit: Option<usize>,
+ level_limit: Option<usize>,
+ ) -> Result<Vec<TreeEntry>> {
+ let (mount_info, rel_path) = self.find_mount(path).await?;
+
+ let mount_prefix = if mount_info.path == "/" {
+ String::new()
+ } else {
+ mount_info.path.clone()
+ };
+
+ let mut entries = mount_info
+ .fs
+ .tree_directory(&rel_path, show_hidden, node_limit, level_limit)
+ .await?;
+
+ for entry in &mut entries {
+ if !mount_prefix.is_empty() {
+ entry.path = if entry.path == "/" {
+ mount_prefix.clone()
+ } else {
+ format!("{}{}", mount_prefix, entry.path)
+ };
+ }
+ }
+
+ Ok(entries)
+ }
}
#[cfg(test)]
@@ -383,12 +416,21 @@ mod tests {
// Mock filesystem for testing
struct MockFS {
name: String,
+ tree_entries: Vec<TreeEntry>,
}
impl MockFS {
fn new(name: &str) -> Self {
Self {
name: name.to_string(),
+ tree_entries: vec![],
+ }
+ }
+
+ fn with_tree_entries(name: &str, entries: Vec<TreeEntry>) -> Self {
+ Self {
+ name: name.to_string(),
+ tree_entries: entries,
}
}
}
@@ -456,17 +498,36 @@ mod tests {
out.add_match(path.to_string(), 1, pattern.to_string());
Ok(out)
}
+
+ async fn tree_directory(
+ &self,
+ _path: &str,
+ _show_hidden: bool,
+ _node_limit: Option<usize>,
+ _level_limit: Option<usize>,
+ ) -> Result<Vec<TreeEntry>> {
+ Ok(self.tree_entries.clone())
+ }
}
// Mock plugin for testing
struct MockPlugin {
name: String,
+ tree_entries: Option<Vec<TreeEntry>>,
}
impl MockPlugin {
fn new(name: &str) -> Self {
Self {
name: name.to_string(),
+ tree_entries: None,
+ }
+ }
+
+ fn with_tree_entries(name: &str, entries: Vec<TreeEntry>) -> Self {
+ Self {
+ name: name.to_string(),
+ tree_entries: Some(entries),
}
}
}
@@ -486,7 +547,14 @@ mod tests {
}
async fn initialize(&self, _config: PluginConfig) -> Result<Box<dyn FileSystem>> {
- Ok(Box::new(MockFS::new(&self.name)))
+ if let Some(ref entries) = self.tree_entries {
+ Ok(Box::new(MockFS::with_tree_entries(
+ &self.name,
+ entries.clone(),
+ )))
+ } else {
+ Ok(Box::new(MockFS::new(&self.name)))
+ }
}
fn config_params(&self) -> &[super::super::types::ConfigParameter] {
@@ -769,4 +837,83 @@ mod tests {
assert_eq!(result.matches[0].line, 1);
assert_eq!(result.matches[0].content, "foo");
}
+
+ fn make_tree_entry(path: &str, rel_path: &str, name: &str, is_dir: bool) -> TreeEntry {
+ TreeEntry {
+ path: path.to_string(),
+ rel_path: rel_path.to_string(),
+ info: FileInfo {
+ name: name.to_string(),
+ size: if is_dir { 0 } else { 100 },
+ mode: if is_dir { 0o755 } else { 0o644 },
+ mod_time: std::time::UNIX_EPOCH,
+ is_dir,
+ },
+ extra: HashMap::new(),
+ }
+ }
+
+ #[tokio::test]
+ async fn test_tree_directory_no_mount_returns_error() {
+ let mfs = MountableFS::new();
+ let result = mfs
+ .tree_directory("/nonexistent/subdir", false, None, None)
+ .await;
+ assert!(result.is_err());
+ }
+
+ #[tokio::test]
+ async fn test_tree_directory_path_rewriting() {
+ let mfs = MountableFS::new();
+ let plugin = MockPlugin::with_tree_entries(
+ "rewrite",
+ vec![make_tree_entry(
+ "/sub/file.txt",
+ "sub/file.txt",
+ "file.txt",
+ false,
+ )],
+ );
+ mfs.register_plugin(plugin).await;
+ mfs.mount(PluginConfig {
+ name: "rewrite".to_string(),
+ mount_path: "/rewrite".to_string(),
+ params: HashMap::new(),
+ })
+ .await
+ .unwrap();
+
+ let result = mfs
+ .tree_directory("/rewrite/sub", false, None, None)
+ .await
+ .unwrap();
+ assert_eq!(result.len(), 1);
+ assert_eq!(result[0].path, "/rewrite/sub/file.txt");
+ assert_eq!(result[0].rel_path, "sub/file.txt");
+ }
+
+ #[tokio::test]
+ async fn test_tree_directory_rel_path_unchanged_by_routing() {
+ let mfs = MountableFS::new();
+ let plugin = MockPlugin::with_tree_entries(
+ "relpath",
+ vec![make_tree_entry("/a/b/c.txt", "a/b/c.txt", "c.txt", false)],
+ );
+ mfs.register_plugin(plugin).await;
+ mfs.mount(PluginConfig {
+ name: "relpath".to_string(),
+ mount_path: "/local/test_account".to_string(),
+ params: HashMap::new(),
+ })
+ .await
+ .unwrap();
+
+ let result = mfs
+ .tree_directory("/local/test_account/a", false, None, None)
+ .await
+ .unwrap();
+ assert_eq!(result.len(), 1);
+ assert_eq!(result[0].path, "/local/test_account/a/b/c.txt");
+ assert_eq!(result[0].rel_path, "a/b/c.txt");
+ }
}
diff --git a/crates/ragfs/src/core/stats.rs b/crates/ragfs/src/core/stats.rs
index 7ef4b8973..0f36b52a9 100644
--- a/crates/ragfs/src/core/stats.rs
+++ b/crates/ragfs/src/core/stats.rs
@@ -40,6 +40,8 @@ pub enum FsOperation {
Grep,
/// Ensure parent directories exist
EnsureParentDirs,
+ /// Tree directory operation
+ TreeDir,
}
impl FsOperation {
@@ -60,6 +62,7 @@ impl FsOperation {
FsOperation::Exists,
FsOperation::Grep,
FsOperation::EnsureParentDirs,
+ FsOperation::TreeDir,
]
}
@@ -80,6 +83,7 @@ impl FsOperation {
FsOperation::Exists => "exists",
FsOperation::Grep => "grep",
FsOperation::EnsureParentDirs => "ensure_parent_dirs",
+ FsOperation::TreeDir => "tree_dir",
}
}
}
diff --git a/crates/ragfs/src/core/stats_wrapper.rs b/crates/ragfs/src/core/stats_wrapper.rs
index dd28e06fa..5837a7628 100644
--- a/crates/ragfs/src/core/stats_wrapper.rs
+++ b/crates/ragfs/src/core/stats_wrapper.rs
@@ -8,7 +8,7 @@ use std::sync::Arc;
use super::{
FileInfo, FileSystem, FsOperation, GrepResult, OperationTimer, Result, StatsCollector,
- WriteFlag,
+ TreeEntry, WriteFlag,
};
/// A wrapper around FileSystem that automatically collects operation statistics
@@ -162,4 +162,20 @@ impl FileSystem for StatsWrappedFS {
timer.finish().await;
result
}
+
+ async fn tree_directory(
+ &self,
+ path: &str,
+ show_hidden: bool,
+ node_limit: Option<usize>,
+ level_limit: Option<usize>,
+ ) -> Result<Vec<TreeEntry>> {
+ let timer = OperationTimer::start(FsOperation::TreeDir, Arc::clone(&self.stats));
+ let result = self
+ .inner
+ .tree_directory(path, show_hidden, node_limit, level_limit)
+ .await;
+ timer.finish().await;
+ result
+ }
}
diff --git a/crates/ragfs/src/core/types.rs b/crates/ragfs/src/core/types.rs
index 039409aa5..1a502c084 100644
--- a/crates/ragfs/src/core/types.rs
+++ b/crates/ragfs/src/core/types.rs
@@ -34,6 +34,29 @@ pub struct GrepResult {
pub count: usize,
}
+/// Tree traversal entry.
+///
+/// Represents one flattened node in a recursive directory traversal.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TreeEntry {
+ /// Internal trait contract: plugin-root-relative absolute path,
+ /// e.g. "/a/file.txt".
+ ///
+ /// External bindings contract: after `MountableFS.tree_directory()`
+ /// rewrites the mount prefix back, Python-visible `path` must be a
+ /// global AGFS absolute path such as "/local/{account}/resources/a/file.txt".
+ pub path: String,
+
+ /// Path relative to traversal root, e.g. "a/file.txt"
+ pub rel_path: String,
+
+ /// File metadata for this node.
+ pub info: FileInfo,
+
+ /// Backend-specific fields required for zero-regression original output.
+ pub extra: HashMap<String, serde_json::Value>,
+}
+
impl GrepResult {
/// Create a new empty GrepResult
pub fn new() -> Self {
diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rs
index 96049005e..f9c29b663 100644
--- a/crates/ragfs/src/plugins/s3fs/client.rs
+++ b/crates/ragfs/src/plugins/s3fs/client.rs
@@ -545,6 +545,61 @@ impl S3Client {
Ok(ListResult { files, directories })
}
+ /// List all objects under a prefix (flat listing, no delimiter).
+ /// Preserves directory marker objects (keys ending with '/').
+ /// Used by tree_directory for efficient flat traversal.
+ pub async fn list_tree_objects(&self, prefix: &str) -> Result<Vec<ObjectMeta>> {
+ let mut objects = Vec::new();
+ let mut continuation_token: Option<String> = None;
+
+ loop {
+ let mut req = self
+ .client
+ .list_objects_v2()
+ .bucket(&self.bucket)
+ .prefix(prefix);
+
+ if let Some(token) = &continuation_token {
+ req = req.continuation_token(token);
+ }
+
+ let resp = req
+ .send()
+ .await
+ .map_err(|e| Error::internal(format!("S3 ListObjectsV2 error: {}", e)))?;
+
+ for obj in resp.contents() {
+ let key = obj.key().unwrap_or("");
+ if key == prefix {
+ continue;
+ }
+
+ let size = obj.size.unwrap_or(0);
+ let last_modified = obj
+ .last_modified()
+ .map(aws_datetime_to_systemtime)
+ .unwrap_or(UNIX_EPOCH);
+
+ let is_dir_marker = key.ends_with('/');
+
+ objects.push(ObjectMeta {
+ key: key.to_string(),
+ size,
+ last_modified,
+ is_dir_marker,
+ });
+ }
+
+ if resp.is_truncated() == Some(true) {
+ continuation_token = resp.next_continuation_token().map(|s| s.to_string());
+ } else {
+ break;
+ }
+ }
+
+ Ok(objects)
+ }
+
/// Copy an object
pub async fn copy_object(&self, src_key: &str, dst_key: &str) -> Result<()> {
let copy_source = format!("{}/{}", self.bucket, src_key);
diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rs
index 8a8ba7e41..2aa82aaf1 100644
--- a/crates/ragfs/src/plugins/s3fs/mod.rs
+++ b/crates/ragfs/src/plugins/s3fs/mod.rs
@@ -15,6 +15,7 @@
pub mod cache;
pub mod client;
+mod tree;
use async_trait::async_trait;
use std::sync::{Arc, Mutex};
@@ -26,10 +27,12 @@ use futures::stream::{self, StreamExt};
use regex::Regex;
use std::sync::atomic::{AtomicUsize, Ordering};
+use crate::core::filesystem::{relative_depth, relative_match_file};
use crate::core::{
ConfigParameter, Error, FileInfo, FileSystem, GrepMatch, GrepResult, PluginConfig, Result,
- ServicePlugin, WriteFlag,
+ ServicePlugin, TreeEntry, WriteFlag,
};
+use tree::build_tree_entries_from_flat_listing;
/// Check whether `path` is under `exclude_path` (including itself).
fn s3_is_excluded_path(path: &str, exclude_path: &str) -> bool {
@@ -43,44 +46,6 @@ fn s3_is_excluded_path(path: &str, exclude_path: &str) -> bool {
.is_some_and(|suffix| suffix.starts_with('/'))
}
-/// Convert an absolute/plugin path to a query-root-relative grep match path.
-fn s3_relative_match_file(query_root: &str, path: &str) -> String {
- let base = if query_root == "/" {
- "/".to_string()
- } else {
- query_root.trim_end_matches('/').to_string()
- };
-
- if path == base {
- return ".".to_string();
- }
-
- if base == "/" {
- return path.trim_start_matches('/').to_string();
- }
-
- match path.strip_prefix(&base) {
- Some(rest) => {
- let rel = rest.trim_start_matches('/');
- if rel.is_empty() {
- ".".to_string()
- } else {
- rel.to_string()
- }
- }
- None => path.trim_start_matches('/').to_string(),
- }
-}
-
-/// Compute depth from a query-root-relative path.
-fn s3_relative_depth(rel: &str) -> usize {
- if rel.is_empty() || rel == "." {
- 0
- } else {
- rel.split('/').filter(|p| !p.is_empty()).count()
- }
-}
-
/// Abstract trait for reading chunks from a file during grep.
/// Extracted to allow unit testing chunk-boundary logic with mock providers.
#[async_trait]
@@ -306,7 +271,7 @@ impl S3FileSystem {
) -> Result<Vec<GrepMatch>> {
let normalized = Self::normalize_path(path);
let key = self.client.build_key(&normalized);
- let rel_file = s3_relative_match_file(base_path, path);
+ let rel_file = relative_match_file(base_path, path);
let mut reader = S3ChunkReader {
client: &self.client,
key: &key,
@@ -774,8 +739,8 @@ impl FileSystem for S3FileSystem {
}
}
if let Some(limit) = level_limit {
- let rel = s3_relative_match_file(&normalized, &normalized);
- if s3_relative_depth(&rel) > limit {
+ let rel = relative_match_file(&normalized, &normalized);
+ if relative_depth(&rel) > limit {
return Ok(GrepResult::new());
}
}
@@ -809,8 +774,8 @@ impl FileSystem for S3FileSystem {
}
if !recursive || level_limit.is_some() {
- let rel = s3_relative_match_file(&normalized, &fs_path);
- let depth = s3_relative_depth(&rel);
+ let rel = relative_match_file(&normalized, &fs_path);
+ let depth = relative_depth(&rel);
if !recursive && depth > 1 {
continue;
@@ -832,6 +797,42 @@ impl FileSystem for S3FileSystem {
self.grep_files_concurrent(&normalized, files, &re, node_limit)
.await
}
+
+ async fn tree_directory(
+ &self,
+ path: &str,
+ show_hidden: bool,
+ node_limit: Option<usize>,
+ level_limit: Option<usize>,
+ ) -> Result<Vec<TreeEntry>> {
+ let normalized = Self::normalize_path(path);
+
+ let prefix = if normalized == "/" {
+ self.client.build_key("")
+ } else {
+ format!("{}/", self.client.build_key(&normalized))
+ };
+
+ let objects = self.client.list_tree_objects(&prefix).await?;
+
+ let ordered = build_tree_entries_from_flat_listing(
+ &normalized,
+ &objects,
+ show_hidden,
+ level_limit,
+ |key| self.client.strip_prefix(key),
+ )?;
+
+ let mut result = Vec::new();
+ for entry in ordered {
+ if node_limit.is_some_and(|limit| result.len() >= limit) {
+ break;
+ }
+ result.push(entry);
+ }
+
+ Ok(result)
+ }
}
/// S3FS Plugin
@@ -1242,64 +1243,64 @@ mod tests {
}
#[test]
- fn test_s3_relative_match_file_same_path() {
- assert_eq!(s3_relative_match_file("/foo", "/foo"), ".");
- assert_eq!(s3_relative_match_file("/", "/"), ".");
+ fn test_relative_match_file_same_path() {
+ assert_eq!(relative_match_file("/foo", "/foo"), ".");
+ assert_eq!(relative_match_file("/", "/"), ".");
}
#[test]
- fn test_s3_relative_match_file_root_base() {
- assert_eq!(s3_relative_match_file("/", "/a"), "a");
- assert_eq!(s3_relative_match_file("/", "/a/b"), "a/b");
+ fn test_relative_match_file_root_base() {
+ assert_eq!(relative_match_file("/", "/a"), "a");
+ assert_eq!(relative_match_file("/", "/a/b"), "a/b");
assert_eq!(
- s3_relative_match_file("/", "/deep/nested/file"),
+ relative_match_file("/", "/deep/nested/file"),
"deep/nested/file"
);
}
#[test]
- fn test_s3_relative_match_file_subdirectory() {
- assert_eq!(s3_relative_match_file("/a", "/a/b"), "b");
- assert_eq!(s3_relative_match_file("/a", "/a/b/c"), "b/c");
- assert_eq!(s3_relative_match_file("/dir", "/dir/file.txt"), "file.txt");
+ fn test_relative_match_file_subdirectory() {
+ assert_eq!(relative_match_file("/a", "/a/b"), "b");
+ assert_eq!(relative_match_file("/a", "/a/b/c"), "b/c");
+ assert_eq!(relative_match_file("/dir", "/dir/file.txt"), "file.txt");
}
#[test]
- fn test_s3_relative_match_file_no_prefix_match() {
- assert_eq!(s3_relative_match_file("/a", "/b"), "b");
- assert_eq!(s3_relative_match_file("/x", "/y/z"), "y/z");
+ fn test_relative_match_file_no_prefix_match() {
+ assert_eq!(relative_match_file("/a", "/b"), "b");
+ assert_eq!(relative_match_file("/x", "/y/z"), "y/z");
}
#[test]
- fn test_s3_relative_match_file_trailing_slash_base() {
- assert_eq!(s3_relative_match_file("/foo/", "/foo/bar"), "bar");
- assert_eq!(s3_relative_match_file("/foo/", "/foo"), ".");
+ fn test_relative_match_file_trailing_slash_base() {
+ assert_eq!(relative_match_file("/foo/", "/foo/bar"), "bar");
+ assert_eq!(relative_match_file("/foo/", "/foo"), ".");
}
#[test]
- fn test_s3_relative_match_file_empty_remainder() {
- assert_eq!(s3_relative_match_file("/dir", "/dir/"), ".");
+ fn test_relative_match_file_empty_remainder() {
+ assert_eq!(relative_match_file("/dir", "/dir/"), ".");
}
#[test]
- fn test_s3_relative_match_file_leading_slash_in_rest() {
- assert_eq!(s3_relative_match_file("/dir", "/dir//sub"), "sub");
+ fn test_relative_match_file_leading_slash_in_rest() {
+ assert_eq!(relative_match_file("/dir", "/dir//sub"), "sub");
}
#[test]
- fn test_s3_relative_depth() {
- assert_eq!(s3_relative_depth("."), 0);
- assert_eq!(s3_relative_depth(""), 0);
- assert_eq!(s3_relative_depth("a"), 1);
- assert_eq!(s3_relative_depth("a/b"), 2);
- assert_eq!(s3_relative_depth("a/b/c"), 3);
- assert_eq!(s3_relative_depth("deep/nested/path/to/file"), 5);
+ fn test_relative_depth() {
+ assert_eq!(relative_depth("."), 0);
+ assert_eq!(relative_depth(""), 0);
+ assert_eq!(relative_depth("a"), 1);
+ assert_eq!(relative_depth("a/b"), 2);
+ assert_eq!(relative_depth("a/b/c"), 3);
+ assert_eq!(relative_depth("deep/nested/path/to/file"), 5);
}
#[test]
- fn test_s3_relative_depth_filters_empty_segments() {
- assert_eq!(s3_relative_depth("a//b"), 2);
- assert_eq!(s3_relative_depth("///"), 0);
+ fn test_relative_depth_filters_empty_segments() {
+ assert_eq!(relative_depth("a//b"), 2);
+ assert_eq!(relative_depth("///"), 0);
}
#[test]
diff --git a/crates/ragfs/src/plugins/s3fs/tree.rs b/crates/ragfs/src/plugins/s3fs/tree.rs
new file mode 100644
index 000000000..9dc8bec85
--- /dev/null
+++ b/crates/ragfs/src/plugins/s3fs/tree.rs
@@ -0,0 +1,325 @@
+//! S3FS tree construction from flat object listing.
+//!
+//! S3 has no native directory tree; `tree_directory` performs a single (or a
+//! few paginated) flat `list_objects` call and reconstructs the full directory
+//! tree in memory here. This module owns the reconstruction algorithm so that
+//! `mod.rs` only keeps routing and the `FileSystem` trait implementation.
+
+use std::collections::HashSet;
+use std::time::SystemTime;
+
+use super::client;
+use crate::core::filesystem::{relative_depth, relative_match_file};
+use crate::core::{FileInfo, Result, TreeEntry};
+
+/// Extract the last path component (file/dir name) from a path.
+fn s3_file_name(path: &str) -> String {
+ path.rsplit('/').next().unwrap_or("").to_string()
+}
+
+/// Build an ordered list of `TreeEntry` from a flat S3 object listing.
+///
+/// Reconstructs synthetic directories (from directory markers and from file
+/// parent prefixes), filters hidden files and entries beyond `level_limit`,
+/// and orders entries by path-component lexicographic order so the output is
+/// stable and consistent with the default DFS implementation.
+///
+/// `query_root` is the normalized query root path; `key_to_path` maps an S3
+/// object key to a filesystem path (allowing prefix stripping to be injected).
+pub(super) fn build_tree_entries_from_flat_listing<F>(
+ query_root: &str,
+ objects: &[client::ObjectMeta],
+ show_hidden: bool,
+ level_limit: Option<usize>,
+ key_to_path: F,
+) -> Result<Vec<TreeEntry>>
+where
+ F: Fn(&str) -> String,
+{
+ let mut seen_dirs: HashSet<String> = HashSet::new();
+ let mut entries: Vec<(String, bool, u64, SystemTime)> = Vec::new();
+
+ for obj in objects {
+ let raw_path = key_to_path(&obj.key);
+ let raw_path = if raw_path.starts_with('/') {
+ raw_path
+ } else {
+ format!("/{}", raw_path)
+ };
+
+ if obj.is_dir_marker {
+ let dir_path = raw_path.trim_end_matches('/').to_string();
+ if dir_path.is_empty() || dir_path == query_root {
+ continue;
+ }
+ if seen_dirs.insert(dir_path.clone()) {
+ entries.push((dir_path, true, 0, SystemTime::now()));
+ }
+ } else {
+ let file_path = raw_path;
+ entries.push((file_path.clone(), false, obj.size as u64, obj.last_modified));
+
+ // Recover intermediate directories *below the query root* from the
+ // file's query-root-relative path. Using the relative path (instead
+ // of the absolute filesystem path) guarantees that ancestors of the
+ // query root are never emitted as tree entries.
+ let rel = relative_match_file(query_root, &file_path);
+ let rel_parts: Vec<&str> = rel.split('/').filter(|p| !p.is_empty()).collect();
+ let base = if query_root == "/" { "" } else { query_root };
+ for i in 1..rel_parts.len() {
+ let dir_path = format!("{}/{}", base, rel_parts[..i].join("/"));
+ if seen_dirs.insert(dir_path.clone()) {
+ entries.push((dir_path, true, 0, SystemTime::now()));
+ }
+ }
+ }
+ }
+
+ let mut filtered: Vec<&(String, bool, u64, SystemTime)> = Vec::new();
+ for entry in &entries {
+ let (path, is_dir, _size, _mod_time) = entry;
+ let name = s3_file_name(path);
+
+ if !*is_dir && name.starts_with('.') && !show_hidden {
+ continue;
+ }
+
+ if let Some(limit) = level_limit {
+ let rel = relative_match_file(query_root, path);
+ let depth = relative_depth(&rel);
+ // S3FS uses `depth > limit` (post-generation filter) vs default
+ // impl's `current_depth >= limit` (pre-expansion guard). Both are
+ // semantically equivalent: level_limit=N means entries at depth
+ // <= N are included, entries at depth > N are excluded.
+ if depth > limit {
+ continue;
+ }
+ }
+
+ filtered.push(entry);
+ }
+
+ filtered.sort_by(|(a_path, ..), (b_path, ..)| {
+ let a_parts: Vec<&str> = a_path
+ .trim_start_matches('/')
+ .split('/')
+ .filter(|p| !p.is_empty())
+ .collect();
+ let b_parts: Vec<&str> = b_path
+ .trim_start_matches('/')
+ .split('/')
+ .filter(|p| !p.is_empty())
+ .collect();
+ a_parts.cmp(&b_parts)
+ });
+
+ let mut result = Vec::new();
+ for (path, is_dir, size, mod_time) in filtered {
+ let rel_path = relative_match_file(query_root, path);
+ let name = s3_file_name(path);
+
+ result.push(TreeEntry {
+ path: path.clone(),
+ rel_path,
+ info: FileInfo {
+ name,
+ size: *size,
+ mode: if *is_dir { 0o755 } else { 0o644 },
+ mod_time: *mod_time,
+ is_dir: *is_dir,
+ },
+ extra: std::collections::HashMap::new(),
+ });
+ }
+
+ Ok(result)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn make_meta(key: &str, size: i64, is_dir_marker: bool) -> client::ObjectMeta {
+ client::ObjectMeta {
+ key: key.to_string(),
+ size,
+ last_modified: std::time::SystemTime::now(),
+ is_dir_marker,
+ }
+ }
+
+ fn identity_key_to_path(key: &str) -> String {
+ format!("/{}", key)
+ }
+
+ #[test]
+ fn test_build_tree_entries_empty_listing() {
+ let objects: Vec<client::ObjectMeta> = vec![];
+ let result = build_tree_entries_from_flat_listing(
+ "/root",
+ &objects,
+ false,
+ None,
+ identity_key_to_path,
+ )
+ .unwrap();
+ assert!(result.is_empty());
+ }
+
+ #[test]
+ fn test_build_tree_entries_dir_from_marker() {
+ let objects = vec![make_meta("mydir/", 0, true)];
+ let result = build_tree_entries_from_flat_listing(
+ "/root",
+ &objects,
+ false,
+ None,
+ identity_key_to_path,
+ )
+ .unwrap();
+ assert_eq!(result.len(), 1);
+ let dir = &result[0];
+ assert!(dir.info.is_dir);
+ assert_eq!(dir.path, "/mydir");
+ assert_eq!(dir.rel_path, "mydir");
+ assert_eq!(dir.info.size, 0);
+ assert_eq!(dir.info.mode, 0o755);
+ }
+
+ #[test]
+ fn test_build_tree_entries_dir_dedup() {
+ let objects = vec![
+ make_meta("mydir/", 0, true),
+ make_meta("mydir/file.txt", 100, false),
+ ];
+ let result =
+ build_tree_entries_from_flat_listing("/", &objects, false, None, identity_key_to_path)
+ .unwrap();
+ assert_eq!(result.len(), 2);
+ assert!(result[0].info.is_dir);
+ assert!(result[1].info.is_dir == false);
+ }
+
+ #[test]
+ fn test_build_tree_entries_dfs_order() {
+ let objects = vec![
+ make_meta("a/a.txt", 100, false),
+ make_meta("a/b/c.txt", 200, false),
+ make_meta("a/b.txt", 150, false),
+ ];
+ let result =
+ build_tree_entries_from_flat_listing("/", &objects, false, None, identity_key_to_path)
+ .unwrap();
+ let paths: Vec<&str> = result.iter().map(|e| e.path.as_str()).collect();
+ assert_eq!(
+ paths,
+ vec!["/a", "/a/a.txt", "/a/b", "/a/b/c.txt", "/a/b.txt"]
+ );
+ }
+
+ #[test]
+ fn test_build_tree_entries_level_limit() {
+ let objects = vec![
+ make_meta("a.txt", 100, false),
+ make_meta("sub/b.txt", 200, false),
+ make_meta("sub/deep/c.txt", 300, false),
+ ];
+ let result = build_tree_entries_from_flat_listing(
+ "/",
+ &objects,
+ false,
+ Some(1),
+ identity_key_to_path,
+ )
+ .unwrap();
+ let paths: Vec<&str> = result.iter().map(|e| e.path.as_str()).collect();
+ assert_eq!(paths, vec!["/a.txt", "/sub"]);
+ }
+
+ #[test]
+ fn test_build_tree_entries_show_hidden_false() {
+ let objects = vec![
+ make_meta("a.txt", 100, false),
+ make_meta(".hidden", 50, false),
+ make_meta("sub/b.txt", 200, false),
+ ];
+ let result = build_tree_entries_from_flat_listing(
+ "/root",
+ &objects,
+ false,
+ None,
+ identity_key_to_path,
+ )
+ .unwrap();
+ let names: Vec<&str> = result.iter().map(|e| e.info.name.as_str()).collect();
+ assert!(names.contains(&"a.txt"));
+ assert!(names.contains(&"sub"));
+ assert!(names.contains(&"b.txt"));
+ assert!(!names.contains(&".hidden"));
+ }
+
+ #[test]
+ fn test_build_tree_entries_show_hidden_true() {
+ let objects = vec![
+ make_meta("a.txt", 100, false),
+ make_meta(".hidden", 50, false),
+ ];
+ let result = build_tree_entries_from_flat_listing(
+ "/root",
+ &objects,
+ true,
+ None,
+ identity_key_to_path,
+ )
+ .unwrap();
+ let names: Vec<&str> = result.iter().map(|e| e.info.name.as_str()).collect();
+ assert!(names.contains(&"a.txt"));
+ assert!(names.contains(&".hidden"));
+ }
+
+ #[test]
+ fn test_build_tree_entries_query_root_skipped() {
+ let objects = vec![make_meta("root/file.txt", 100, false)];
+ let result = build_tree_entries_from_flat_listing("/root", &objects, false, None, |key| {
+ format!("/{}", key)
+ })
+ .unwrap();
+ assert_eq!(result.len(), 1);
+ assert_eq!(result[0].path, "/root/file.txt");
+ }
+
+ #[test]
+ fn test_build_tree_entries_nested_dir_recovery() {
+ let objects = vec![make_meta("a/b/c/d/file.txt", 100, false)];
+ let result =
+ build_tree_entries_from_flat_listing("/", &objects, false, None, identity_key_to_path)
+ .unwrap();
+ let paths: Vec<&str> = result.iter().map(|e| e.path.as_str()).collect();
+ assert_eq!(
+ paths,
+ vec!["/a", "/a/b", "/a/b/c", "/a/b/c/d", "/a/b/c/d/file.txt"]
+ );
+ }
+
+ /// Regression test: when the query root has ancestor directories, those
+ /// ancestors must NOT leak into the tree result. Reproduces the bug where
+ /// `ov tree viking://resources/resource_xxx` showed parent entries such as
+ /// `tenant_xxx` and `tenant_xxx/resources`.
+ #[test]
+ fn test_build_tree_entries_no_ancestor_leak() {
+ // query_root is several levels deep; listed objects live under it.
+ let objects = vec![make_meta("tenant_x/resources/res_1/res_1.md", 171, false)];
+ let result = build_tree_entries_from_flat_listing(
+ "/tenant_x/resources/res_1",
+ &objects,
+ false,
+ None,
+ identity_key_to_path,
+ )
+ .unwrap();
+ let paths: Vec<&str> = result.iter().map(|e| e.path.as_str()).collect();
+ // Only the single file under the query root should appear; no ancestors
+ // (`/tenant_x`, `/tenant_x/resources`) and not the query root itself.
+ assert_eq!(paths, vec!["/tenant_x/resources/res_1/res_1.md"]);
+ }
+}
diff --git a/crates/ragfs/src/server/handlers.rs b/crates/ragfs/src/server/handlers.rs
index 2d7b67555..036c41180 100644
--- a/crates/ragfs/src/server/handlers.rs
+++ b/crates/ragfs/src/server/handlers.rs
@@ -82,6 +82,22 @@ pub struct StatsQuery {
pub path: Option<String>,
}
+/// Query parameters for tree (recursive directory listing) operations
+#[derive(Debug, Deserialize)]
+pub struct TreeQuery {
+ /// Directory path to traverse
+ pub path: String,
+ /// Whether to include hidden entries (names starting with '.')
+ #[serde(default)]
+ pub show_hidden: bool,
+ /// Maximum number of nodes to return (None = unlimited)
+ #[serde(default)]
+ pub node_limit: Option<usize>,
+ /// Maximum depth relative to the query root (None = unlimited)
+ #[serde(default)]
+ pub level_limit: Option<usize>,
+}
+
/// Statistics response for a single mount
#[derive(Debug, Serialize)]
pub struct MountStats {
@@ -452,6 +468,33 @@ pub async fn grep_content(State(state): State<AppState>, Json(req): Json<GrepReq
}
}
+/// GET /api/v1/tree - Recursively list a directory tree.
+///
+/// Mirrors the binding-side `tree_directory` so HTTP clients have the same
+/// surface as the native binding client.
+pub async fn tree_directory(
+ State(state): State<AppState>,
+ Query(query): Query<TreeQuery>,
+) -> Response {
+ match state
+ .fs
+ .tree_directory(
+ &query.path,
+ query.show_hidden,
+ query.node_limit,
+ query.level_limit,
+ )
+ .await
+ {
+ Ok(entries) => (StatusCode::OK, Json(ApiResponse::success(entries))).into_response(),
+ Err(e) => (
+ StatusCode::NOT_FOUND,
+ Json(ApiResponse::<()>::error(e.to_string())),
+ )
+ .into_response(),
+ }
+}
+
/// GET /api/v1/health - Health check
pub async fn health_check() -> Response {
let response = HealthResponse {
diff --git a/crates/ragfs/src/server/router.rs b/crates/ragfs/src/server/router.rs
index 44c5fc00f..379451013 100644
--- a/crates/ragfs/src/server/router.rs
+++ b/crates/ragfs/src/server/router.rs
@@ -15,7 +15,7 @@ use tracing::Level;
use super::handlers::{
create_directory, create_file, delete_file, ensure_parent_dirs, get_stats, grep_content,
health_check, list_directory, list_mounts, mount_filesystem, read_file, stat_file,
- unmount_filesystem, write_file, AppState,
+ tree_directory, unmount_filesystem, write_file, AppState,
};
/// Create the main application router
@@ -31,6 +31,8 @@ pub fn create_router(state: AppState, enable_cors: bool) -> Router {
.route("/directories", get(list_directory))
.route("/directories", post(create_directory))
.route("/directories/ensure-parent", post(ensure_parent_dirs))
+ // Tree (recursive directory listing)
+ .route("/tree", get(tree_directory))
// Mount management
.route("/mounts", get(list_mounts))
.route("/mount", post(mount_filesystem))
diff --git a/openviking/pyagfs/async_client.py b/openviking/pyagfs/async_client.py
index 57423db87..87e243efa 100644
--- a/openviking/pyagfs/async_client.py
+++ b/openviking/pyagfs/async_client.py
@@ -88,3 +88,18 @@ class AsyncAGFSClient:
async def grep(self, **kwargs: Any) -> Dict[str, Any]:
return await self.run("grep", **kwargs)
+
+ async def tree_directory(
+ self,
+ path: str,
+ show_hidden: bool = False,
+ node_limit: int | None = None,
+ level_limit: int | None = None,
+ ) -> list[Dict[str, Any]]:
+ return await self.run(
+ "tree_directory",
+ path,
+ show_hidden=show_hidden,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ )
diff --git a/openviking/pyagfs/client.py b/openviking/pyagfs/client.py
index d9c647ed1..2af379d3f 100644
--- a/openviking/pyagfs/client.py
+++ b/openviking/pyagfs/client.py
@@ -127,6 +127,45 @@ class AGFSClient:
except Exception as e:
self._handle_request_error(e)
+ def tree_directory(
+ self,
+ path: str,
+ show_hidden: bool = False,
+ node_limit: Optional[int] = None,
+ level_limit: Optional[int] = None,
+ ) -> List[Dict[str, Any]]:
+ """Recursively list a directory tree.
+
+ Mirrors the native binding client's ``tree_directory`` so the HTTP
+ client exposes the same surface.
+
+ Args:
+ path: Directory path to traverse.
+ show_hidden: Include hidden entries (names starting with '.').
+ node_limit: Maximum number of nodes to return (None = unlimited).
+ level_limit: Maximum depth relative to the query root (None = unlimited).
+
+ Returns:
+ List of tree entry dicts (path, rel_path, info, extra).
+ """
+ try:
+ params: Dict[str, Any] = {"path": path}
+ if show_hidden:
+ params["show_hidden"] = "true"
+ if node_limit is not None:
+ params["node_limit"] = str(node_limit)
+ if level_limit is not None:
+ params["level_limit"] = str(level_limit)
+ response = self.session.get(
+ f"{self.api_base}/tree", params=params, timeout=self.timeout
+ )
+ response.raise_for_status()
+ data = response.json()
+ entries = data.get("data")
+ return entries if entries is not None else []
+ except Exception as e:
+ self._handle_request_error(e)
+
def read(self, path: str, offset: int = 0, size: int = -1, stream: bool = False):
return self.cat(path, offset, size, stream)
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 89cf1bc9d..0a83a42a1 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -1217,10 +1217,10 @@ class VikingFS:
level_limit: int | None = 3 (maximum depth level to traverse, None means unlimited)
output="original"
- [{'name': '.abstract.md', 'size': 100, 'mode': 420, 'modTime': '2026-02-11T16:52:16.256334192+08:00', 'isDir': False, 'meta': {...}, 'rel_path': '.abstract.md', 'uri': 'viking://resources...'}]
+ [{'name': '.abstract.md', 'size': 100, 'mode': 420, 'modTime': '2026-02-11T16:52:16.256334192+08:00', 'isDir': False, 'rel_path': '.abstract.md', 'uri': 'viking://resources...'}]
output="agent"
- [{'name': '.abstract.md', 'size': 100, 'modTime': '2026-02-11 16:52:16', 'isDir': False, 'rel_path': '.abstract.md', 'uri': 'viking://resources...', 'abstract': "..."}]
+ [{'uri': 'viking://resources...', 'size': 100, 'isDir': False, 'modTime': '2026-02-11 16:52:16', 'rel_path': '.abstract.md', 'abstract': "..."}]
"""
self._ensure_access(uri, ctx)
if output == "original":
@@ -1241,37 +1241,29 @@ class VikingFS:
ctx: Optional[RequestContext] = None,
) -> List[Dict[str, Any]]:
"""Recursively list all contents (original format)."""
- path = self._uri_to_path(uri, ctx=ctx)
- all_entries = []
- real_ctx = self._ctx_or_default(ctx)
-
- async def _walk(current_path: str, current_rel: str, current_depth: int):
- if node_limit is not None and len(all_entries) >= node_limit:
- return
- if level_limit is not None and current_depth >= level_limit:
- return
- for entry in await self._ls_entries(current_path):
- if node_limit is not None and len(all_entries) >= node_limit:
- break
- name = entry.get("name", "")
- if name in [".", ".."]:
- continue
- rel_path = f"{current_rel}/{name}" if current_rel else name
- new_entry = dict(entry)
- new_entry["rel_path"] = rel_path
- new_entry["uri"] = self._path_to_uri(f"{current_path}/{name}", ctx=ctx)
- if not self._is_accessible(new_entry["uri"], real_ctx):
- continue
- if entry.get("isDir"):
- all_entries.append(new_entry)
- await _walk(f"{current_path}/{name}", rel_path, current_depth + 1)
- elif not name.startswith("."):
- all_entries.append(new_entry)
- elif show_all_hidden:
- all_entries.append(new_entry)
-
- await _walk(path, "", 0)
- return all_entries
+ result = []
+ async for entry, entry_uri in self._iter_visible_tree_entries(
+ uri,
+ show_all_hidden=show_all_hidden,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ ):
+ info = entry["info"]
+ new_entry = dict(entry.get("extra", {}))
+ new_entry.update(
+ {
+ "name": info["name"],
+ "size": info["size"],
+ "mode": info["mode"],
+ "modTime": info["modTime"],
+ "isDir": info["isDir"],
+ "rel_path": entry["rel_path"],
+ "uri": entry_uri,
+ }
+ )
+ result.append(new_entry)
+ return result
async def _tree_agent(
self,
@@ -1283,46 +1275,31 @@ class VikingFS:
ctx: Optional[RequestContext] = None,
) -> List[Dict[str, Any]]:
"""Recursively list all contents (agent format with abstracts)."""
- path = self._uri_to_path(uri, ctx=ctx)
- all_entries = []
+ result = []
now = datetime.now(timezone.utc)
- real_ctx = self._ctx_or_default(ctx)
- async def _walk(current_path: str, current_rel: str, current_depth: int):
- if node_limit is not None and len(all_entries) >= node_limit:
- return
- if level_limit is not None and current_depth >= level_limit:
- return
- for entry in await self._ls_entries(current_path):
- if node_limit is not None and len(all_entries) >= node_limit:
- break
- name = entry.get("name", "")
- if name in [".", ".."]:
- continue
- rel_path = f"{current_rel}/{name}" if current_rel else name
- is_dir = entry.get("isDir", False)
- new_entry = {
- "uri": self._path_to_uri(f"{current_path}/{name}", ctx=ctx),
- "size": 0 if is_dir else entry.get("size", 0),
+ async for entry, entry_uri in self._iter_visible_tree_entries(
+ uri,
+ show_all_hidden=show_all_hidden,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ ):
+ info = entry["info"]
+ is_dir = info["isDir"]
+ result.append(
+ {
+ "uri": entry_uri,
+ "size": 0 if is_dir else info["size"],
"isDir": is_dir,
- "modTime": format_simplified(parse_iso_datetime(entry.get("modTime", "")), now),
+ "modTime": format_simplified(parse_iso_datetime(info["modTime"]), now),
+ "rel_path": entry["rel_path"],
}
- new_entry["rel_path"] = rel_path
- if not self._is_accessible(new_entry["uri"], real_ctx):
- continue
- if is_dir:
- all_entries.append(new_entry)
- await _walk(f"{current_path}/{name}", rel_path, current_depth + 1)
- elif not name.startswith("."):
- all_entries.append(new_entry)
- elif show_all_hidden:
- all_entries.append(new_entry)
-
- await _walk(path, "", 0)
+ )
- await self._batch_fetch_abstracts(all_entries, abs_limit, ctx=ctx)
+ await self._batch_fetch_abstracts(result, abs_limit, ctx=ctx)
- return all_entries
+ return result
# ========== VikingFS Specific Capabilities ==========
@@ -1781,6 +1758,138 @@ class VikingFS:
path = self._uri_to_path(uri, ctx=ctx)
return await self._read_relation_table(path, ctx=ctx)
+ # ========== Tree Traversal (Refactored) ==========
+
+ def _is_name_visible_at_path(self, name: str, parent_path: str) -> bool:
+ """Check if name would appear in _ls_entries(parent_path).
+
+ At account root (/local/{account}), uses LISTABLE_SCOPES whitelist.
+ At other levels, uses _INTERNAL_NAMES blacklist.
+ """
+ parts = [p for p in parent_path.strip("/").split("/") if p]
+ if len(parts) == 2 and parts[0] == "local":
+ return name in VikingURI.LISTABLE_SCOPES
+ return name not in self._INTERNAL_NAMES
+
+ def _ancestor_is_filtered(self, entry_path: str, base_path: str) -> bool:
+ """Check if any ancestor directory of entry_path would be filtered by _ls_entries.
+
+ Walks from base_path (exclusive) to entry's parent directory (exclusive),
+ checking each component against _is_name_visible_at_path.
+ """
+ base_parts = [p for p in base_path.strip("/").split("/") if p]
+ entry_parts = [p for p in entry_path.strip("/").split("/") if p]
+
+ for i in range(len(base_parts), len(entry_parts) - 1):
+ name = entry_parts[i]
+ parent_parts = entry_parts[:i]
+ parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"
+ if not self._is_name_visible_at_path(name, parent_path):
+ return True
+ return False
+
+ def _is_tree_entry_visible(
+ self, entry: Dict[str, Any], base_path: str, ctx: RequestContext
+ ) -> bool:
+ """Check visibility for a single TreeEntry returned by Rust tree_directory.
+
+ Applies three layers of filtering:
+ 1. Ancestor chain — if any ancestor directory would be filtered by _ls_entries,
+ all descendants are invisible.
+ 2. Self — the entry's own name must pass _ls_entries at its parent level.
+ 3. ACL — the entry must be accessible by the requesting context.
+ """
+ entry_path = entry["path"]
+
+ if self._ancestor_is_filtered(entry_path, base_path):
+ return False
+
+ entry_parts = [p for p in entry_path.strip("/").split("/") if p]
+ if entry_parts:
+ name = entry_parts[-1]
+ parent_parts = entry_parts[:-1]
+ parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"
+ if not self._is_name_visible_at_path(name, parent_path):
+ return False
+
+ uri = self._path_to_uri(entry_path, ctx=ctx)
+ if not self._is_accessible(uri, ctx):
+ return False
+
+ return True
+
+ # Over-fetch multiplier for bounded tree traversal. When a node_limit is
+ # set, we push down node_limit * this factor as the raw-node limit to Rust,
+ # leaving headroom for ACL/internal-name filtering before re-fetching.
+ _TREE_OVERFETCH_FACTOR = 4
+
+ async def _iter_visible_tree_entries(
+ self,
+ uri: str,
+ show_all_hidden: bool = False,
+ node_limit: Optional[int] = None,
+ level_limit: Optional[int] = None,
+ ctx: Optional[RequestContext] = None,
+ ):
+ """Shared generator: fetch raw TreeEntry list from Rust, yield (entry, uri) tuples.
+
+ node_limit counts ACL-visible entries (see design §6.5), so the user's
+ node_limit cannot be pushed directly to Rust — doing so would truncate
+ before filtering and drop entries that should be visible.
+
+ To keep memory bounded without changing that semantic, we push down an
+ *amplified* raw-node limit (node_limit * _TREE_OVERFETCH_FACTOR). If ACL
+ filtering leaves fewer than node_limit visible entries while Rust still
+ returned a full page (i.e. more raw nodes may exist), we double the raw
+ limit and re-fetch. Because Rust truncates a deterministic sorted prefix,
+ this yields exactly the same result as an unbounded fetch, while avoiding
+ materializing the entire prefix in the common case.
+
+ When node_limit is None (full-tree callers), no limit is pushed down.
+ level_limit IS always passed to Rust.
+ """
+ path = self._uri_to_path(uri, ctx=ctx)
+ real_ctx = self._ctx_or_default(ctx)
+
+ if node_limit is None:
+ raw_limit: Optional[int] = None
+ else:
+ raw_limit = max(node_limit * self._TREE_OVERFETCH_FACTOR, node_limit)
+
+ while True:
+ raw_entries = await self._async_agfs.tree_directory(
+ path,
+ show_hidden=show_all_hidden,
+ node_limit=raw_limit,
+ level_limit=level_limit,
+ )
+
+ visible: List[tuple] = []
+ for entry in raw_entries:
+ if node_limit is not None and len(visible) >= node_limit:
+ break
+ if not self._is_tree_entry_visible(entry, path, real_ctx):
+ continue
+ entry_uri = self._path_to_uri(entry["path"], ctx=ctx)
+ visible.append((entry, entry_uri))
+
+ # If we still lack enough visible entries but Rust returned a full
+ # page (raw_limit reached), more raw nodes may exist — re-fetch with
+ # a doubled limit. Otherwise Rust is exhausted and we yield as-is.
+ need_more = (
+ node_limit is not None
+ and len(visible) < node_limit
+ and raw_limit is not None
+ and len(raw_entries) >= raw_limit
+ )
+ if need_more:
+ raw_limit *= 2
+ continue
+
+ for item in visible:
+ yield item
+ return
+
# ========== URI Conversion ==========
# Maximum bytes for a single filename component (filesystem limit is typically 255)
diff --git a/tests/storage/test_viking_fs_tree.py b/tests/storage/test_viking_fs_tree.py
new file mode 100644
index 000000000..3171f8c2d
--- /dev/null
+++ b/tests/storage/test_viking_fs_tree.py
@@ -0,0 +1,554 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import pytest
+
+from openviking.server.identity import RequestContext, Role
+from openviking.storage.viking_fs import VikingFS
+from openviking_cli.session.user_id import UserIdentifier
+
+
+class _DummyAgfs:
+ pass
+
+
+def _default_ctx() -> RequestContext:
+ return RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+
+
+# ── Shared fixtures / builders ──────────────────────────────────────────────
+
+
+def fs() -> VikingFS:
+ """Fresh VikingFS backed by a dummy AGFS client."""
+ return VikingFS(agfs=_DummyAgfs())
+
+
+def _std_path_to_uri(path, **_kwargs):
+ """Map account-isolated path back to a viking:// URI (test convention)."""
+ return path.replace("/local/test_account/", "viking://")
+
+
+def make_entry(
+ path,
+ name=None,
+ *,
+ size=0,
+ mode=0o755,
+ mod_time="2026-01-01T00:00:00Z",
+ is_dir=True,
+ extra=None,
+):
+ """Build a Rust-shaped TreeEntry dict for tests.
+
+ ``rel_path`` is derived from ``path`` by stripping the account prefix, and
+ ``name`` defaults to the last path component when not given.
+ """
+ if name is None:
+ name = path.rstrip("/").rsplit("/", 1)[-1]
+ return {
+ "path": path,
+ "rel_path": path.replace("/local/test_account/", ""),
+ "info": {
+ "name": name,
+ "size": size,
+ "mode": mode,
+ "modTime": mod_time,
+ "isDir": is_dir,
+ },
+ "extra": extra or {},
+ }
+
+
+def patch_visibility(monkeypatch, fs, *, is_accessible=True):
+ """Patch ACL + URI mapping used by visibility checks."""
+ monkeypatch.setattr(
+ fs,
+ "_is_accessible",
+ is_accessible if callable(is_accessible) else (lambda _uri, _ctx: is_accessible),
+ )
+ monkeypatch.setattr(fs, "_path_to_uri", _std_path_to_uri)
+
+
+def patch_tree_env(
+ monkeypatch,
+ fs,
+ entries_or_fn,
+ *,
+ is_accessible=True,
+ uri_to_path="/local/test_account/resources",
+ batch_fetch=None,
+):
+ """Install the standard monkeypatch set for tree traversal tests.
+
+ ``entries_or_fn`` may be a list of entries (returned by a generated fake
+ ``tree_directory``) or a custom async ``tree_directory`` callable when a
+ test needs to capture the arguments passed to Rust.
+ """
+ if callable(entries_or_fn):
+ fake_tree = entries_or_fn
+ else:
+
+ async def fake_tree(_path, **_kwargs):
+ return entries_or_fn
+
+ monkeypatch.setattr(fs._async_agfs, "tree_directory", fake_tree)
+ monkeypatch.setattr(fs, "_uri_to_path", lambda _uri, **_kwargs: uri_to_path)
+ monkeypatch.setattr(fs, "_ctx_or_default", lambda _ctx=None: _default_ctx())
+ patch_visibility(monkeypatch, fs, is_accessible=is_accessible)
+ if batch_fetch is not None:
+ monkeypatch.setattr(fs, "_batch_fetch_abstracts", batch_fetch)
+
+
+async def default_batch_fetch(entries, abs_limit, **_kwargs):
+ """Shared abstract enrichment fake: dirs get a mock abstract (truncated to
+ abs_limit), files get an empty abstract."""
+ for entry in entries:
+ abstract = "mock abstract" if entry.get("isDir") else ""
+ if len(abstract) > abs_limit:
+ abstract = abstract[: abs_limit - 3] + "..."
+ entry["abstract"] = abstract
+
+
+# ── _is_name_visible_at_path / _ancestor_is_filtered tests ──
+
+
+ "name,parent_path,expected",
+ [
+ ("resources", "/local/test_account", True),
+ ("user", "/local/test_account", True),
+ ("agent", "/local/test_account", True),
+ ("session", "/local/test_account", True),
+ ("tasks", "/local/test_account", False),
+ ("_system", "/local/test_account", False),
+ ("temp", "/local/test_account", False),
+ ],
+)
+def test_is_name_visible_at_account_root(fs, name, parent_path, expected):
+ """PY-FLT-001, PY-FLT-002: Account root LISTABLE_SCOPES whitelist."""
+ assert fs._is_name_visible_at_path(name, parent_path) == expected
+
+
+ "name,parent_path,expected",
+ [
+ ("my_dir", "/local/test_account/resources", True),
+ ("normal_dir", "/local/test_account/resources/foo", True),
+ ("_system", "/local/test_account/resources", False),
+ ("tasks", "/local/test_account/resources/bar", False),
+ (".path.ovlock", "/local/test_account/resources", False),
+ ],
+)
+def test_is_name_visible_at_non_root(fs, name, parent_path, expected):
+ """PY-FLT-004: Non-root _INTERNAL_NAMES blacklist."""
+ assert fs._is_name_visible_at_path(name, parent_path) == expected
+
+
+ "entry_path,base_path,expected",
+ [
+ ("/local/test_account/resources/a", "/local/test_account", False),
+ ("/local/test_account/tasks/foo", "/local/test_account", True),
+ ("/local/test_account/tasks/foo/bar.txt", "/local/test_account", True),
+ ("/local/test_account/resources/_system/secret.txt", "/local/test_account", True),
+ ("/local/test_account/resources/normal/file.txt", "/local/test_account", False),
+ ("/local/test_account/resources/a/b/c", "/local/test_account", False),
+ ],
+)
+def test_ancestor_is_filtered(fs, entry_path, base_path, expected):
+ """PY-FLT-003, PY-FLT-006, PY-FLT-007: Ancestor chain filtering."""
+ assert fs._ancestor_is_filtered(entry_path, base_path) == expected
+
+
+# ── _is_tree_entry_visible tests ──
+
+
+def test_is_tree_entry_visible_visible(monkeypatch, fs):
+ """PY-FLT-005, PY-FLT-009: Normal visible entry."""
+ patch_visibility(monkeypatch, fs, is_accessible=True)
+ entry = make_entry("/local/test_account/resources/a", "a")
+ assert fs._is_tree_entry_visible(entry, "/local/test_account", _default_ctx()) is True
+
+
+def test_is_tree_entry_visible_acl_filtered(monkeypatch, fs):
+ """PY-FLT-008: ACL filtering."""
+ patch_visibility(monkeypatch, fs, is_accessible=False)
+ entry = make_entry("/local/test_account/resources/secret", "secret")
+ assert fs._is_tree_entry_visible(entry, "/local/test_account", _default_ctx()) is False
+
+
+def test_is_tree_entry_visible_hidden_scope_filtered(monkeypatch, fs):
+ """PY-FLT-007: tasks scope filtered at account root."""
+ patch_visibility(monkeypatch, fs, is_accessible=True)
+ entry = make_entry("/local/test_account/tasks/foo", "foo")
+ assert fs._is_tree_entry_visible(entry, "/local/test_account", _default_ctx()) is False
+
+
+def test_is_tree_entry_visible_path_ovlock_filtered(monkeypatch, fs):
+ """PY-FLT-011: .path.ovlock is filtered."""
+ patch_visibility(monkeypatch, fs, is_accessible=True)
+ entry = make_entry("/local/test_account/resources/.path.ovlock", ".path.ovlock", is_dir=False)
+ assert fs._is_tree_entry_visible(entry, "/local/test_account", _default_ctx()) is False
+
+
+def test_is_tree_entry_visible_default_ctx(monkeypatch, fs):
+ """PY-FLT-010: ctx=None uses default context."""
+ patch_visibility(monkeypatch, fs, is_accessible=True)
+ entry = make_entry("/local/test_account/resources/a", "a")
+ assert fs._is_tree_entry_visible(entry, "/local/test_account", _default_ctx()) is True
+
+
+# ── _iter_visible_tree_entries tests ──
+
+
+async def test_iter_visible_tree_entries_node_limit_amplified_to_rust(monkeypatch, fs):
+ """PY-ITER-001: node_limit is amplified (× overfetch factor) when pushed to
+ Rust, keeping memory bounded while preserving the ACL-after-filter semantic."""
+ captured = {}
+
+ async def fake_tree_directory(_path, **kwargs):
+ captured["node_limit"] = kwargs.get("node_limit")
+ return []
+
+ patch_tree_env(monkeypatch, fs, fake_tree_directory)
+
+ async for _ in fs._iter_visible_tree_entries(
+ "viking://resources", node_limit=10, ctx=_default_ctx()
+ ):
+ pass
+
+ assert captured["node_limit"] == 10 * fs._TREE_OVERFETCH_FACTOR
+
+
+async def test_iter_visible_tree_entries_node_limit_none_not_pushed(monkeypatch, fs):
+ """PY-ITER-001b: node_limit=None (full-tree callers) pushes no raw limit."""
+ captured = {"set": False, "value": "unset"}
+
+ async def fake_tree_directory(_path, **kwargs):
+ captured["set"] = True
+ captured["value"] = kwargs.get("node_limit")
+ return []
+
+ patch_tree_env(monkeypatch, fs, fake_tree_directory)
+
+ async for _ in fs._iter_visible_tree_entries(
+ "viking://resources", node_limit=None, ctx=_default_ctx()
+ ):
+ pass
+
+ assert captured["set"] is True
+ assert captured["value"] is None
+
+
+async def test_iter_visible_tree_entries_level_limit_passed_to_rust(monkeypatch, fs):
+ """PY-ITER-003: level_limit IS passed to Rust layer."""
+ captured = {}
+
+ async def fake_tree_directory(_path, **kwargs):
+ captured["level_limit"] = kwargs.get("level_limit")
+ return []
+
+ patch_tree_env(monkeypatch, fs, fake_tree_directory)
+
+ async for _ in fs._iter_visible_tree_entries(
+ "viking://resources", level_limit=3, ctx=_default_ctx()
+ ):
+ pass
+
+ assert captured["level_limit"] == 3
+
+
+async def test_iter_visible_tree_entries_node_limit_after_acl(monkeypatch, fs):
+ """PY-ITER-002: node_limit applied AFTER ACL filtering."""
+ visible_count = 0
+
+ entries = [
+ make_entry(f"/local/test_account/resources/{name}", name, is_dir=False)
+ for name in ["a", "b", "c", "d", "e"]
+ ]
+
+ def fake_is_accessible(_uri, _ctx):
+ nonlocal visible_count
+ visible_count += 1
+ return True
+
+ patch_tree_env(monkeypatch, fs, entries, is_accessible=fake_is_accessible)
+
+ results = []
+ async for entry, _entry_uri in fs._iter_visible_tree_entries(
+ "viking://resources", node_limit=3, ctx=_default_ctx()
+ ):
+ results.append(entry)
+
+ assert len(results) == 3
+ assert visible_count == 3, "only 3 entries should have ACL checked before limit hit"
+
+
+async def test_iter_visible_tree_entries_refetch_when_under_limit(monkeypatch, fs):
+ """PY-ITER-004: when ACL filtering leaves too few visible entries and Rust
+ returned a full page, the raw limit is doubled and re-fetched (zero-regression
+ bounded over-fetch)."""
+ factor = fs._TREE_OVERFETCH_FACTOR
+ node_limit = 2
+ # First page (size == raw_limit) is entirely invisible; second page after
+ # doubling contains the visible entries.
+ invisible = [
+ make_entry(f"/local/test_account/resources/h{i}/x", "x", is_dir=False)
+ for i in range(node_limit * factor)
+ ]
+ visible = [
+ make_entry(f"/local/test_account/resources/v{i}", f"v{i}", is_dir=False)
+ for i in range(node_limit)
+ ]
+ captured_limits = []
+
+ async def fake_tree_directory(_path, **kwargs):
+ raw_limit = kwargs.get("node_limit")
+ captured_limits.append(raw_limit)
+ # First call returns a full page of invisible entries; subsequent call
+ # (doubled limit) returns invisible + visible.
+ if len(captured_limits) == 1:
+ return invisible[:raw_limit]
+ return invisible + visible
+
+ def fake_is_accessible(uri, _ctx):
+ # Entries under an "hN" directory are invisible (ACL denied).
+ return "/h" not in uri
+
+ patch_tree_env(monkeypatch, fs, fake_tree_directory, is_accessible=fake_is_accessible)
+
+ results = []
+ async for entry, _uri in fs._iter_visible_tree_entries(
+ "viking://resources", node_limit=node_limit, ctx=_default_ctx()
+ ):
+ results.append(entry)
+
+ assert len(results) == node_limit
+ # Re-fetch happened: first raw_limit = node_limit*factor, then doubled.
+ assert captured_limits[0] == node_limit * factor
+ assert captured_limits[1] == node_limit * factor * 2
+
+
+async def test_iter_visible_tree_entries_show_hidden_passthrough(monkeypatch, fs):
+ """PY-ITER-005: show_all_hidden passthrough."""
+ captured = {}
+
+ async def fake_tree_directory(_path, **kwargs):
+ captured["show_hidden"] = kwargs.get("show_hidden")
+ return []
+
+ patch_tree_env(monkeypatch, fs, fake_tree_directory)
+
+ async for _ in fs._iter_visible_tree_entries(
+ "viking://resources", show_all_hidden=True, ctx=_default_ctx()
+ ):
+ pass
+
+ assert captured["show_hidden"] is True
+
+
+# ── _tree_original tests ──
+
+
+async def test_tree_original_structure(monkeypatch, fs):
+ """PY-ORIG-001: Return structure contains expected fields."""
+ entries = [
+ make_entry(
+ "/local/test_account/resources/a",
+ "a",
+ size=100,
+ mode=0o644,
+ is_dir=False,
+ extra={"meta": {"Name": "s3fs"}},
+ )
+ ]
+ patch_tree_env(monkeypatch, fs, entries)
+
+ result = await fs._tree_original("viking://resources", ctx=_default_ctx())
+
+ assert len(result) == 1
+ e = result[0]
+ assert e["name"] == "a"
+ assert e["size"] == 100
+ assert e["mode"] == 0o644
+ assert e["modTime"] == "2026-01-01T00:00:00Z"
+ assert e["isDir"] is False
+ assert e["rel_path"] == "resources/a"
+ assert e["uri"] == "viking://resources/a"
+
+
+async def test_tree_original_extra_fields_preserved(monkeypatch, fs):
+ """PY-ORIG-002: extra fields preserved."""
+ entries = [
+ make_entry(
+ "/local/test_account/resources/a",
+ "a",
+ size=100,
+ mode=0o644,
+ is_dir=False,
+ extra={"meta": {"Name": "s3fs", "Type": "s3"}},
+ )
+ ]
+ patch_tree_env(monkeypatch, fs, entries)
+
+ result = await fs._tree_original("viking://resources", ctx=_default_ctx())
+ assert result[0]["meta"] == {"Name": "s3fs", "Type": "s3"}
+
+
+async def test_tree_original_dfs_order(monkeypatch, fs):
+ """PY-ORIG-006: DFS order preserved — directories before their children."""
+ entries = [
+ make_entry("/local/test_account/resources/sub", "sub", is_dir=True),
+ make_entry(
+ "/local/test_account/resources/sub/file.txt",
+ "file.txt",
+ size=100,
+ mode=0o644,
+ is_dir=False,
+ ),
+ ]
+ patch_tree_env(monkeypatch, fs, entries)
+
+ result = await fs._tree_original("viking://resources", ctx=_default_ctx())
+ assert result[0]["name"] == "sub"
+ assert result[0]["isDir"] is True
+ assert result[1]["name"] == "file.txt"
+
+
+# ── _tree_agent tests ──
+
+
+async def test_tree_agent_structure(monkeypatch, fs):
+ """PY-AGENT-001: Agent output structure."""
+ entries = [
+ make_entry("/local/test_account/resources/a", "a", size=100, mode=0o644, is_dir=False),
+ make_entry("/local/test_account/resources/sub", "sub", is_dir=True),
+ ]
+ patch_tree_env(monkeypatch, fs, entries, batch_fetch=default_batch_fetch)
+
+ result = await fs._tree_agent("viking://resources", abs_limit=256, ctx=_default_ctx())
+
+ assert len(result) == 2
+ assert set(result[0].keys()) == {"uri", "size", "isDir", "modTime", "rel_path", "abstract"}
+ assert "name" not in result[0]
+
+
+async def test_tree_agent_dir_size_zero(monkeypatch, fs):
+ """PY-AGENT-002: Directory size is always 0."""
+ entries = [make_entry("/local/test_account/resources/sub", "sub", size=999, is_dir=True)]
+ patch_tree_env(monkeypatch, fs, entries, batch_fetch=default_batch_fetch)
+
+ result = await fs._tree_agent("viking://resources", abs_limit=256, ctx=_default_ctx())
+ assert result[0]["size"] == 0
+
+
+async def test_tree_agent_non_dir_abstract_empty(monkeypatch, fs):
+ """PY-AGENT-004: Non-directory entries have empty abstract."""
+ entries = [
+ make_entry("/local/test_account/resources/a", "a", size=100, mode=0o644, is_dir=False)
+ ]
+ patch_tree_env(monkeypatch, fs, entries, batch_fetch=default_batch_fetch)
+
+ result = await fs._tree_agent("viking://resources", abs_limit=256, ctx=_default_ctx())
+ assert result[0]["abstract"] == ""
+
+
+async def test_tree_agent_modtime_formatted(monkeypatch, fs):
+ """PY-AGENT-003: modTime is formatted via format_simplified."""
+ entries = [
+ make_entry("/local/test_account/resources/a", "a", size=100, mode=0o644, is_dir=False)
+ ]
+ patch_tree_env(monkeypatch, fs, entries, batch_fetch=default_batch_fetch)
+
+ result = await fs._tree_agent("viking://resources", abs_limit=256, ctx=_default_ctx())
+ assert result[0]["modTime"] == "2026-01-01"
+
+
+async def test_tree_agent_abs_limit_truncation(monkeypatch, fs):
+ """PY-AGENT-005: abstract is truncated when exceeding abs_limit."""
+ entries = [make_entry("/local/test_account/resources/sub", "sub", is_dir=True)]
+
+ async def fake_batch_fetch(entries_arg, abs_limit, **_kwargs):
+ for entry in entries_arg:
+ abstract = "x" * (abs_limit + 10) if entry.get("isDir") else ""
+ if len(abstract) > abs_limit:
+ abstract = abstract[: abs_limit - 3] + "..."
+ entry["abstract"] = abstract
+
+ patch_tree_env(monkeypatch, fs, entries, batch_fetch=fake_batch_fetch)
+
+ result = await fs._tree_agent("viking://resources", abs_limit=10, ctx=_default_ctx())
+ assert len(result[0]["abstract"]) <= 10
+ assert result[0]["abstract"].endswith("...")
+
+
+async def test_tree_agent_batch_fetch_input_order(monkeypatch, fs):
+ """PY-AGENT-006: _batch_fetch_abstracts receives entries in correct order."""
+ captured_entries = None
+
+ entries = [
+ make_entry("/local/test_account/resources/a", "a", size=100, mode=0o644, is_dir=False),
+ make_entry(
+ "/local/test_account/resources/b",
+ "b",
+ size=200,
+ mode=0o644,
+ mod_time="2026-01-02T00:00:00Z",
+ is_dir=False,
+ ),
+ ]
+
+ async def fake_batch_fetch(entries_arg, _abs_limit, **_kwargs):
+ nonlocal captured_entries
+ captured_entries = list(entries_arg)
+ for entry in entries_arg:
+ entry["abstract"] = ""
+
+ patch_tree_env(monkeypatch, fs, entries, batch_fetch=fake_batch_fetch)
+
+ await fs._tree_agent("viking://resources", abs_limit=256, ctx=_default_ctx())
+ assert len(captured_entries) == 2
+ assert captured_entries[0]["uri"] == "viking://resources/a"
+ assert captured_entries[1]["uri"] == "viking://resources/b"
+
+
+async def test_tree_agent_node_limit_before_enrichment(monkeypatch, fs):
+ """PY-AGENT-007: node_limit applied before abstract enrichment."""
+ enriched_count = 0
+
+ entries = [
+ make_entry(f"/local/test_account/resources/{name}", name, is_dir=False)
+ for name in ["a", "b", "c", "d", "e"]
+ ]
+
+ async def fake_batch_fetch(entries_arg, _abs_limit, **_kwargs):
+ nonlocal enriched_count
+ enriched_count = len(entries_arg)
+ for entry in entries_arg:
+ entry["abstract"] = ""
+
+ patch_tree_env(monkeypatch, fs, entries, batch_fetch=fake_batch_fetch)
+
+ result = await fs._tree_agent(
+ "viking://resources", node_limit=2, abs_limit=256, ctx=_default_ctx()
+ )
+ assert len(result) == 2
+ assert enriched_count == 2