Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

Optimize glob walkdir (#3013)

baojun-zhang6a33ebb7ca06
diff --git a/Cargo.lock b/Cargo.lock
index 8ebae849a..352657a3d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3886,6 +3886,7 @@ dependencies = [
"gix-features",
"gix-hash",
"gix-object",
+ "globset",
"grep",
"grep-matcher",
"grep-regex",
diff --git a/benchmark/custom/session_contention_benchmark.py b/benchmark/custom/session_contention_benchmark.py
index f2f7ea251..44cc6b38b 100644
--- a/benchmark/custom/session_contention_benchmark.py
+++ b/benchmark/custom/session_contention_benchmark.py
@@ -994,7 +994,7 @@ class BenchmarkRunner:
"glob",
lambda: adapter.glob(
uri=self.config.data_root_uri,
- pattern="*.md",
+ pattern="**/*.md",
limit=self.config.find_limit,
),
worker_id=worker_id,
diff --git a/crates/ov_cli/test_ov.sh b/crates/ov_cli/test_ov.sh
index 563bdb086..e398bb64d 100755
--- a/crates/ov_cli/test_ov.sh
+++ b/crates/ov_cli/test_ov.sh
@@ -203,7 +203,7 @@ fi
echo ""
echo "5.4. Glob pattern search..."
-if $OV_BIN glob "*.md" --uri "viking://resources"; then
+if $OV_BIN glob "**/*.md" --uri "viking://resources"; then
print_success "Glob search completed"
else
print_error "Glob search failed"
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs
index 88aabf30c..f0363764d 100644
--- a/crates/ragfs-python/src/lib.rs
+++ b/crates/ragfs-python/src/lib.rs
@@ -20,7 +20,7 @@ use ragfs::cache::{
use ragfs::core::builder::EncryptionConfig;
use ragfs::core::{
build_default_stack, register_builtin_plugins, ConfigValue, FileInfo, FileSystem,
- FilesystemStats, FsContext, FsContextInner, FsOperation, GrepResult, MountableFS,
+ FilesystemStats, FsContext, FsContextInner, FsOperation, GlobPage, GrepResult, MountableFS,
OperationStats, PluginConfig, RagfsConfig, StatsWrappedFS, TreeEntry, WriteFlag, FS_CTX,
};
@@ -1618,6 +1618,57 @@ impl RAGFSBindingClient {
})
}
+ /// Return one page of flat glob results.
+ ///
+ /// Args:
+ /// path: The root path of the traversal
+ /// pattern: Glob pattern matched against query-root-relative paths
+ /// show_hidden: Whether to include hidden files (default: False)
+ /// page_size: Maximum number of matched entries returned in this page
+ /// level_limit: Maximum depth relative to query root (default: None)
+ /// continuation_token: Opaque token returned by the previous page
+ /// ctx: Optional FsContext dict (e.g. {"account_id": ...})
+ ///
+ /// Returns:
+ /// A dict with keys: entries (list[GlobEntry]), next_token (str | None)
+ #[pyo3(signature = (path, pattern, show_hidden=false, page_size=None, level_limit=None, continuation_token=None, ctx=None))]
+ fn glob_directory(
+ &self,
+ py: Python<'_>,
+ path: String,
+ pattern: String,
+ show_hidden: bool,
+ page_size: Option<i32>,
+ level_limit: Option<i32>,
+ continuation_token: Option<String>,
+ ctx: Option<HashMap<String, String>>,
+ ) -> PyResult<Py<PyAny>> {
+ let fs_ctx = build_fs_context(ctx);
+ let top = self.top.clone();
+ let page_size = page_size.map(|n| if n < 0 { 0 } else { n as usize });
+ let level_limit_usize = level_limit.map(|n| if n < 0 { 0 } else { n as usize });
+
+ let page: GlobPage = self
+ .run_scoped(py, fs_ctx, move || async move {
+ top.glob_directory(
+ &path,
+ &pattern,
+ show_hidden,
+ page_size,
+ level_limit_usize,
+ continuation_token,
+ )
+ .await
+ })
+ .map_err(to_py_err)?;
+
+ Python::attach(|py| {
+ let value = serde_json::to_value(&page)
+ .map_err(|err| PyRuntimeError::new_err(err.to_string()))?;
+ serde_json_to_py(py, &value)
+ })
+ }
+
/// Query multi-write sync status under a file or directory path.
///
/// Args:
diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.toml
index 8ea09680c..d23af4e6c 100644
--- a/crates/ragfs/Cargo.toml
+++ b/crates/ragfs/Cargo.toml
@@ -80,6 +80,7 @@ lru = "0.12"
# Regular expressions for grep
regex = "1.10"
mime_guess = "2.0"
+globset = "0.4"
# Encryption (envelope encryption: AES-256-GCM + HKDF-SHA256)
aes-gcm = "0.10"
diff --git a/crates/ragfs/src/cache/wrapper.rs b/crates/ragfs/src/cache/wrapper.rs
index 67f8587af..d36151b46 100644
--- a/crates/ragfs/src/cache/wrapper.rs
+++ b/crates/ragfs/src/cache/wrapper.rs
@@ -9,7 +9,8 @@ use crate::core::filesystem::{
relative_match_file,
};
use crate::core::{
- FileInfo, FileSystem, GrepMatch, GrepResult, MultiWriteWrappedFS, Result, TreeEntry, WriteFlag,
+ FileInfo, FileSystem, GlobPage, GrepMatch, GrepResult, MultiWriteWrappedFS, Result,
+ TreeEntry, WriteFlag,
};
use async_trait::async_trait;
use bytes::Bytes;
@@ -1233,6 +1234,27 @@ impl FileSystem for CachedFileSystem {
.tree_directory(path, show_hidden, node_limit, level_limit)
.await
}
+
+ async fn glob_directory(
+ &self,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ page_size: Option<usize>,
+ level_limit: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> Result<GlobPage> {
+ self.backend
+ .glob_directory(
+ path,
+ pattern,
+ show_hidden,
+ page_size,
+ level_limit,
+ continuation_token,
+ )
+ .await
+ }
}
fn normalize_path(path: &str) -> String {
diff --git a/crates/ragfs/src/core/encryption_wrapper.rs b/crates/ragfs/src/core/encryption_wrapper.rs
index 98e5f3e82..163fb88a3 100644
--- a/crates/ragfs/src/core/encryption_wrapper.rs
+++ b/crates/ragfs/src/core/encryption_wrapper.rs
@@ -22,7 +22,7 @@ use crate::shape::SHAPE_MANIFEST_PATH;
use super::context::FsContextView;
use super::errors::{Error, Result};
use super::filesystem::{compile_grep_regex, normalize_prefix_path, FileSystem};
-use super::types::{FileInfo, GrepResult, TreeEntry, WriteFlag};
+use super::types::{FileInfo, GlobPage, GrepResult, TreeEntry, WriteFlag};
const SYSTEM_ACCOUNT_ID: &str = "_system";
const TEMP_ROOT_CACHE_TTL: Duration = Duration::from_secs(15 * 60);
@@ -438,6 +438,31 @@ impl FileSystem for EncryptionWrappedFS {
.collect())
}
+ async fn glob_directory(
+ &self,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ page_size: Option<usize>,
+ level_limit: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> Result<GlobPage> {
+ let mut page = self
+ .inner
+ .glob_directory(
+ path,
+ pattern,
+ show_hidden,
+ page_size,
+ level_limit,
+ continuation_token,
+ )
+ .await?;
+ page.entries
+ .retain(|entry| !Self::is_shape_manifest_path(&entry.path));
+ Ok(page)
+ }
+
async fn ensure_parent_dirs(&self, path: &str, mode: u32) -> Result<()> {
self.inner.ensure_parent_dirs(path, mode).await
}
diff --git a/crates/ragfs/src/core/filesystem.rs b/crates/ragfs/src/core/filesystem.rs
index c8d4c3fa7..acc779e2c 100644
--- a/crates/ragfs/src/core/filesystem.rs
+++ b/crates/ragfs/src/core/filesystem.rs
@@ -10,7 +10,10 @@ use std::any::Any;
use std::cmp::Ordering;
use super::errors::{Error, Result};
-use super::types::{FileInfo, GrepResult, TreeEntry, WriteFlag};
+use super::glob::{
+ compare_rel_paths, decode_offset_token, encode_offset_token, PreparedGlob,
+};
+use super::types::{FileInfo, GlobEntry, GlobPage, GrepResult, TreeEntry, WriteFlag};
/// Normalize a path for prefix comparisons.
///
@@ -470,6 +473,65 @@ pub trait FileSystem: Send + Sync + Any {
Ok(result)
}
+ /// Return one page of flat glob results under `path`.
+ ///
+ /// The default implementation preserves the current Python behavior by
+ /// reusing `tree_directory()` and matching against the returned `rel_path`
+ /// values, then slicing matches with an opaque continuation token.
+ async fn glob_directory(
+ &self,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ page_size: Option<usize>,
+ level_limit: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> Result<GlobPage> {
+ let matcher = PreparedGlob::new(pattern)?;
+ if matches!(page_size, Some(0)) {
+ return Err(Error::invalid_operation("page_size must be positive"));
+ }
+
+ let entries = self
+ .tree_directory(path, show_hidden, None, level_limit)
+ .await?;
+
+ let mut matched = Vec::new();
+ for entry in entries {
+ if matcher.is_match(&entry.rel_path) {
+ matched.push(GlobEntry {
+ path: entry.path,
+ rel_path: entry.rel_path,
+ name: entry.info.name,
+ is_dir: entry.info.is_dir,
+ });
+ }
+ }
+ matched.sort_by(|left, right| compare_rel_paths(&left.rel_path, &right.rel_path));
+
+ let start = decode_offset_token(
+ continuation_token.as_deref(),
+ path,
+ pattern,
+ show_hidden,
+ level_limit,
+ )?;
+ if start > matched.len() {
+ return Err(Error::invalid_operation("continuation token out of range"));
+ }
+ let end = page_size
+ .map(|limit| start.saturating_add(limit))
+ .unwrap_or(matched.len())
+ .min(matched.len());
+ let next_token = (end < matched.len())
+ .then(|| encode_offset_token(end, path, pattern, show_hidden, level_limit));
+
+ Ok(GlobPage {
+ entries: matched[start..end].to_vec(),
+ next_token,
+ })
+ }
+
/// Internal recursive helper for tree_directory.
///
/// # Arguments
@@ -1063,4 +1125,139 @@ mod tests {
assert!(names.contains(&"secret.txt".to_string()));
assert!(!names.contains(&".hidden_file".to_string()));
}
+
+ /// Test helper that calls `glob_directory` with a fixed `/root` query root.
+ ///
+ /// Args:
+ /// - `fs`: The `TreeFS` instance under test.
+ /// - `pattern`: The glob pattern to match.
+ /// - `page_size`: The requested page size.
+ /// - `continuation_token`: The pagination token for the next page.
+ ///
+ /// Returns:
+ /// - A `GlobPage` on success. In tests this helper uses `unwrap()`, so any
+ /// error fails the test immediately.
+ async fn root_glob(
+ fs: &TreeFS,
+ pattern: &str,
+ page_size: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> crate::core::GlobPage {
+ fs.glob_directory("/root", pattern, false, page_size, None, continuation_token)
+ .await
+ .unwrap()
+ }
+
+ /// Test helper that extracts each entry's `rel_path` from a `GlobPage`.
+ ///
+ /// Args:
+ /// - `page`: The glob page whose relative paths should be collected.
+ ///
+ /// Returns:
+ /// - A list of `rel_path` values in their original order, suitable for
+ /// result-content and ordering assertions.
+ fn glob_rel_paths(page: &crate::core::GlobPage) -> Vec<String> {
+ page.entries
+ .iter()
+ .map(|entry| entry.rel_path.clone())
+ .collect()
+ }
+
+ #[tokio::test]
+ async fn test_glob_directory_matches_full_relative_path_semantics() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/root", vec![("sub", true), ("top.md", false)])
+ .with_dir_entries(
+ "/root/sub",
+ vec![("nested.md", false), ("nested.txt", false)],
+ );
+
+ let page = root_glob(&fs, "**/*.md", None, None).await;
+
+ assert_eq!(glob_rel_paths(&page), vec!["sub/nested.md", "top.md"]);
+ assert!(page.next_token.is_none());
+ }
+
+ #[tokio::test]
+ async fn test_glob_directory_anchors_multi_segment_patterns_at_root() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/root", vec![("a", true), ("x", true)])
+ .with_dir_entries("/root/a", vec![("b", true)])
+ .with_dir_entries("/root/a/b", vec![("c.md", false)])
+ .with_dir_entries("/root/x", vec![("a", true)])
+ .with_dir_entries("/root/x/a", vec![("b", true)])
+ .with_dir_entries("/root/x/a/b", vec![("c.md", false)]);
+
+ let page = root_glob(&fs, "a/**/*.md", None, None).await;
+
+ assert_eq!(glob_rel_paths(&page), vec!["a/b/c.md"]);
+ }
+
+ #[tokio::test]
+ async fn test_glob_directory_paginates_with_opaque_offset_tokens() {
+ let fs = TreeFS::default().with_dir_entries(
+ "/root",
+ vec![("a.md", false), ("b.md", false), ("c.md", false)],
+ );
+
+ let first = root_glob(&fs, "*.md", Some(2), None).await;
+ assert_eq!(glob_rel_paths(&first), vec!["a.md", "b.md"]);
+ assert!(first.next_token.is_some());
+
+ let second = root_glob(&fs, "*.md", Some(2), first.next_token).await;
+ assert_eq!(glob_rel_paths(&second), vec!["c.md"]);
+ assert!(second.next_token.is_none());
+ }
+
+ #[tokio::test]
+ async fn test_glob_directory_rejects_token_from_different_query_scope() {
+ let fs = TreeFS::default().with_dir_entries(
+ "/root",
+ vec![("a.md", false), ("b.md", false), ("c.md", false)],
+ );
+
+ let first = root_glob(&fs, "*.md", Some(2), None).await;
+ let err = fs
+ .glob_directory("/root", "*.txt", false, Some(2), None, first.next_token)
+ .await
+ .unwrap_err();
+
+ assert!(matches!(err, Error::InvalidOperation(_)));
+ }
+
+ #[tokio::test]
+ async fn test_glob_directory_empty_pattern_is_invalid() {
+ let fs = TreeFS::default().with_dir_entries("/root", vec![("a.md", false)]);
+
+ let err = fs
+ .glob_directory("/root", "", false, None, None, None)
+ .await
+ .unwrap_err();
+
+ assert!(matches!(err, Error::InvalidOperation(_)));
+ }
+
+ #[tokio::test]
+ async fn test_glob_directory_empty_pattern_is_invalid_for_empty_directory() {
+ let fs = TreeFS::default().with_dir_entries("/root", vec![]);
+
+ let err = fs
+ .glob_directory("/root", "", false, None, None, None)
+ .await
+ .unwrap_err();
+
+ assert!(matches!(err, Error::InvalidOperation(_)));
+ }
+
+ #[tokio::test]
+ async fn test_glob_directory_zero_page_size_is_invalid() {
+ let fs = TreeFS::default().with_dir_entries("/root", vec![("a.md", false)]);
+
+ let err = fs
+ .glob_directory("/root", "*.md", false, Some(0), None, None)
+ .await
+ .unwrap_err();
+
+ assert!(matches!(err, Error::InvalidOperation(_)));
+ }
}
diff --git a/crates/ragfs/src/core/glob.rs b/crates/ragfs/src/core/glob.rs
new file mode 100644
index 000000000..b4c44d053
--- /dev/null
+++ b/crates/ragfs/src/core/glob.rs
@@ -0,0 +1,251 @@
+//! Glob helpers shared by the default trait implementation and backend
+//! overrides.
+
+use std::cmp::Ordering;
+
+use globset::{GlobBuilder, GlobMatcher};
+use sha2::{Digest, Sha256};
+
+use crate::core::{Error, Result};
+
+/// Precompiled standard glob matcher for query-root-relative paths.
+pub struct PreparedGlob {
+ matcher: Option<GlobMatcher>,
+}
+
+impl PreparedGlob {
+ /// Compile a glob pattern once for repeated full relative-path checks.
+ pub fn new(pattern: &str) -> Result<Self> {
+ validate_pattern(pattern)?;
+ let normalized = normalize_rel_path(pattern);
+ let matcher = if normalized.is_empty() {
+ None
+ } else {
+ Some(
+ GlobBuilder::new(&normalized)
+ .literal_separator(true)
+ .build()
+ .map_err(|err| {
+ Error::invalid_operation(format!("invalid glob pattern: {err}"))
+ })?
+ .compile_matcher(),
+ )
+ };
+ Ok(Self { matcher })
+ }
+
+ /// Match one path relative to the query root using standard glob semantics.
+ pub fn is_match(&self, rel_path: &str) -> bool {
+ self.matcher
+ .as_ref()
+ .is_some_and(|matcher| matcher.is_match(normalize_rel_path(rel_path)))
+ }
+}
+
+#[allow(dead_code)]
+pub fn standard_glob_match(rel_path: &str, pattern: &str) -> Result<bool> {
+ Ok(PreparedGlob::new(pattern)?.is_match(rel_path))
+}
+
+/// Decode the opaque offset token used by the default implementations.
+pub fn decode_offset_token(
+ token: Option<&str>,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ level_limit: Option<usize>,
+) -> Result<usize> {
+ match token {
+ None => Ok(0),
+ Some(raw) if raw.is_empty() => Err(Error::invalid_operation("empty continuation token")),
+ Some(raw) => {
+ let Some((scope, offset)) = raw.split_once(':') else {
+ return Err(Error::invalid_operation("invalid continuation token"));
+ };
+ if scope != token_scope(path, pattern, show_hidden, level_limit) {
+ return Err(Error::invalid_operation(
+ "continuation token scope mismatch",
+ ));
+ }
+ offset
+ .parse::<usize>()
+ .map_err(|_| Error::invalid_operation("invalid continuation token"))
+ }
+ }
+}
+
+/// Encode the opaque offset token used by the default implementations.
+pub fn encode_offset_token(
+ offset: usize,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ level_limit: Option<usize>,
+) -> String {
+ format!(
+ "{}:{offset}",
+ token_scope(path, pattern, show_hidden, level_limit)
+ )
+}
+
+/// Validate whether a glob pattern is acceptable for matching.
+///
+/// Separator-only patterns such as `"/"` are valid inputs, but they do not
+/// match any relative path.
+pub fn validate_pattern(pattern: &str) -> Result<()> {
+ if pattern.is_empty() {
+ return Err(Error::invalid_operation("empty glob pattern"));
+ }
+
+ let mut saw_segment = false;
+ for segment in pattern.split('/') {
+ if segment.is_empty() {
+ continue;
+ }
+ saw_segment = true;
+ if segment != "." {
+ return Ok(());
+ }
+ }
+
+ if saw_segment {
+ return Err(Error::invalid_operation("empty glob pattern"));
+ }
+
+ Ok(())
+}
+
+/// Compare two relative paths lexicographically by path component, ignoring
+/// empty segments and `.` segments.
+pub fn compare_rel_paths(left: &str, right: &str) -> Ordering {
+ split_segments(left).cmp(&split_segments(right))
+}
+
+fn split_segments(value: &str) -> Vec<String> {
+ value
+ .split('/')
+ .filter(|segment| !segment.is_empty() && *segment != ".")
+ .map(ToString::to_string)
+ .collect()
+}
+
+fn normalize_rel_path(value: &str) -> String {
+ split_segments(value).join("/")
+}
+
+fn token_scope(path: &str, pattern: &str, show_hidden: bool, level_limit: Option<usize>) -> String {
+ let mut hasher = Sha256::new();
+ hasher.update(path.as_bytes());
+ hasher.update([0]);
+ hasher.update(pattern.as_bytes());
+ hasher.update([0, show_hidden as u8, 0]);
+ hasher.update(level_limit.unwrap_or(usize::MAX).to_string().as_bytes());
+ format!("{:x}", hasher.finalize())
+}
+
+#[cfg(test)]
+mod tests {
+ use std::cmp::Ordering;
+
+ use super::{
+ compare_rel_paths, decode_offset_token, encode_offset_token, standard_glob_match,
+ validate_pattern, PreparedGlob,
+ };
+
+ #[test]
+ fn test_globset_match_uses_full_relative_path() {
+ assert!(!standard_glob_match("foo/bar.rs", "*.rs").unwrap());
+ assert!(!standard_glob_match("sub/resource_a", "resource_*").unwrap());
+ assert!(standard_glob_match("foo/bar.rs", "**/*.rs").unwrap());
+ assert!(!standard_glob_match("foo/bar.rs", "*.md").unwrap());
+ }
+
+ #[test]
+ fn test_globset_match_uses_standard_recursive_semantics() {
+ assert!(standard_glob_match("a/b/c.md", "a/**/*.md").unwrap());
+ assert!(!standard_glob_match("a/b/c.md", "*/c.md").unwrap());
+ assert!(standard_glob_match("a/c.md", "*/c.md").unwrap());
+ assert!(standard_glob_match("foo/bar", "foo/**").unwrap());
+ assert!(standard_glob_match("foo/bar/baz", "foo/**").unwrap());
+ assert!(standard_glob_match("foo", "**/foo").unwrap());
+ assert!(standard_glob_match("a.md", "**/*.md").unwrap());
+ assert!(standard_glob_match("x/a.md", "**/*.md").unwrap());
+ }
+
+ #[test]
+ fn test_standard_glob_match_empty_pattern_rejected() {
+ assert!(standard_glob_match("a", "").is_err());
+ }
+
+ #[test]
+ fn test_standard_glob_match_root_only_pattern_returns_false() {
+ assert!(!standard_glob_match("a", "/").unwrap());
+ assert!(!standard_glob_match("a", "///").unwrap());
+ }
+
+ #[test]
+ fn test_standard_glob_match_dot_only_pattern_still_rejected() {
+ assert!(standard_glob_match("a", ".").is_err());
+ assert!(standard_glob_match("a", "./").is_err());
+ assert!(standard_glob_match("a", "././").is_err());
+ }
+
+ #[test]
+ fn test_standard_glob_match_invalid_segment_is_rejected() {
+ assert!(standard_glob_match("foo", "[").is_err());
+ assert!(standard_glob_match("foo", "foo[bar").is_err());
+ assert!(standard_glob_match("foo", "[]").is_err());
+ }
+
+ #[test]
+ fn test_standard_glob_match_normalizes_dot_segments() {
+ assert!(standard_glob_match("a/./b.md", "a/b.md").unwrap());
+ assert!(!standard_glob_match("a/b.md", "./b.md").unwrap());
+ assert!(standard_glob_match("./a/b.md", "a/b.md").unwrap());
+ }
+
+ #[test]
+ fn test_prepared_glob_matches_globset_behavior() {
+ let cases = [
+ ("foo/bar.rs", "**/*.rs"),
+ ("a/b/c.md", "**/*.md"),
+ ("foo", "**/foo"),
+ ("foo/bar/baz", "foo/**"),
+ ("a", "/"),
+ ];
+
+ for (path, pattern) in cases {
+ let prepared = PreparedGlob::new(pattern).unwrap();
+ assert_eq!(prepared.is_match(path), standard_glob_match(path, pattern).unwrap());
+ }
+ }
+
+ #[test]
+ fn test_offset_token_rejects_scope_mismatch() {
+ let token = encode_offset_token(2, "/root", "*.md", false, None);
+
+ assert_eq!(
+ decode_offset_token(Some(&token), "/root", "*.md", false, None).unwrap(),
+ 2
+ );
+ assert!(decode_offset_token(Some(&token), "/root", "*.txt", false, None).is_err());
+ }
+
+ #[test]
+ fn test_validate_pattern_distinguishes_root_only_and_dot_only() {
+ validate_pattern("/").unwrap();
+ validate_pattern("///").unwrap();
+ assert!(validate_pattern(".").is_err());
+ assert!(validate_pattern("./").is_err());
+ validate_pattern("a/./b").unwrap();
+ }
+
+ #[test]
+ fn test_compare_rel_paths_uses_path_component_order() {
+ assert_eq!(compare_rel_paths("a", "a"), Ordering::Equal);
+ assert_eq!(compare_rel_paths("a", "a/b"), Ordering::Less);
+ assert_eq!(compare_rel_paths("a/b/c.txt", "a/b.txt"), Ordering::Less);
+ assert_eq!(compare_rel_paths("a//b", "a/b"), Ordering::Equal);
+ assert_eq!(compare_rel_paths("./a", "a"), Ordering::Equal);
+ }
+}
diff --git a/crates/ragfs/src/core/mod.rs b/crates/ragfs/src/core/mod.rs
index b7469ce8b..6bc43e939 100644
--- a/crates/ragfs/src/core/mod.rs
+++ b/crates/ragfs/src/core/mod.rs
@@ -12,6 +12,7 @@ pub mod context;
pub mod encryption_wrapper;
pub mod errors;
pub mod filesystem;
+pub mod glob;
pub mod mountable;
pub mod multibackend_wrapper;
pub mod plugin;
@@ -36,7 +37,7 @@ pub use stats::{FilesystemStats, FsOperation, OperationStats, OperationTimer, St
pub use stats_wrapper::StatsWrappedFS;
pub use types::{
BackendItemConfig, BackendRole, BackendSyncState, BackendsConfig, ConfigParameter, ConfigValue,
- EncryptionConfig, FileInfo, GrepMatch, GrepResult, OperationItemConfig, PluginConfig,
- RedirectEntry, RedirectMeta, RedirectPolicy, SyncLogEntry, SyncLogMeta, SyncOp, SyncType,
- TreeEntry, WriteFlag,
+ EncryptionConfig, FileInfo, GlobEntry, GlobPage, GrepMatch, GrepResult,
+ OperationItemConfig, PluginConfig, RedirectEntry, RedirectMeta, RedirectPolicy,
+ SyncLogEntry, SyncLogMeta, SyncOp, SyncType, TreeEntry, WriteFlag,
};
diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs
index 4dac8a5d1..0c5bced5a 100644
--- a/crates/ragfs/src/core/mountable.rs
+++ b/crates/ragfs/src/core/mountable.rs
@@ -26,7 +26,9 @@ use super::multibackend_wrapper::MultiWriteWrappedFS;
use super::plugin::ServicePlugin;
use super::stats::{FilesystemStats, StatsCollector};
use super::stats_wrapper::StatsWrappedFS;
-use super::types::{BackendsConfig, FileInfo, GrepResult, PluginConfig, TreeEntry, WriteFlag};
+use super::types::{
+ BackendsConfig, FileInfo, GlobPage, GrepResult, PluginConfig, TreeEntry, WriteFlag,
+};
#[cfg(feature = "cache")]
use crate::cache::{
CacheNamespace, CachePolicy, CacheProvider, CacheTraversalMode, CachedFileSystem,
@@ -909,6 +911,56 @@ impl FileSystem for MountableFS {
Ok(entries)
}
+
+ async fn glob_directory(
+ &self,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ page_size: Option<usize>,
+ level_limit: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> Result<GlobPage> {
+ let (mount_info, rel_path) = self.find_mount(path).await?;
+
+ let mount_prefix = if mount_info.path == "/" {
+ String::new()
+ } else {
+ mount_info.path.clone()
+ };
+
+ let mut page = mount_info
+ .fs
+ .glob_directory(
+ &rel_path,
+ pattern,
+ show_hidden,
+ page_size,
+ level_limit,
+ continuation_token,
+ )
+ .await?;
+
+ for entry in &mut page.entries {
+ if !mount_prefix.is_empty() {
+ entry.path = if entry.path == "/" {
+ mount_prefix.clone()
+ } else {
+ format!("{}{}", mount_prefix, entry.path)
+ };
+ }
+ }
+
+ page.entries.retain(|entry| {
+ entry
+ .path
+ .rsplit('/')
+ .next()
+ .map_or(true, |name| name != PATH_LOCK_FILE)
+ });
+
+ Ok(page)
+ }
}
#[cfg(test)]
diff --git a/crates/ragfs/src/core/multibackend_wrapper.rs b/crates/ragfs/src/core/multibackend_wrapper.rs
index dab454415..4dbb600c0 100644
--- a/crates/ragfs/src/core/multibackend_wrapper.rs
+++ b/crates/ragfs/src/core/multibackend_wrapper.rs
@@ -22,8 +22,12 @@ use super::context::{FsContext, FS_CTX};
use super::errors::{Error, Result};
use super::filesystem::{normalize_prefix_path, relative_match_file, FileSystem};
use super::types::{
- BackendRole, BackendSyncState, FileInfo, GrepResult, OperationItemConfig, RedirectEntry,
- RedirectPolicy, SyncLogEntry, SyncOp, SyncType, TreeEntry, WriteFlag,
+ BackendRole, BackendSyncState, FileInfo, GlobEntry, GlobPage, GrepResult,
+ OperationItemConfig, RedirectEntry, RedirectPolicy, SyncLogEntry, SyncOp, SyncType, TreeEntry,
+ WriteFlag,
+};
+use crate::core::glob::{
+ compare_rel_paths, decode_offset_token, encode_offset_token, PreparedGlob,
};
use crate::multibackend::meta::{
current_required_ctx, file_name, parent_dir, DefaultFsContextResolver, FsContextResolver,
@@ -1601,6 +1605,76 @@ impl FileSystem for MultiWriteWrappedFS {
Ok(result)
}
+ async fn glob_directory(
+ &self,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ page_size: Option<usize>,
+ level_limit: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> Result<GlobPage> {
+ if self.inner.redirects.is_empty() {
+ return self
+ .inner
+ .primary()
+ .backend
+ .glob_directory(
+ path,
+ pattern,
+ show_hidden,
+ page_size,
+ level_limit,
+ continuation_token,
+ )
+ .await;
+ }
+
+ let matcher = PreparedGlob::new(pattern)?;
+ if matches!(page_size, Some(0)) {
+ return Err(Error::invalid_operation("page_size must be positive"));
+ }
+
+ let entries = self
+ .tree_directory(path, show_hidden, None, level_limit)
+ .await?;
+
+ let mut matched = Vec::new();
+ for entry in entries {
+ if matcher.is_match(&entry.rel_path) {
+ matched.push(GlobEntry {
+ path: entry.path,
+ rel_path: entry.rel_path,
+ name: entry.info.name,
+ is_dir: entry.info.is_dir,
+ });
+ }
+ }
+ matched.sort_by(|left, right| compare_rel_paths(&left.rel_path, &right.rel_path));
+
+ let start = decode_offset_token(
+ continuation_token.as_deref(),
+ path,
+ pattern,
+ show_hidden,
+ level_limit,
+ )?;
+ if start > matched.len() {
+ return Err(Error::invalid_operation("continuation token out of range"));
+ }
+ let end = page_size
+ .map(|limit| start.saturating_add(limit))
+ .unwrap_or(matched.len())
+ .min(matched.len());
+ let next_token = (end < matched.len())
+ .then(|| encode_offset_token(end, path, pattern, show_hidden, level_limit));
+
+ Ok(GlobPage {
+ entries: matched[start..end].to_vec(),
+ next_token,
+ })
+ }
+
async fn tree_directory(
&self,
path: &str,
diff --git a/crates/ragfs/src/core/stats.rs b/crates/ragfs/src/core/stats.rs
index 0f36b52a9..0a626c7c5 100644
--- a/crates/ragfs/src/core/stats.rs
+++ b/crates/ragfs/src/core/stats.rs
@@ -42,6 +42,8 @@ pub enum FsOperation {
EnsureParentDirs,
/// Tree directory operation
TreeDir,
+ /// Glob directory operation
+ GlobDir,
}
impl FsOperation {
@@ -63,6 +65,7 @@ impl FsOperation {
FsOperation::Grep,
FsOperation::EnsureParentDirs,
FsOperation::TreeDir,
+ FsOperation::GlobDir,
]
}
@@ -84,6 +87,7 @@ impl FsOperation {
FsOperation::Grep => "grep",
FsOperation::EnsureParentDirs => "ensure_parent_dirs",
FsOperation::TreeDir => "tree_dir",
+ FsOperation::GlobDir => "glob_dir",
}
}
}
diff --git a/crates/ragfs/src/core/stats_wrapper.rs b/crates/ragfs/src/core/stats_wrapper.rs
index cfc76f254..3ee069ca1 100644
--- a/crates/ragfs/src/core/stats_wrapper.rs
+++ b/crates/ragfs/src/core/stats_wrapper.rs
@@ -7,8 +7,8 @@ use async_trait::async_trait;
use std::sync::Arc;
use super::{
- FileInfo, FileSystem, FsOperation, GrepResult, OperationTimer, Result, StatsCollector,
- TreeEntry, WriteFlag,
+ FileInfo, FileSystem, FsOperation, GlobPage, GrepResult, OperationTimer, Result,
+ StatsCollector, TreeEntry, WriteFlag,
};
/// A wrapper around FileSystem that automatically collects operation statistics
@@ -204,4 +204,29 @@ impl FileSystem for StatsWrappedFS {
timer.finish().await;
result
}
+
+ async fn glob_directory(
+ &self,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ page_size: Option<usize>,
+ level_limit: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> Result<GlobPage> {
+ let timer = OperationTimer::start(FsOperation::GlobDir, Arc::clone(&self.stats));
+ let result = self
+ .inner
+ .glob_directory(
+ path,
+ pattern,
+ show_hidden,
+ page_size,
+ level_limit,
+ continuation_token,
+ )
+ .await;
+ timer.finish().await;
+ result
+ }
}
diff --git a/crates/ragfs/src/core/types.rs b/crates/ragfs/src/core/types.rs
index eb98a5be9..767091b0b 100644
--- a/crates/ragfs/src/core/types.rs
+++ b/crates/ragfs/src/core/types.rs
@@ -58,6 +58,37 @@ pub struct TreeEntry {
pub extra: HashMap<String, serde_json::Value>,
}
+/// Flat glob match entry.
+///
+/// Represents one path matched by `glob_directory`, preserving enough metadata
+/// for Python-side visibility and URI alias handling without reconstructing a
+/// full `TreeEntry`.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GlobEntry {
+ /// Plugin-root-relative absolute path, matching the `TreeEntry.path`
+ /// contract after mount rewriting.
+ pub path: String,
+
+ /// Path relative to the glob query root.
+ pub rel_path: String,
+
+ /// Final path component.
+ pub name: String,
+
+ /// Whether the matched entry is a directory.
+ pub is_dir: bool,
+}
+
+/// One page of glob results.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GlobPage {
+ /// Matched entries for this page.
+ pub entries: Vec<GlobEntry>,
+
+ /// Opaque continuation token for the next page.
+ pub next_token: Option<String>,
+}
+
impl GrepResult {
/// Create a new empty GrepResult
pub fn new() -> Self {
diff --git a/crates/ragfs/src/lib.rs b/crates/ragfs/src/lib.rs
index f4582e298..082e9b4e1 100644
--- a/crates/ragfs/src/lib.rs
+++ b/crates/ragfs/src/lib.rs
@@ -41,8 +41,8 @@ pub mod shape;
// Re-export core types for convenience
pub use core::{
ConfigParameter, ConfigValue, Error, FileInfo, FileSystem, FilesystemStats, FsOperation,
- HealthStatus, MountableFS, OperationStats, OperationTimer, PluginConfig, PluginRegistry,
- Result, ServicePlugin, StatsCollector, StatsWrappedFS, WriteFlag,
+ GlobEntry, GlobPage, HealthStatus, MountableFS, OperationStats, OperationTimer, PluginConfig,
+ PluginRegistry, Result, ServicePlugin, StatsCollector, StatsWrappedFS, WriteFlag,
};
/// Version of RAGFS
diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rs
index 8ab96b6c7..dc357087c 100644
--- a/crates/ragfs/src/plugins/localfs/mod.rs
+++ b/crates/ragfs/src/plugins/localfs/mod.rs
@@ -18,8 +18,9 @@ use serde::Deserialize;
use crate::core::errors::{Error, Result};
use crate::core::filesystem::FileSystem;
+use crate::core::glob::{decode_offset_token, encode_offset_token, PreparedGlob};
use crate::core::plugin::ServicePlugin;
-use crate::core::types::{ConfigParameter, FileInfo, GrepResult, PluginConfig, WriteFlag};
+use crate::core::types::{ConfigParameter, FileInfo, GlobEntry, GlobPage, GrepResult, PluginConfig, WriteFlag};
/// LocalFS - Local file system implementation
pub struct LocalFileSystem {
@@ -87,8 +88,8 @@ impl LocalFileSystem {
}
}
- /// Run blocking grep work on a dedicated thread and normalize join errors.
- async fn run_blocking_grep<T, F>(job: F) -> Result<T>
+ /// Run blocking local filesystem work on a dedicated thread.
+ async fn run_blocking_fs<T, F>(job: F) -> Result<T>
where
T: Send + 'static,
F: FnOnce() -> Result<T> + Send + 'static,
@@ -509,14 +510,158 @@ impl LocalFileSystem {
}
fn local_to_query_relative_path(query_root: &Path, local: &Path) -> Option<String> {
let rel = local.strip_prefix(query_root).ok()?;
- let s = rel.to_string_lossy();
- if s.is_empty() {
+ let parts = rel
+ .components()
+ .map(|component| component.as_os_str().to_string_lossy())
+ .collect::<Vec<_>>();
+ if parts.is_empty() {
Some(".".to_string())
} else {
- Some(s.to_string())
+ Some(parts.join("/"))
+ }
+ }
+
+ /// Build the plugin-root-relative path required by the `GlobEntry.path` contract.
+ fn local_glob_entry_path(virtual_root: &str, rel_path: &str) -> String {
+ if virtual_root == "/" {
+ format!("/{}", rel_path)
+ } else {
+ format!("{}/{}", virtual_root.trim_end_matches('/'), rel_path)
+ }
+ }
+
+ fn glob_static_max_depth(pattern: &str) -> Option<usize> {
+ let mut depth = 0usize;
+ for segment in pattern.split('/') {
+ if segment.is_empty() || segment == "." {
+ continue;
+ }
+ if segment == "**" {
+ return None;
+ }
+ depth += 1;
+ }
+ Some(depth)
+ }
+
+ fn effective_glob_max_depth(pattern: &str, level_limit: Option<usize>) -> Option<usize> {
+ match (Self::glob_static_max_depth(pattern), level_limit) {
+ (Some(pattern_depth), Some(level_depth)) => Some(pattern_depth.min(level_depth)),
+ (Some(pattern_depth), None) => Some(pattern_depth),
+ (None, Some(level_depth)) => Some(level_depth),
+ (None, None) => None,
}
}
+ /// Implement glob pagination via `ignore::WalkBuilder` without materializing the whole tree.
+ ///
+ /// Args:
+ /// - `base_path`: Absolute local path to the mounted root directory.
+ /// - `virtual_path`: Query path inside the plugin mount.
+ /// - `pattern`: Standard glob pattern matched against query-root-relative paths.
+ /// - `show_hidden`: Whether hidden files should be included.
+ /// - `page_size`: Page size, or `None` to return all matches.
+ /// - `level_limit`: Maximum traversal depth.
+ /// - `continuation_token`: Continuation token returned by the previous page.
+ ///
+ /// Returns:
+ /// - The current `GlobPage`, or an error if the path is missing, not a directory,
+ /// the token is invalid, or traversal fails.
+ fn glob_via_walk(
+ base_path: &Path,
+ virtual_path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ page_size: Option<usize>,
+ level_limit: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> Result<GlobPage> {
+ let matcher = PreparedGlob::new(pattern)?;
+ if matches!(page_size, Some(0)) {
+ return Err(Error::invalid_operation("page_size must be positive"));
+ }
+
+ let query_root = Self::resolve_virtual_path(base_path, virtual_path);
+ if !query_root.exists() {
+ return Err(Error::NotFound(virtual_path.to_string()));
+ }
+ if !query_root.is_dir() {
+ return Err(Error::NotADirectory(virtual_path.to_string()));
+ }
+
+ let start = decode_offset_token(
+ continuation_token.as_deref(),
+ virtual_path,
+ pattern,
+ show_hidden,
+ level_limit,
+ )?;
+ let limit = page_size.unwrap_or(usize::MAX);
+ let mut builder = WalkBuilder::new(&query_root);
+ builder
+ .hidden(false)
+ .parents(false)
+ .ignore(false)
+ .git_ignore(false)
+ .git_global(false)
+ .git_exclude(false)
+ .sort_by_file_path(|left, right| left.cmp(right));
+ if let Some(max_depth) = Self::effective_glob_max_depth(pattern, level_limit) {
+ builder.max_depth(Some(max_depth));
+ }
+
+ let mut walker = builder.build();
+ let mut raw_seen = 0usize;
+ let mut entries = Vec::new();
+ let mut next_offset = None;
+
+ while let Some(dent) = walker.next() {
+ if entries.len() >= limit {
+ next_offset = Some(raw_seen);
+ break;
+ }
+
+ let dent = dent.map_err(|e| Error::plugin(format!("failed to walk directory: {}", e)))?;
+ raw_seen += 1;
+
+ if raw_seen <= start {
+ continue;
+ }
+
+ let rel_path = Self::local_to_query_relative_path(&query_root, dent.path())
+ .ok_or_else(|| Error::InvalidPath(dent.path().display().to_string()))?;
+ if rel_path == "." {
+ continue;
+ }
+
+ let name = dent.file_name().to_string_lossy().to_string();
+ let is_dir = dent
+ .file_type()
+ .map(|file_type| file_type.is_dir())
+ .unwrap_or_else(|| dent.path().is_dir());
+
+ if !show_hidden && !is_dir && name.starts_with('.') {
+ continue;
+ }
+ if !matcher.is_match(&rel_path) {
+ continue;
+ }
+
+ entries.push(GlobEntry {
+ path: Self::local_glob_entry_path(virtual_path, &rel_path),
+ rel_path,
+ name,
+ is_dir,
+ });
+ }
+
+ let next_token = next_offset
+ .filter(|_| page_size.is_some())
+ .map(|offset| encode_offset_token(offset, virtual_path, pattern, show_hidden, level_limit));
+
+ Ok(GlobPage { entries, next_token })
+ }
+
fn resolve_virtual_path(base_path: &Path, path: &str) -> PathBuf {
let relative = path.strip_prefix('/').unwrap_or(path);
if relative.is_empty() {
@@ -687,68 +832,77 @@ impl FileSystem for LocalFileSystem {
async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {
let local_path = self.resolve_path(path);
+ let path = path.to_string();
- // Check if directory exists
- if !local_path.exists() {
- return Err(Error::NotFound(path.to_string()));
- }
+ Self::run_blocking_fs(move || {
+ // Check if directory exists
+ if !local_path.exists() {
+ return Err(Error::NotFound(path.clone()));
+ }
- if !local_path.is_dir() {
- return Err(Error::plugin(format!("not a directory: {}", path)));
- }
+ if !local_path.is_dir() {
+ return Err(Error::plugin(format!("not a directory: {}", path)));
+ }
- // Read directory
- let entries = fs::read_dir(&local_path)
- .map_err(|e| Error::plugin(format!("failed to read directory: {}", e)))?;
+ // Read directory
+ let entries = fs::read_dir(&local_path)
+ .map_err(|e| Error::plugin(format!("failed to read directory: {}", e)))?;
- let mut files = Vec::new();
- for entry in entries {
- let entry = entry.map_err(|e| Error::plugin(format!("failed to read entry: {}", e)))?;
- let metadata = entry
- .metadata()
- .map_err(|e| Error::plugin(format!("failed to get metadata: {}", e)))?;
+ let mut files = Vec::new();
+ for entry in entries {
+ let entry =
+ entry.map_err(|e| Error::plugin(format!("failed to read entry: {}", e)))?;
+ let metadata = entry
+ .metadata()
+ .map_err(|e| Error::plugin(format!("failed to get metadata: {}", e)))?;
+
+ let name = entry.file_name().to_string_lossy().to_string();
+ let mode = if metadata.is_dir() { 0o755 } else { 0o644 };
+ let mod_time = metadata
+ .modified()
+ .unwrap_or(std::time::SystemTime::UNIX_EPOCH);
+
+ files.push(FileInfo::new(
+ name,
+ metadata.len(),
+ mode,
+ mod_time,
+ metadata.is_dir(),
+ ));
+ }
- let name = entry.file_name().to_string_lossy().to_string();
+ Ok(files)
+ })
+ .await
+ }
+
+ async fn stat(&self, path: &str) -> Result<FileInfo> {
+ let local_path = self.resolve_path(path);
+ let path = path.to_string();
+
+ Self::run_blocking_fs(move || {
+ // Get file metadata
+ let metadata = fs::metadata(&local_path).map_err(|_| Error::NotFound(path.clone()))?;
+
+ let name = Path::new(&path)
+ .file_name()
+ .unwrap_or(path.as_ref())
+ .to_string_lossy()
+ .to_string();
let mode = if metadata.is_dir() { 0o755 } else { 0o644 };
let mod_time = metadata
.modified()
.unwrap_or(std::time::SystemTime::UNIX_EPOCH);
- files.push(FileInfo::new(
+ Ok(FileInfo::new(
name,
metadata.len(),
mode,
mod_time,
metadata.is_dir(),
- ));
- }
-
- Ok(files)
- }
-
- async fn stat(&self, path: &str) -> Result<FileInfo> {
- let local_path = self.resolve_path(path);
-
- // Get file metadata
- let metadata = fs::metadata(&local_path).map_err(|_| Error::NotFound(path.to_string()))?;
-
- let name = Path::new(path)
- .file_name()
- .unwrap_or(path.as_ref())
- .to_string_lossy()
- .to_string();
- let mode = if metadata.is_dir() { 0o755 } else { 0o644 };
- let mod_time = metadata
- .modified()
- .unwrap_or(std::time::SystemTime::UNIX_EPOCH);
-
- Ok(FileInfo::new(
- name,
- metadata.len(),
- mode,
- mod_time,
- metadata.is_dir(),
- ))
+ ))
+ })
+ .await
}
async fn rename(&self, old_path: &str, new_path: &str) -> Result<()> {
@@ -910,7 +1064,7 @@ impl FileSystem for LocalFileSystem {
let rg_pattern = pattern_owned.clone();
let rg_base_path = base_path.clone();
let rg_exclude = exclude_owned.clone();
- let rg_res = Self::run_blocking_grep(move || {
+ let rg_res = Self::run_blocking_fs(move || {
LocalFileSystem::grep_via_rg(
rg_base_path.as_path(),
local.as_path(),
@@ -930,7 +1084,7 @@ impl FileSystem for LocalFileSystem {
}
// Fallback: also run in a blocking thread (dir walking + file reads are blocking IO).
- Self::run_blocking_grep(move || {
+ Self::run_blocking_fs(move || {
LocalFileSystem::grep_via_libs(
base_path.as_path(),
&path_owned,
@@ -944,11 +1098,51 @@ impl FileSystem for LocalFileSystem {
})
.await
}
+
+ /// Return one glob page in local stable DFS order without building the full directory tree first.
+ ///
+ /// Args:
+ /// - `path`: Query directory path inside the plugin mount.
+ /// - `pattern`: Glob pattern.
+ /// - `show_hidden`: Whether hidden files should be included.
+ /// - `page_size`: Page size, or `None` to return all matches.
+ /// - `level_limit`: Maximum traversal depth.
+ /// - `continuation_token`: Token returned by the previous page.
+ ///
+ /// Returns:
+ /// - The current `GlobPage`, or an error if arguments are invalid or traversal fails.
+ async fn glob_directory(
+ &self,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ page_size: Option<usize>,
+ level_limit: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> Result<GlobPage> {
+ let base_path = self.base_path.clone();
+ let path_owned = path.to_string();
+ let pattern_owned = pattern.to_string();
+
+ Self::run_blocking_fs(move || {
+ LocalFileSystem::glob_via_walk(
+ base_path.as_path(),
+ &path_owned,
+ &pattern_owned,
+ show_hidden,
+ page_size,
+ level_limit,
+ continuation_token,
+ )
+ })
+ .await
+ }
}
#[cfg(test)]
mod tests {
use super::*;
+ use crate::core::FileSystem;
use std::path::Path;
use tempfile::TempDir;
@@ -1201,6 +1395,166 @@ mod tests {
assert_eq!(result.matches[0].file, "a.txt");
assert_eq!(result.matches[0].content, "hello");
}
+
+ #[tokio::test]
+ async fn test_localfs_glob_paginates_with_local_cursor_tokens() {
+ let (dir, fs) = fallback_localfs();
+ write_file(dir.path(), "a.md", "");
+ write_file(dir.path(), "b.md", "");
+ write_file(dir.path(), "c.md", "");
+
+ let first = fs
+ .glob_directory("/", "**/*.md", false, Some(2), None, None)
+ .await
+ .unwrap();
+ assert_eq!(
+ first
+ .entries
+ .iter()
+ .map(|entry| entry.rel_path.clone())
+ .collect::<Vec<_>>(),
+ vec!["a.md", "b.md"]
+ );
+ assert!(first.next_token.is_some());
+ assert_eq!(first.entries[0].path, "/a.md");
+
+ let second = fs
+ .glob_directory("/", "**/*.md", false, Some(2), None, first.next_token)
+ .await
+ .unwrap();
+ assert_eq!(
+ second
+ .entries
+ .iter()
+ .map(|entry| entry.rel_path.clone())
+ .collect::<Vec<_>>(),
+ vec!["c.md"]
+ );
+ assert!(second.next_token.is_none());
+ }
+
+ #[tokio::test]
+ async fn test_localfs_glob_keeps_directory_matches() {
+ let (dir, fs) = fallback_localfs();
+ std::fs::create_dir_all(dir.path().join("folder")).unwrap();
+
+ let out = fs.glob_directory("/", "**/*", false, None, None, None).await.unwrap();
+ assert_eq!(
+ out.entries
+ .iter()
+ .map(|entry| entry.rel_path.clone())
+ .collect::<Vec<_>>(),
+ vec!["folder"]
+ );
+ assert!(out.entries[0].is_dir);
+ }
+
+ #[test]
+ fn test_localfs_relative_path_uses_forward_slashes() {
+ let root = Path::new("root");
+ let local = root.join("a").join("b.md");
+
+ assert_eq!(
+ LocalFileSystem::local_to_query_relative_path(root, &local).unwrap(),
+ "a/b.md"
+ );
+ }
+
+ #[tokio::test]
+ async fn test_localfs_glob_respects_level_limit() {
+ let (dir, fs) = fallback_localfs();
+ write_file(dir.path(), "top.md", "");
+ write_file(dir.path(), "sub/nested.md", "");
+ write_file(dir.path(), "sub/deeper/late.md", "");
+
+ let out = fs
+ .glob_directory("/", "**/*.md", false, None, Some(1), None)
+ .await
+ .unwrap();
+ assert_eq!(
+ out.entries
+ .iter()
+ .map(|entry| entry.rel_path.clone())
+ .collect::<Vec<_>>(),
+ vec!["top.md"]
+ );
+ }
+
+ #[tokio::test]
+ async fn test_localfs_glob_rejects_token_from_different_query_scope() {
+ let (dir, fs) = fallback_localfs();
+ write_file(dir.path(), "a.md", "");
+ write_file(dir.path(), "b.md", "");
+ write_file(dir.path(), "c.md", "");
+
+ let first = fs
+ .glob_directory("/", "**/*.md", false, Some(2), None, None)
+ .await
+ .unwrap();
+ let err = fs
+ .glob_directory("/", "**/*.txt", false, Some(2), None, first.next_token)
+ .await
+ .unwrap_err();
+ assert!(matches!(err, Error::InvalidOperation(_)));
+ }
+
+ #[tokio::test]
+ async fn test_localfs_glob_skips_hidden_files_but_keeps_hidden_dirs() {
+ let (dir, fs) = fallback_localfs();
+ write_file(dir.path(), ".hidden.md", "");
+ write_file(dir.path(), ".hidden_dir/nested.md", "");
+
+ let out = fs.glob_directory("/", "**/*.md", false, None, None, None).await.unwrap();
+ assert_eq!(
+ out.entries
+ .iter()
+ .map(|entry| entry.rel_path.clone())
+ .collect::<Vec<_>>(),
+ vec![".hidden_dir/nested.md"]
+ );
+ }
+
+ #[cfg(unix)]
+ #[tokio::test]
+ async fn test_localfs_glob_defers_late_subtree_errors_to_later_pages() {
+ use std::os::unix::fs::PermissionsExt;
+
+ let (dir, fs) = fallback_localfs();
+ write_file(dir.path(), "a.md", "");
+ write_file(dir.path(), "b.md", "");
+ write_file(dir.path(), "c.md", "");
+ std::fs::create_dir_all(dir.path().join("z_blocked")).unwrap();
+ write_file(dir.path(), "z_blocked/late.md", "");
+
+ let blocked_dir = dir.path().join("z_blocked");
+ let old_mode = std::fs::metadata(&blocked_dir).unwrap().permissions().mode();
+ let mut perms = std::fs::metadata(&blocked_dir).unwrap().permissions();
+ perms.set_mode(0);
+ std::fs::set_permissions(&blocked_dir, perms).unwrap();
+
+ let first = fs.glob_directory("/", "**/*.md", false, Some(2), None, None).await;
+ assert!(first.is_ok());
+ let first = first.unwrap();
+ assert_eq!(
+ first
+ .entries
+ .iter()
+ .map(|entry| entry.rel_path.clone())
+ .collect::<Vec<_>>(),
+ vec!["a.md", "b.md"]
+ );
+ assert!(first.next_token.is_some());
+
+ let out = fs
+ .glob_directory("/", "**/*.md", false, Some(2), None, first.next_token)
+ .await;
+
+ let mut restore = std::fs::metadata(&blocked_dir).unwrap().permissions();
+ restore.set_mode(old_mode);
+ std::fs::set_permissions(&blocked_dir, restore).unwrap();
+
+ assert!(out.is_err());
+ }
}
/// LocalFS plugin
diff --git a/crates/ragfs/src/plugins/s3fs/client.rs b/crates/ragfs/src/plugins/s3fs/client.rs
index c6b405bc0..dddda4593 100644
--- a/crates/ragfs/src/plugins/s3fs/client.rs
+++ b/crates/ragfs/src/plugins/s3fs/client.rs
@@ -252,7 +252,7 @@ impl DirectoryMarkerMode {
}
/// Object metadata from HeadObject
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct ObjectMeta {
/// Object key
pub key: String,
@@ -264,6 +264,13 @@ pub struct ObjectMeta {
pub is_dir_marker: bool,
}
+/// One flat `ListObjectsV2` page used by S3-backed tree/glob traversal.
+#[derive(Debug, Clone)]
+pub struct ListTreePage {
+ pub objects: Vec<ObjectMeta>,
+ pub next_continuation_token: Option<String>,
+}
+
/// Result of a ListObjects operation
#[derive(Debug)]
pub struct ListResult {
@@ -800,6 +807,59 @@ impl S3Client {
Ok(objects)
}
+ /// List one flat object page under a prefix (no delimiter).
+ pub async fn list_tree_objects_page(
+ &self,
+ prefix: &str,
+ continuation_token: Option<&str>,
+ max_keys: usize,
+ ) -> Result<ListTreePage> {
+ let mut req = self
+ .client
+ .list_objects_v2()
+ .bucket(&self.bucket)
+ .prefix(prefix)
+ .max_keys(max_keys.min(1000) as i32);
+
+ if let Some(token) = continuation_token {
+ req = req.continuation_token(token);
+ }
+
+ let resp = req.send().await.map_err(|e| {
+ format_sdk_s3_error(
+ "ListObjectsV2",
+ &format!("bucket={} prefix={prefix}", self.bucket),
+ &e,
+ )
+ })?;
+
+ let mut objects = Vec::new();
+ for obj in resp.contents() {
+ let key = obj.key().unwrap_or("");
+ if key == prefix {
+ continue;
+ }
+
+ let size = obj.size.unwrap_or(0);
+ let last_modified = obj
+ .last_modified()
+ .map(aws_datetime_to_systemtime)
+ .unwrap_or(UNIX_EPOCH);
+
+ objects.push(ObjectMeta {
+ key: key.to_string(),
+ size,
+ last_modified,
+ is_dir_marker: key.ends_with('/'),
+ });
+ }
+
+ Ok(ListTreePage {
+ objects,
+ next_continuation_token: resp.next_continuation_token().map(|s| s.to_string()),
+ })
+ }
+
/// Copy an object
pub async fn copy_object(&self, src_key: &str, dst_key: &str) -> Result<()> {
let copy_source = format!("{}/{}", self.bucket, src_key);
diff --git a/crates/ragfs/src/plugins/s3fs/mod.rs b/crates/ragfs/src/plugins/s3fs/mod.rs
index db764aec2..38e378094 100644
--- a/crates/ragfs/src/plugins/s3fs/mod.rs
+++ b/crates/ragfs/src/plugins/s3fs/mod.rs
@@ -18,21 +18,23 @@ pub mod client;
mod tree;
use async_trait::async_trait;
+use sha2::{Digest, Sha256};
use std::sync::{Arc, Mutex};
use std::time::SystemTime;
use cache::{S3ListDirCache, S3StatCache};
-use client::S3Client;
+use client::{ListTreePage, S3Client};
use futures::stream::{self, StreamExt};
use regex::Regex;
use std::sync::atomic::{AtomicUsize, Ordering};
use crate::core::filesystem::{relative_depth, relative_match_file, sort_directory_entries};
+use crate::core::glob::{PreparedGlob, validate_pattern};
use crate::core::{
- ConfigParameter, Error, FileInfo, FileSystem, GrepMatch, GrepResult, PluginConfig, Result,
- ServicePlugin, TreeEntry, WriteFlag,
+ ConfigParameter, Error, FileInfo, FileSystem, GlobEntry, GlobPage, GrepMatch, GrepResult,
+ PluginConfig, Result, ServicePlugin, TreeEntry, WriteFlag,
};
-use tree::build_tree_entries_from_flat_listing;
+use tree::{build_tree_entries_from_flat_listing, build_tree_entries_from_listing_page, rel_parts};
/// Check whether `path` is under `exclude_path` (including itself).
fn s3_is_excluded_path(path: &str, exclude_path: &str) -> bool {
@@ -59,6 +61,15 @@ struct S3ChunkReader<'a> {
key: &'a str,
}
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+struct S3GlobToken {
+ scope: String,
+ scan_token: Option<String>,
+ page_entries: Vec<GlobEntry>,
+ page_offset: usize,
+ last_rel_parts: Vec<String>,
+}
+
#[async_trait]
impl ChunkReader for S3ChunkReader<'_> {
async fn read_chunk(&mut self, offset: u64, size: u64) -> Result<Vec<u8>> {
@@ -237,6 +248,95 @@ impl S3FileSystem {
result
}
+ fn decode_glob_token(
+ token: Option<&str>,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ level_limit: Option<usize>,
+ ) -> Result<S3GlobToken> {
+ let scope = Self::glob_token_scope(path, pattern, show_hidden, level_limit);
+ match token {
+ None => Ok(S3GlobToken {
+ scope,
+ scan_token: None,
+ page_entries: Vec::new(),
+ page_offset: 0,
+ last_rel_parts: Vec::new(),
+ }),
+ Some(raw) if raw.is_empty() => Err(Error::invalid_operation("empty continuation token")),
+ Some(raw) => {
+ let state: S3GlobToken = serde_json::from_str(raw)
+ .map_err(|_| Error::invalid_operation("invalid continuation token"))?;
+ if state.scope != scope {
+ return Err(Error::invalid_operation("continuation token scope mismatch"));
+ }
+ Ok(state)
+ }
+ }
+ }
+
+ fn encode_glob_token(state: &S3GlobToken) -> Result<String> {
+ // ponytail: the token carries the remaining entries from the current
+ // page directly to avoid extra state storage; the ceiling is bounded
+ // by `page_size` and the S3 single-page limit of 1000 keys.
+ serde_json::to_string(state).map_err(|err| Error::Serialization(err.to_string()))
+ }
+
+ fn glob_token_scope(
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ level_limit: Option<usize>,
+ ) -> String {
+ let mut hasher = Sha256::new();
+ hasher.update(path.as_bytes());
+ hasher.update([0]);
+ hasher.update(pattern.as_bytes());
+ hasher.update([0, show_hidden as u8, 0]);
+ hasher.update(level_limit.unwrap_or(usize::MAX).to_string().as_bytes());
+ format!("{:x}", hasher.finalize())
+ }
+
+ fn glob_entries_from_listing_page(
+ &self,
+ query_root: &str,
+ page: &ListTreePage,
+ matcher: &PreparedGlob,
+ show_hidden: bool,
+ level_limit: Option<usize>,
+ last_rel_parts: &[String],
+ ) -> Result<(Vec<GlobEntry>, Vec<String>)> {
+ let ordered = build_tree_entries_from_listing_page(
+ query_root,
+ &page.objects,
+ show_hidden,
+ level_limit,
+ last_rel_parts,
+ |key| self.client.strip_prefix(key),
+ )?;
+
+ let mut matched = Vec::new();
+ let next_last_rel_parts = ordered
+ .last()
+ .map(|entry| rel_parts(&entry.rel_path))
+ .unwrap_or_else(|| last_rel_parts.to_vec());
+
+ for entry in ordered {
+ if matcher.is_match(&entry.rel_path) {
+ matched.push(GlobEntry {
+ path: entry.path,
+ rel_path: entry.rel_path,
+ name: entry.info.name,
+ is_dir: entry.info.is_dir,
+ });
+ }
+ }
+
+ Ok((matched, next_last_rel_parts))
+ }
+
+
/// Get file name from path
fn file_name(path: &str) -> String {
if path == "/" {
@@ -864,6 +964,98 @@ impl FileSystem for S3FileSystem {
Ok(result)
}
+
+ async fn glob_directory(
+ &self,
+ path: &str,
+ pattern: &str,
+ show_hidden: bool,
+ page_size: Option<usize>,
+ level_limit: Option<usize>,
+ continuation_token: Option<String>,
+ ) -> Result<GlobPage> {
+ let matcher = PreparedGlob::new(pattern)?;
+ if matches!(page_size, Some(0)) {
+ return Err(Error::invalid_operation("page_size must be positive"));
+ }
+
+ let normalized = Self::normalize_path(path);
+ let info = self.stat(&normalized).await?;
+ if !info.is_dir {
+ return Err(Error::NotADirectory(normalized));
+ }
+
+ let prefix = if normalized == "/" {
+ self.client.build_key("")
+ } else {
+ format!("{}/", self.client.build_key(&normalized))
+ };
+ let mut state = Self::decode_glob_token(
+ continuation_token.as_deref(),
+ path,
+ pattern,
+ show_hidden,
+ level_limit,
+ )?;
+ let target_limit = page_size.unwrap_or(usize::MAX);
+ let fetch_size = 1000;
+ let mut matched = Vec::new();
+
+ loop {
+ while state.page_offset < state.page_entries.len() {
+ let entry = state.page_entries[state.page_offset].clone();
+ state.page_offset += 1;
+ matched.push(entry);
+ if matched.len() >= target_limit {
+ let next_token = if state.page_offset < state.page_entries.len()
+ || state.scan_token.is_some()
+ {
+ Some(Self::encode_glob_token(&state)?)
+ } else {
+ None
+ };
+ return Ok(GlobPage {
+ entries: matched,
+ next_token,
+ });
+ }
+ }
+
+ state.page_entries.clear();
+ state.page_offset = 0;
+
+ if state.scan_token.is_none() && !state.last_rel_parts.is_empty() {
+ break;
+ }
+
+ let page = self
+ .client
+ .list_tree_objects_page(&prefix, state.scan_token.as_deref(), fetch_size)
+ .await?;
+ state.scan_token = page.next_continuation_token.clone();
+ let (entries, next_last_rel_parts) = self.glob_entries_from_listing_page(
+ &normalized,
+ &page,
+ &matcher,
+ show_hidden,
+ level_limit,
+ &state.last_rel_parts,
+ )?;
+ state.last_rel_parts = next_last_rel_parts;
+ if entries.is_empty() {
+ if state.scan_token.is_none() {
+ break;
+ }
+ continue;
+ }
+ state.page_entries = entries;
+ }
+
+ Ok(GlobPage {
+ entries: matched,
+ next_token: None,
+ })
+ }
}
/// S3FS Plugin
@@ -1428,7 +1620,7 @@ mod tests {
.await
}
- // ── Case 3: 正则无效 ──
+ // ── Case 3: invalid regex ──
#[test]
fn test_grep_invalid_regex() {
@@ -1442,7 +1634,7 @@ mod tests {
);
}
- // ── Case 17: case_insensitive 正则构建 ──
+ // ── Case 17: case-insensitive regex construction ──
#[test]
fn test_case_insensitive_regex() {
@@ -1456,7 +1648,8 @@ mod tests {
assert!(re2.is_match("world"), "uppercase pattern matches lowercase");
}
- // ── Case 9: 文件大小恰好为 CHUNK_SIZE(单 chunk,is_last 靠 offset+size>=file_size) ──
+ // ── Case 9: file size exactly equals CHUNK_SIZE (single chunk, with
+ // `is_last` determined by `offset + size >= file_size`) ──
#[tokio::test]
async fn test_grep_stream_exact_chunk_single() {
@@ -1472,7 +1665,8 @@ mod tests {
assert_eq!(matches[1].content, "world");
}
- // ── Case 10: 最后 chunk 恰好等于 CHUNK_SIZE(多 chunk 场景) ──
+ // ── Case 10: last chunk size exactly equals CHUNK_SIZE (multi-chunk
+ // scenario) ──
#[tokio::test]
async fn test_grep_stream_last_chunk_exact_size() {
@@ -1483,7 +1677,7 @@ mod tests {
assert_eq!(matches[3].content, "line4");
}
- // ── Case 11: 跨 chunk 边界行的拼接 ──
+ // ── Case 11: line stitching across chunk boundaries ──
#[tokio::test]
async fn test_grep_stream_cross_chunk_line() {
@@ -1495,7 +1689,7 @@ mod tests {
assert_eq!(matches[0].content, "lo wo", "stitched across boundary");
}
- // ── Case 12: 连续多 chunk 无换行符 ──
+ // ── Case 12: multiple consecutive chunks with no newline ──
#[tokio::test]
async fn test_grep_stream_multi_chunk_no_newline() {
@@ -1511,7 +1705,7 @@ mod tests {
);
}
- // ── Case 13: 超长行 > GREP_MAX_PARTIAL_SIZE ──
+ // ── Case 13: oversized line > GREP_MAX_PARTIAL_SIZE ──
#[tokio::test]
async fn test_grep_stream_line_exceeds_max_partial() {
@@ -1524,7 +1718,7 @@ mod tests {
);
}
- // ── Case 14: 二进制内容(含无效 UTF-8) ──
+ // ── Case 14: binary content (including invalid UTF-8) ──
#[tokio::test]
async fn test_grep_stream_binary_content() {
diff --git a/crates/ragfs/src/plugins/s3fs/tree.rs b/crates/ragfs/src/plugins/s3fs/tree.rs
index aad437780..e1f8f1506 100644
--- a/crates/ragfs/src/plugins/s3fs/tree.rs
+++ b/crates/ragfs/src/plugins/s3fs/tree.rs
@@ -148,6 +148,52 @@ where
Ok(result)
}
+/// Build one ordered page of `TreeEntry` values from a single S3 listing page.
+///
+/// Compared with `build_tree_entries_from_flat_listing`, this variant drops the
+/// leading ancestor entries that were already emitted by a previous listing
+/// page, so callers can stitch paginated S3 scans into one stable DFS stream.
+pub(super) fn build_tree_entries_from_listing_page<F>(
+ query_root: &str,
+ objects: &[client::ObjectMeta],
+ show_hidden: bool,
+ level_limit: Option<usize>,
+ prev_rel_parts: &[String],
+ key_to_path: F,
+) -> Result<Vec<TreeEntry>>
+where
+ F: Fn(&str) -> String,
+{
+ let mut entries =
+ build_tree_entries_from_flat_listing(query_root, objects, show_hidden, level_limit, key_to_path)?;
+ if prev_rel_parts.is_empty() || entries.is_empty() {
+ return Ok(entries);
+ }
+
+ let mut skip = 0usize;
+ while skip < entries.len() {
+ let parts = rel_parts(&entries[skip].rel_path);
+ if parts.len() <= prev_rel_parts.len() && prev_rel_parts[..parts.len()] == parts[..] {
+ skip += 1;
+ continue;
+ }
+ break;
+ }
+
+ if skip > 0 {
+ entries.drain(..skip);
+ }
+ Ok(entries)
+}
+
+pub(super) fn rel_parts(rel_path: &str) -> Vec<String> {
+ rel_path
+ .split('/')
+ .filter(|segment| !segment.is_empty() && *segment != ".")
+ .map(ToString::to_string)
+ .collect()
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -335,4 +381,57 @@ mod tests {
// (`/tenant_x`, `/tenant_x/resources`) and not the query root itself.
assert_eq!(paths, vec!["/tenant_x/resources/res_1/res_1.md"]);
}
+
+ #[test]
+ fn test_build_tree_entries_page_drops_cross_page_duplicate_ancestors() {
+ let first_page = vec![make_meta("a/b/first.txt", 100, false)];
+ let first = build_tree_entries_from_listing_page(
+ "/",
+ &first_page,
+ false,
+ None,
+ &[],
+ identity_key_to_path,
+ )
+ .unwrap();
+ let last_rel_parts = first
+ .last()
+ .unwrap()
+ .rel_path
+ .split('/')
+ .map(str::to_string)
+ .collect::<Vec<_>>();
+
+ let second_page = vec![make_meta("a/b/second.txt", 100, false)];
+ let second = build_tree_entries_from_listing_page(
+ "/",
+ &second_page,
+ false,
+ None,
+ &last_rel_parts,
+ identity_key_to_path,
+ )
+ .unwrap();
+
+ let paths: Vec<&str> = second.iter().map(|e| e.path.as_str()).collect();
+ assert_eq!(paths, vec!["/a/b/second.txt"]);
+ }
+
+ #[test]
+ fn test_build_tree_entries_page_keeps_diverging_suffix_after_common_prefix() {
+ let prev_rel_parts = vec!["a".to_string(), "b".to_string(), "first.txt".to_string()];
+ let page = vec![make_meta("a/c/second.txt", 100, false)];
+ let result = build_tree_entries_from_listing_page(
+ "/",
+ &page,
+ false,
+ None,
+ &prev_rel_parts,
+ identity_key_to_path,
+ )
+ .unwrap();
+
+ let paths: Vec<&str> = result.iter().map(|e| e.path.as_str()).collect();
+ assert_eq!(paths, vec!["/a/c", "/a/c/second.txt"]);
+ }
}
diff --git a/docs/en/api/03-filesystem.md b/docs/en/api/03-filesystem.md
index 01a1cbb1f..9289cbebc 100644
--- a/docs/en/api/03-filesystem.md
+++ b/docs/en/api/03-filesystem.md
@@ -913,7 +913,7 @@ Search content by pattern.
| pattern | str | Yes | - | Search pattern (regex) |
| case_insensitive | bool | No | False | Ignore case |
| exclude_uri | str | No | None | URI prefix to exclude from search |
-| node_limit | int | No | None | Maximum number of results |
+| node_limit | int | No | 256 | Maximum number of results. Omitted requests default to 256; pass a larger integer when you need more results |
| level_limit | int | No | Python SDK: 5; HTTP API / CLI / Go SDK: 10 | Maximum directory depth to traverse. The Go SDK currently uses the HTTP API default. |
**Python SDK (Embedded / HTTP)**
@@ -1016,7 +1016,9 @@ print(f"Found {results['count']} Python files")
**Go SDK**
```go
-result, err := client.Glob(ctx, "**/*.md", "viking://resources/")
+result, err := client.Glob(ctx, "**/*.md", "viking://resources/", &openviking.GlobOptions{
+ NodeLimit: openviking.Int(1024),
+})
if err != nil {
return err
}
diff --git a/docs/en/api/06-retrieval.md b/docs/en/api/06-retrieval.md
index b159f3a8a..8e7c8219b 100644
--- a/docs/en/api/06-retrieval.md
+++ b/docs/en/api/06-retrieval.md
@@ -635,7 +635,7 @@ The `glob()` method uses file wildcard pattern matching URIs, similar to Unix sh
- `[]` matches character range
**Code Entry Points**:
-- `openviking_cli/client/sync_http.py:SyncHTTPClient.glob()` - Python SDK entry (HTTP)
+- `sdk/python/openviking_sdk/client.py:SyncHTTPClient.glob()` - Python SDK entry (HTTP)
- `openviking/server/routers/search.py:glob()` - HTTP router
- `crates/ov_cli/src/commands/search.rs:glob()` - Rust CLI command
@@ -647,7 +647,7 @@ The `glob()` method uses file wildcard pattern matching URIs, similar to Unix sh
|-----------|------|----------|---------|-------------|
| pattern | str | Yes | - | Glob pattern (e.g., `**/*.md`) |
| uri | str | No | "viking://" | Starting URI |
-| node_limit | int | No | None | Maximum number of matches to return |
+| node_limit | int | No | 256 | Maximum number of matches to return. Omitted requests default to 256; pass a larger integer when you need more results |
#### 3. Usage Examples
@@ -675,21 +675,23 @@ import openviking as ov
client = ov.SyncHTTPClient(url="http://localhost:1933", api_key="your-key")
client.initialize()
-# Find all markdown files
+# Find all markdown files (defaults to returning at most 256 matches)
results = client.glob("**/*.md", "viking://resources")
print(f"Found {results['count']} markdown files:")
for uri in results['matches']:
print(f" {uri}")
-# Find all Python files
-results = client.glob("**/*.py", "viking://resources")
+# Find all Python files with a higher explicit cap
+results = client.glob("**/*.py", "viking://resources", node_limit=1024)
print(f"Found {results['count']} Python files")
```
**Go SDK**
```go
-result, err := client.Glob(ctx, "**/*.md", "viking://resources")
+result, err := client.Glob(ctx, "**/*.md", "viking://resources", &openviking.GlobOptions{
+ NodeLimit: openviking.Int(1024),
+})
if err != nil {
return err
}
diff --git a/docs/zh/api/03-filesystem.md b/docs/zh/api/03-filesystem.md
index 8504d77f7..092be2f40 100644
--- a/docs/zh/api/03-filesystem.md
+++ b/docs/zh/api/03-filesystem.md
@@ -998,7 +998,7 @@ openviking grep "authentication" --uri viking://resources/ [--ignore-case]
|------|------|------|--------|------|
| pattern | str | 是 | - | Glob 模式(例如 `**/*.md`) |
| uri | str | 否 | "viking://" | 起始 URI |
-| node_limit | int | 否 | None | 最大返回匹配数 |
+| node_limit | int | 否 | 256 | 最大返回匹配数。省略时默认使用 256;如需更多结果,请显式传入更大的整数 |
**Python SDK (Embedded / HTTP)**
@@ -1017,7 +1017,9 @@ print(f"Found {results['count']} Python files")
**Go SDK**
```go
-result, err := client.Glob(ctx, "**/*.md", "viking://resources/")
+result, err := client.Glob(ctx, "**/*.md", "viking://resources/", &openviking.GlobOptions{
+ NodeLimit: openviking.Int(1024),
+})
if err != nil {
return err
}
diff --git a/docs/zh/api/06-retrieval.md b/docs/zh/api/06-retrieval.md
index 1146473da..a93ac0494 100644
--- a/docs/zh/api/06-retrieval.md
+++ b/docs/zh/api/06-retrieval.md
@@ -637,7 +637,7 @@ openviking grep "TODO" --uri viking://resources --level-limit 3
- `[]` 匹配字符范围
**代码入口**:
-- `openviking_cli/client/sync_http.py:SyncHTTPClient.glob()` - Python SDK 入口(HTTP)
+- `sdk/python/openviking_sdk/client.py:SyncHTTPClient.glob()` - Python SDK 入口(HTTP)
- `openviking/server/routers/search.py:glob()` - HTTP 路由
- `crates/ov_cli/src/commands/search.rs:glob()` - Rust CLI 命令
@@ -649,7 +649,7 @@ openviking grep "TODO" --uri viking://resources --level-limit 3
|------|------|------|--------|------|
| pattern | str | 是 | - | Glob 模式(例如 `**/*.md`)|
| uri | str | 否 | "viking://" | 起始 URI |
-| node_limit | int | 否 | None | 最大返回匹配数 |
+| node_limit | int | 否 | 256 | 最大返回匹配数。省略时默认使用 256;如需更多结果,请显式传入更大的整数 |
#### 3. 使用示例
@@ -677,21 +677,23 @@ import openviking as ov
client = ov.SyncHTTPClient(url="http://localhost:1933", api_key="your-key")
client.initialize()
-# 查找所有 markdown 文件
+# 查找所有 markdown 文件(默认最多返回 256 条)
results = client.glob("**/*.md", "viking://resources")
print(f"Found {results['count']} markdown files:")
for uri in results['matches']:
print(f" {uri}")
-# 查找所有 Python 文件
-results = client.glob("**/*.py", "viking://resources")
+# 查找所有 Python 文件,并显式放宽返回上限
+results = client.glob("**/*.py", "viking://resources", node_limit=1024)
print(f"Found {results['count']} Python files")
```
**Go SDK**
```go
-result, err := client.Glob(ctx, "**/*.md", "viking://resources")
+result, err := client.Glob(ctx, "**/*.md", "viking://resources", &openviking.GlobOptions{
+ NodeLimit: openviking.Int(1024),
+})
if err != nil {
return err
}
diff --git a/openviking/parse/image_rewrite.py b/openviking/parse/image_rewrite.py
index 7652d1d9f..c971529e1 100644
--- a/openviking/parse/image_rewrite.py
+++ b/openviking/parse/image_rewrite.py
@@ -227,7 +227,7 @@ async def rewrite_image_uris(
root_prefix = root_uri.rstrip("/")
# Find all .md files recursively
- glob_result = await viking_fs.glob("*.md", uri=root_uri, ctx=ctx)
+ glob_result = await viking_fs.glob("**/*.md", uri=root_uri, ctx=ctx)
md_uris = glob_result.get("matches", [])
if not md_uris:
diff --git a/openviking/parse/parsers/markdown.py b/openviking/parse/parsers/markdown.py
index 02f929eae..1a2b636db 100644
--- a/openviking/parse/parsers/markdown.py
+++ b/openviking/parse/parsers/markdown.py
@@ -567,7 +567,7 @@ class MarkdownParser(BaseParser):
viking_fs = self._get_viking_fs()
# Find all processed markdown files under the root directory
- glob_result = await viking_fs.glob("*.md", uri=root_dir)
+ glob_result = await viking_fs.glob("**/*.md", uri=root_dir)
md_uris = glob_result.get("matches", [])
if not md_uris:
return
diff --git a/openviking/pyagfs/async_client.py b/openviking/pyagfs/async_client.py
index 09e57d43f..931479951 100644
--- a/openviking/pyagfs/async_client.py
+++ b/openviking/pyagfs/async_client.py
@@ -224,6 +224,28 @@ class AsyncAGFSClient:
ctx=_fs_ctx_or_default(path, fs_ctx),
)
+ async def glob_directory(
+ self,
+ path: str,
+ pattern: str,
+ show_hidden: bool = False,
+ page_size: int | None = None,
+ level_limit: int | None = None,
+ continuation_token: str | None = None,
+ *,
+ fs_ctx: Dict[str, str] | None = None,
+ ) -> Dict[str, Any]:
+ return await self.run(
+ "glob_directory",
+ path,
+ pattern,
+ show_hidden=show_hidden,
+ page_size=page_size,
+ level_limit=level_limit,
+ continuation_token=continuation_token,
+ ctx=_fs_ctx_or_default(path, fs_ctx),
+ )
+
async def system_sync_status(
self, path: str, *, fs_ctx: Dict[str, str] | None = None
) -> Dict[str, Any]:
diff --git a/openviking/pyagfs/protocols.py b/openviking/pyagfs/protocols.py
index b6b6c19c8..d04fe6680 100644
--- a/openviking/pyagfs/protocols.py
+++ b/openviking/pyagfs/protocols.py
@@ -93,6 +93,17 @@ class AGFSSyncClientProtocol(Protocol):
) -> list[Dict[str, Any]]:
"""Return a tree view for the given AGFS directory."""
+ def glob_directory(
+ self,
+ path: str,
+ pattern: str,
+ show_hidden: bool = False,
+ page_size: int | None = None,
+ level_limit: int | None = None,
+ continuation_token: str | None = None,
+ ) -> Dict[str, Any]:
+ """Return one page of flat glob matches for the given AGFS directory."""
+
def system_sync_status(self, path: str) -> Dict[str, Any]:
"""Return multi-write sync status for a file or directory path."""
diff --git a/openviking/server/routers/search.py b/openviking/server/routers/search.py
index 6e46f7b06..ec278446d 100644
--- a/openviking/server/routers/search.py
+++ b/openviking/server/routers/search.py
@@ -149,7 +149,7 @@ class GlobRequest(BaseModel):
pattern: str
uri: str = "viking://"
- node_limit: Optional[int] = None
+ node_limit: Optional[int] = 256
@router.post("/find")
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index 8ff448a72..257b4d86f 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -971,7 +971,7 @@ class SemanticProcessor(DequeueHandlerBase):
if root_prefix != target_prefix:
try:
- glob_result = await viking_fs.glob("*.md", uri=target_prefix, ctx=ctx)
+ glob_result = await viking_fs.glob("**/*.md", uri=target_prefix, ctx=ctx)
target_md_uris = glob_result.get("matches", [])
except Exception:
target_md_uris = []
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 983a19b78..701d11de2 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -22,7 +22,6 @@ import time
from contextlib import contextmanager
from dataclasses import dataclass, field
from datetime import datetime, timezone
-from pathlib import PurePath
from typing import TYPE_CHECKING, Any, Awaitable, Callable, Dict, List, Optional, TypeVar, Union
from openviking.core.context import ContextLevel
@@ -1555,15 +1554,60 @@ class VikingFS:
ctx: Optional[RequestContext] = None,
) -> Dict:
"""File pattern matching, supports **/*.md recursive."""
- entries = await self.tree(uri, node_limit=1000000, level_limit=None, ctx=ctx)
+ _ensure_non_empty_search_query(pattern)
+ self._ensure_access(uri, ctx)
+ real_ctx = self._ctx_or_default(ctx)
+ primary_path = self._uri_to_path(uri, ctx=ctx)
+ path: Optional[str] = None
+ for candidate_path in self._read_paths(uri, ctx=ctx):
+ if not await self._read_path_visible(uri, candidate_path, primary_path, real_ctx):
+ continue
+ if await self._agfs_path_exists(candidate_path):
+ path = candidate_path
+ break
+ if path is None:
+ if self._is_legacy_session_root_uri(uri):
+ return {"matches": [], "count": 0}
+ raise NotFoundError(uri, "directory")
+
+ page_size = self._glob_page_size(node_limit)
+ continuation_token: Optional[str] = None
matches = []
- for entry in entries:
- rel_path = entry.get("rel_path", "")
- if PurePath(rel_path).match(pattern):
- matches.append(entry["uri"])
- # Now apply node limit to the filtered matches
- if node_limit is not None and node_limit > 0:
- matches = matches[:node_limit]
+ while True:
+ page = await self._async_agfs.glob_directory(
+ path,
+ pattern,
+ show_hidden=False,
+ page_size=page_size,
+ level_limit=None,
+ continuation_token=continuation_token,
+ )
+
+ for entry in page.get("entries", []):
+ if node_limit is not None and node_limit > 0 and len(matches) >= node_limit:
+ return {"matches": matches, "count": len(matches)}
+ if not self._is_path_entry_visible(
+ entry["path"],
+ entry.get("name") or entry["path"].rsplit("/", 1)[-1],
+ path,
+ real_ctx,
+ ):
+ continue
+ if not await self._read_path_visible(uri, entry["path"], primary_path, real_ctx):
+ continue
+ entry_uri = self._alias_uri_for_path(
+ request_uri=uri,
+ base_path=path,
+ entry_path=entry["path"],
+ ctx=ctx,
+ )
+ matches.append(entry_uri)
+
+ if node_limit is not None and node_limit > 0 and len(matches) >= node_limit:
+ return {"matches": matches, "count": len(matches)}
+ continuation_token = page.get("next_token")
+ if not continuation_token:
+ break
return {"matches": matches, "count": len(matches)}
async def _batch_fetch_abstracts(
@@ -2226,25 +2270,15 @@ class VikingFS:
return True
return False
- def _is_tree_entry_visible(
- self, entry: Dict[str, Any], base_path: str, ctx: RequestContext
+ def _is_path_entry_visible(
+ self, entry_path: str, name: str, base_path: str, ctx: RequestContext
) -> bool:
- """Check visibility for a single TreeEntry returned by Rust tree_directory.
-
- Applies three layers of filtering:
- 1. Ancestor chain — if any ancestor directory would be filtered by _ls_entries,
- all descendants are invisible.
- 2. Self — the entry's own name must pass _ls_entries at its parent level.
- 3. ACL — the entry must be accessible by the requesting context.
- """
- entry_path = entry["path"]
-
+ """Check visibility for one flattened path entry returned by Rust."""
if self._ancestor_is_filtered(entry_path, base_path):
return False
entry_parts = [p for p in entry_path.strip("/").split("/") if p]
if entry_parts:
- name = entry_parts[-1]
parent_parts = entry_parts[:-1]
parent_path = "/" + "/".join(parent_parts) if parent_parts else "/"
if not self._is_name_visible_at_path(name, parent_path):
@@ -2256,10 +2290,26 @@ class VikingFS:
return True
+ def _is_tree_entry_visible(
+ self, entry: Dict[str, Any], base_path: str, ctx: RequestContext
+ ) -> bool:
+ """Check visibility for a single TreeEntry returned by Rust tree_directory."""
+ entry_path = entry["path"]
+ entry_info = entry.get("info", {})
+ name = entry_info.get("name") or entry_path.rstrip("/").rsplit("/", 1)[-1]
+ return self._is_path_entry_visible(entry_path, name, base_path, ctx)
+
# Over-fetch multiplier for bounded tree traversal. When a node_limit is
# set, we push down node_limit * this factor as the raw-node limit to Rust,
# leaving headroom for ACL/internal-name filtering before re-fetching.
_TREE_OVERFETCH_FACTOR = 4
+ _GLOB_PAGE_SIZE_DEFAULT = 1024
+
+ def _glob_page_size(self, node_limit: Optional[int]) -> int:
+ """Return the backend page size used by glob_directory."""
+ if node_limit is None or node_limit <= 0:
+ return self._GLOB_PAGE_SIZE_DEFAULT
+ return node_limit
async def _iter_visible_tree_entries(
self,
@@ -2457,21 +2507,32 @@ class VikingFS:
entry_path: str,
ctx: Optional[RequestContext],
) -> str:
+ base = base_path.rstrip("/")
normalized_request, request_parts = self._normalized_uri_parts(request_uri)
- # Only legacy namespaces need alias remapping:
- # - Old format viking://agent/{agent_id}/...
- # - viking://session/...
- is_legacy_namespace = request_parts and (
- request_parts[0] == "session"
- or (request_parts[0] == "agent" and self._is_legacy_agent_id_uri(request_uri))
+ request_root = (
+ normalized_request
+ if normalized_request == "viking://"
+ else normalized_request.rstrip("/")
+ )
+ preserve_request_alias = normalized_request in {"viking://", "viking://user"} or bool(
+ request_parts
+ and (
+ request_parts[0] == "session"
+ or (request_parts[0] == "agent" and self._is_legacy_agent_id_uri(request_uri))
+ )
)
- if not is_legacy_namespace:
- return self._path_to_uri(entry_path, ctx=ctx)
- base = base_path.rstrip("/")
rel_path = entry_path[len(base) :].strip("/") if entry_path.startswith(base) else ""
- if not rel_path:
- return normalized_request.rstrip("/")
- return f"{normalized_request.rstrip('/')}/{rel_path}"
+ if entry_path.startswith(base):
+ separator = "" if request_root.endswith("://") else "/"
+ candidate_uri = request_root if not rel_path else f"{request_root}{separator}{rel_path}"
+ if preserve_request_alias:
+ return candidate_uri
+ try:
+ if self._uri_to_path(candidate_uri, ctx=ctx) == entry_path:
+ return candidate_uri
+ except Exception:
+ pass
+ return self._path_to_uri(entry_path, ctx=ctx)
async def _agfs_path_exists(self, path: str) -> bool:
try:
diff --git a/sdk/go/retrieval.go b/sdk/go/retrieval.go
index bc6bd5b69..cdded2cc4 100644
--- a/sdk/go/retrieval.go
+++ b/sdk/go/retrieval.go
@@ -97,14 +97,21 @@ func (c *Client) Grep(ctx context.Context, uri, pattern string, opts *GrepOption
}
// Glob finds files by glob pattern.
-func (c *Client) Glob(ctx context.Context, pattern string, uri string) (map[string]any, error) {
+func (c *Client) Glob(ctx context.Context, pattern string, uri string, opts *GlobOptions) (map[string]any, error) {
+ if opts == nil {
+ opts = &GlobOptions{}
+ }
if uri == "" {
uri = "viking://"
}
- var result map[string]any
- err := c.doJSON(ctx, http.MethodPost, "/api/v1/search/glob", nil, map[string]any{
+ payload := map[string]any{
"pattern": pattern,
"uri": NormalizeURI(uri),
- }, &result)
+ }
+ if opts.NodeLimit != nil {
+ payload["node_limit"] = *opts.NodeLimit
+ }
+ var result map[string]any
+ err := c.doJSON(ctx, http.MethodPost, "/api/v1/search/glob", nil, payload, &result)
return result, err
}
diff --git a/sdk/go/types.go b/sdk/go/types.go
index 2bf5cd0de..d6d078ce4 100644
--- a/sdk/go/types.go
+++ b/sdk/go/types.go
@@ -225,6 +225,11 @@ type GrepOptions struct {
ExcludeURI string
}
+// GlobOptions controls Glob.
+type GlobOptions struct {
+ NodeLimit *int
+}
+
// CreateSessionOptions controls CreateSession.
type CreateSessionOptions struct {
SessionID string
diff --git a/sdk/python/openviking_sdk/client.py b/sdk/python/openviking_sdk/client.py
index e13612631..ff0b959e1 100644
--- a/sdk/python/openviking_sdk/client.py
+++ b/sdk/python/openviking_sdk/client.py
@@ -953,10 +953,19 @@ class AsyncHTTPClient:
response = await self._http.post("/api/v1/search/grep", json=request_json)
return self._handle_response(response)
- async def glob(self, pattern: str, uri: str = "viking://") -> Dict[str, Any]:
+ async def glob(
+ self,
+ pattern: str,
+ uri: str = "viking://",
+ node_limit: int = 256,
+ ) -> Dict[str, Any]:
response = await self._http.post(
"/api/v1/search/glob",
- json={"pattern": pattern, "uri": VikingURI.normalize(uri)},
+ json={
+ "pattern": pattern,
+ "uri": VikingURI.normalize(uri),
+ "node_limit": node_limit,
+ },
)
return self._handle_response(response)
@@ -1617,9 +1626,7 @@ class SyncHTTPClient:
skill_name: str,
target_uri: Optional[str] = None,
) -> Dict[str, Any]:
- return run_async(
- self._async_client.delete_skill(skill_name, target_uri=target_uri)
- )
+ return run_async(self._async_client.delete_skill(skill_name, target_uri=target_uri))
def list_watches(
self,
@@ -1859,8 +1866,13 @@ class SyncHTTPClient:
)
)
- def glob(self, pattern: str, uri: str = "viking://") -> Dict[str, Any]:
- return run_async(self._async_client.glob(pattern, uri=uri))
+ def glob(
+ self,
+ pattern: str,
+ uri: str = "viking://",
+ node_limit: int = 256,
+ ) -> Dict[str, Any]:
+ return run_async(self._async_client.glob(pattern, uri=uri, node_limit=node_limit))
def relations(self, uri: str) -> List[Any]:
return run_async(self._async_client.relations(uri))
diff --git a/sdk/python/tests/test_async_client_behaviors.py b/sdk/python/tests/test_async_client_behaviors.py
index 9f2d40a84..659f5841f 100644
--- a/sdk/python/tests/test_async_client_behaviors.py
+++ b/sdk/python/tests/test_async_client_behaviors.py
@@ -534,11 +534,11 @@ async def test_glob_normalizes_scope_uri():
"matches": ["viking://resources/demo.md"],
}
- await client.glob("*.md", uri="/resources/")
+ await client.glob("**/*.md", uri="/resources/")
fake_http.post.assert_awaited_once_with(
"/api/v1/search/glob",
- json={"pattern": "*.md", "uri": "viking://resources/"},
+ json={"pattern": "**/*.md", "uri": "viking://resources/"},
)
diff --git a/tests/api_test/resources/slow/test_resource_e2e.py b/tests/api_test/resources/slow/test_resource_e2e.py
index f90faa27d..c8c47e5fe 100644
--- a/tests/api_test/resources/slow/test_resource_e2e.py
+++ b/tests/api_test/resources/slow/test_resource_e2e.py
@@ -136,7 +136,7 @@ class TestResourceEndToEnd:
return
root_uri = add_resp.json().get("result", {}).get("root_uri", "")
- glob_resp = api_client.glob(pattern="*.md", uri=root_uri)
+ glob_resp = api_client.glob(pattern="**/*.md", uri=root_uri)
if glob_resp.status_code == 200:
matches = glob_resp.json().get("result", {}).get("matches", [])
assert len(matches) >= 1, (
diff --git a/tests/api_test/retrieval/slow/test_search_advanced.py b/tests/api_test/retrieval/slow/test_search_advanced.py
index 8240623bd..0e754c6f6 100644
--- a/tests/api_test/retrieval/slow/test_search_advanced.py
+++ b/tests/api_test/retrieval/slow/test_search_advanced.py
@@ -98,14 +98,14 @@ class TestSearchAdvanced:
assert data.get("status") == "ok"
def test_glob_with_extension_pattern(self, api_client):
- glob_resp = api_client.glob(pattern="*.md", uri="viking://resources/")
+ glob_resp = api_client.glob(pattern="**/*.md", uri="viking://resources/")
assert glob_resp.status_code == 200, (
- f"glob *.md should return 200, got {glob_resp.status_code}"
+ f"glob **/*.md should return 200, got {glob_resp.status_code}"
)
matches = glob_resp.json().get("result", {}).get("matches", [])
for match in matches:
uri = match if isinstance(match, str) else match.get("uri", "")
- assert ".md" in uri, f"glob *.md should only match .md files, got {uri}"
+ assert ".md" in uri, f"glob **/*.md should only match .md files, got {uri}"
def test_glob_with_double_star_pattern(self, api_client):
glob_resp = api_client.glob(pattern="**/*.md", uri="viking://resources/")
diff --git a/tests/api_test/retrieval/slow/test_search_result_quality.py b/tests/api_test/retrieval/slow/test_search_result_quality.py
index af3cb8a38..43a51168e 100644
--- a/tests/api_test/retrieval/slow/test_search_result_quality.py
+++ b/tests/api_test/retrieval/slow/test_search_result_quality.py
@@ -214,7 +214,7 @@ class TestSearchResultQuality:
assert match["line"] >= 1, f"line number should be >= 1, got {match['line']}"
def test_glob_result_structure(self, api_client):
- glob_resp = api_client.glob(pattern="*", uri="viking://resources/")
+ glob_resp = api_client.glob(pattern="**/*", uri="viking://resources/")
assert glob_resp.status_code == 200, f"glob should return 200, got {glob_resp.status_code}"
data = glob_resp.json()
assert data.get("status") == "ok"
@@ -242,7 +242,7 @@ class TestSearchResultQuality:
if add_resp.status_code != 200:
return
- glob_resp = api_client.glob(pattern=f"*{unique_name}*", uri="viking://resources/")
+ glob_resp = api_client.glob(pattern=f"**/*{unique_name}*", uri="viking://resources/")
if glob_resp.status_code == 200:
matches = glob_resp.json().get("result", {}).get("matches", [])
assert len(matches) >= 1, (
@@ -277,7 +277,7 @@ class TestSearchResultQuality:
)
def test_glob_nonexistent_uri(self, api_client):
- glob_resp = api_client.glob(pattern="*", uri="viking://resources/nonexistent_glob_uri")
+ glob_resp = api_client.glob(pattern="**/*", uri="viking://resources/nonexistent_glob_uri")
assert glob_resp.status_code == 404, (
f"glob on nonexistent URI should return 200/404/500, got {glob_resp.status_code}"
)
diff --git a/tests/cli/test_cli_search.py b/tests/cli/test_cli_search.py
index c181f55d4..0713614d1 100644
--- a/tests/cli/test_cli_search.py
+++ b/tests/cli/test_cli_search.py
@@ -2,6 +2,8 @@
# SPDX-License-Identifier: AGPL-3.0
"""CLI search operation tests (find, search, grep, glob)."""
+import time
+
import pytest
from conftest import ov, skip_if_auth_error
@@ -115,7 +117,7 @@ class TestSearchGlob:
def test_glob_with_uri(self, test_dir_uri):
r = None
for _attempt in range(5):
- r = ov(["glob", "*.md", "-u", test_dir_uri, "-o", "json"])
+ r = ov(["glob", "**/*.md", "-u", test_dir_uri, "-o", "json"])
if r["exit_code"] == 0:
break
time.sleep(10)
diff --git a/tests/client/test_file_operations.py b/tests/client/test_file_operations.py
index d292911ea..2df2a910c 100644
--- a/tests/client/test_file_operations.py
+++ b/tests/client/test_file_operations.py
@@ -117,7 +117,7 @@ class TestGlob:
client, uri = client_with_resource
parent_uri = "/".join(uri.split("/")[:-1]) + "/"
- result = await client.glob(pattern="*.md", uri=parent_uri)
+ result = await client.glob(pattern="**/*.md", uri=parent_uri)
assert isinstance(result, dict)
assert "matches" in result and result["count"] > 0
diff --git a/tests/parse/test_markdown_link_rewrite.py b/tests/parse/test_markdown_link_rewrite.py
index 9270b4193..2f99ba884 100644
--- a/tests/parse/test_markdown_link_rewrite.py
+++ b/tests/parse/test_markdown_link_rewrite.py
@@ -245,8 +245,8 @@ class FakeVikingFS:
async def glob(self, pattern, uri="", **kw):
# Mirror VikingFS.glob enough for _ingest_local_images, which only ever asks
- # for "*.md" under a root: list stored files matching the pattern's suffix.
- suffix = pattern.lstrip("*")
+ # for recursive Markdown files under a root: list files matching the suffix.
+ suffix = pattern.rsplit("*", 1)[-1]
prefix = uri.rstrip("/") + "/"
matches = [u for u in self.files if u.startswith(prefix) and u.endswith(suffix)]
return {"matches": matches}
diff --git a/tests/server/test_api_search.py b/tests/server/test_api_search.py
index 87ef3dee0..7506e9f97 100644
--- a/tests/server/test_api_search.py
+++ b/tests/server/test_api_search.py
@@ -1007,7 +1007,7 @@ async def test_glob(client_with_resource):
client, _ = client_with_resource
resp = await client.post(
"/api/v1/search/glob",
- json={"pattern": "*.md"},
+ json={"pattern": "**/*.md"},
)
assert resp.status_code == 200
assert resp.json()["status"] == "ok"
diff --git a/tests/server/test_mcp_endpoint.py b/tests/server/test_mcp_endpoint.py
index da0984491..702621b0c 100644
--- a/tests/server/test_mcp_endpoint.py
+++ b/tests/server/test_mcp_endpoint.py
@@ -667,7 +667,7 @@ async def test_grep_case_insensitive(service):
async def test_glob_no_matches(service):
- result = await glob(pattern="zzz_nonexistent_*.xyz")
+ result = await glob(pattern="**/zzz_nonexistent_*.xyz")
assert "No files found" in result
@@ -678,7 +678,7 @@ async def test_glob_match_all_md(service, client_with_resource):
async def test_glob_with_uri_scope(service):
- result = await glob(pattern="*", uri="viking://resources")
+ result = await glob(pattern="**/*", uri="viking://resources")
assert isinstance(result, str)
diff --git a/tests/storage/test_viking_fs_glob.py b/tests/storage/test_viking_fs_glob.py
new file mode 100644
index 000000000..754ff5a73
--- /dev/null
+++ b/tests/storage/test_viking_fs_glob.py
@@ -0,0 +1,334 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import pytest
+
+from openviking.server.identity import RequestContext, Role
+from openviking.storage.viking_fs import VikingFS
+from openviking_cli.exceptions import InvalidArgumentError
+from openviking_cli.session.user_id import UserIdentifier
+
+
+class _DummyAgfs:
+ pass
+
+
+def _default_ctx() -> RequestContext:
+ return RequestContext(user=UserIdentifier.the_default_user(), role=Role.ROOT)
+
+
+def fs(monkeypatch):
+ viking_fs = VikingFS(agfs=_DummyAgfs())
+ monkeypatch.setattr(viking_fs, "_ctx_or_default", lambda _ctx=None: _default_ctx())
+ monkeypatch.setattr(
+ viking_fs, "_uri_to_path", lambda _uri, **_kwargs: "/local/test_account/resources"
+ )
+ monkeypatch.setattr(
+ viking_fs, "_read_paths", lambda _uri, **_kwargs: ["/local/test_account/resources"]
+ )
+ monkeypatch.setattr(viking_fs, "_agfs_path_exists", lambda _path: _async_true())
+ monkeypatch.setattr(viking_fs, "_read_path_visible", lambda *_args, **_kwargs: _async_true())
+ monkeypatch.setattr(
+ viking_fs,
+ "_path_to_uri",
+ lambda path, **_kwargs: path.replace("/local/test_account/", "viking://"),
+ )
+ monkeypatch.setattr(viking_fs, "_is_accessible", lambda _uri, _ctx: True)
+ return viking_fs
+
+
+async def _async_true():
+ return True
+
+
+async def test_glob_delegates_to_agfs_with_paging_and_visibility(monkeypatch, fs):
+ calls = []
+
+ pages = [
+ {
+ "entries": [
+ {
+ "path": "/local/test_account/resources/group/a.md",
+ "rel_path": "group/a.md",
+ "name": "a.md",
+ "is_dir": False,
+ },
+ {
+ "path": "/local/test_account/resources/_system/secret.md",
+ "rel_path": "_system/secret.md",
+ "name": "secret.md",
+ "is_dir": False,
+ },
+ ],
+ "next_token": "tok-1",
+ },
+ {
+ "entries": [
+ {
+ "path": "/local/test_account/resources/group/b.md",
+ "rel_path": "group/b.md",
+ "name": "b.md",
+ "is_dir": False,
+ }
+ ],
+ "next_token": None,
+ },
+ ]
+
+ async def fake_glob_directory(path, pattern, **kwargs):
+ calls.append({"path": path, "pattern": pattern, **kwargs})
+ return pages[len(calls) - 1]
+
+ monkeypatch.setattr(fs._async_agfs, "glob_directory", fake_glob_directory)
+
+ result = await fs.glob("**/*.md", uri="viking://resources", node_limit=2, ctx=_default_ctx())
+
+ assert result == {
+ "matches": [
+ "viking://resources/group/a.md",
+ "viking://resources/group/b.md",
+ ],
+ "count": 2,
+ }
+ assert [call["continuation_token"] for call in calls] == [None, "tok-1"]
+ assert all(call["page_size"] == 2 for call in calls)
+
+
+async def test_glob_stops_after_reaching_limit_at_page_end(monkeypatch, fs):
+ calls = []
+
+ async def fake_glob_directory(path, pattern, **kwargs):
+ calls.append({"path": path, "pattern": pattern, **kwargs})
+ return {
+ "entries": [
+ {
+ "path": "/local/test_account/resources/group/a.md",
+ "rel_path": "group/a.md",
+ "name": "a.md",
+ "is_dir": False,
+ },
+ {
+ "path": "/local/test_account/resources/group/b.md",
+ "rel_path": "group/b.md",
+ "name": "b.md",
+ "is_dir": False,
+ },
+ ],
+ "next_token": "tok-should-not-be-used",
+ }
+
+ monkeypatch.setattr(fs._async_agfs, "glob_directory", fake_glob_directory)
+
+ result = await fs.glob("**/*.md", uri="viking://resources", node_limit=2, ctx=_default_ctx())
+
+ assert result == {
+ "matches": [
+ "viking://resources/group/a.md",
+ "viking://resources/group/b.md",
+ ],
+ "count": 2,
+ }
+ assert len(calls) == 1
+
+
+async def test_glob_trusts_backend_glob_matches(monkeypatch, fs):
+ async def fake_glob_directory(path, pattern, **kwargs):
+ return {
+ "entries": [
+ {
+ "path": "/local/test_account/resources/a.md",
+ "rel_path": "a.md",
+ "name": "a.md",
+ "is_dir": False,
+ }
+ ],
+ "next_token": None,
+ }
+
+ monkeypatch.setattr(fs._async_agfs, "glob_directory", fake_glob_directory)
+
+ result = await fs.glob("**/*.md", uri="viking://resources", ctx=_default_ctx())
+
+ assert result == {"matches": ["viking://resources/a.md"], "count": 1}
+
+
+async def test_glob_rejects_empty_pattern(fs):
+ with pytest.raises(InvalidArgumentError):
+ await fs.glob("", uri="viking://resources", ctx=_default_ctx())
+
+
+async def test_glob_checks_access_before_listing(monkeypatch, fs):
+ called = False
+
+ def fake_ensure_access(uri, ctx):
+ nonlocal called
+ called = True
+ raise PermissionError(f"denied: {uri}")
+
+ monkeypatch.setattr(fs, "_ensure_access", fake_ensure_access)
+
+ with pytest.raises(PermissionError):
+ await fs.glob("**/*.md", uri="viking://resources", ctx=_default_ctx())
+
+ assert called is True
+
+
+async def test_glob_preserves_request_uri_alias(monkeypatch, fs):
+ monkeypatch.setattr(fs, "_uri_to_path", lambda _uri, **_kwargs: "/local/test_account/user")
+ monkeypatch.setattr(fs, "_read_paths", lambda _uri, **_kwargs: ["/local/test_account/user"])
+ monkeypatch.setattr(
+ fs, "_path_to_uri", lambda _path, **_kwargs: "viking://user/test_account/demo.md"
+ )
+
+ async def fake_glob_directory(path, pattern, **kwargs):
+ return {
+ "entries": [
+ {
+ "path": "/local/test_account/user/demo.md",
+ "rel_path": "demo.md",
+ "name": "demo.md",
+ "is_dir": False,
+ }
+ ],
+ "next_token": None,
+ }
+
+ monkeypatch.setattr(fs._async_agfs, "glob_directory", fake_glob_directory)
+
+ result = await fs.glob("**/*.md", uri="viking://user", ctx=_default_ctx())
+
+ assert result == {"matches": ["viking://user/demo.md"], "count": 1}
+
+
+async def test_glob_preserves_root_uri(monkeypatch, fs):
+ monkeypatch.setattr(fs, "_uri_to_path", lambda _uri, **_kwargs: "/local/test_account")
+ monkeypatch.setattr(fs, "_read_paths", lambda _uri, **_kwargs: ["/local/test_account"])
+ monkeypatch.setattr(
+ fs, "_path_to_uri", lambda _path, **_kwargs: "viking://resources/should-not-leak.md"
+ )
+
+ async def fake_glob_directory(path, pattern, **kwargs):
+ return {
+ "entries": [
+ {
+ "path": "/local/test_account/resources/demo.md",
+ "rel_path": "resources/demo.md",
+ "name": "demo.md",
+ "is_dir": False,
+ }
+ ],
+ "next_token": None,
+ }
+
+ monkeypatch.setattr(fs._async_agfs, "glob_directory", fake_glob_directory)
+
+ result = await fs.glob("**/*.md", uri="viking://", ctx=_default_ctx())
+
+ assert result == {"matches": ["viking://resources/demo.md"], "count": 1}
+
+
+async def test_glob_keeps_directory_matches(monkeypatch, fs):
+ async def fake_glob_directory(path, pattern, **kwargs):
+ return {
+ "entries": [
+ {
+ "path": "/local/test_account/resources/folder",
+ "rel_path": "folder",
+ "name": "folder",
+ "is_dir": True,
+ }
+ ],
+ "next_token": None,
+ }
+
+ monkeypatch.setattr(fs._async_agfs, "glob_directory", fake_glob_directory)
+
+ result = await fs.glob("**/*", uri="viking://resources", ctx=_default_ctx())
+
+ assert result == {"matches": ["viking://resources/folder"], "count": 1}
+
+
+async def test_glob_preserves_legacy_session_alias(monkeypatch, fs):
+ """中文注释:legacy session URI 仍应保持旧别名返回。"""
+ monkeypatch.setattr(
+ fs, "_uri_to_path", lambda _uri, **_kwargs: "/local/test_account/user/alice/sessions/sess_1"
+ )
+ monkeypatch.setattr(
+ fs,
+ "_read_paths",
+ lambda _uri, **_kwargs: ["/local/test_account/user/alice/sessions/sess_1"],
+ )
+
+ async def fake_glob_directory(path, pattern, **kwargs):
+ return {
+ "entries": [
+ {
+ "path": "/local/test_account/user/alice/sessions/sess_1/messages.jsonl",
+ "rel_path": "messages.jsonl",
+ "name": "messages.jsonl",
+ "is_dir": False,
+ }
+ ],
+ "next_token": None,
+ }
+
+ monkeypatch.setattr(fs._async_agfs, "glob_directory", fake_glob_directory)
+
+ result = await fs.glob("**/*.jsonl", uri="viking://session/alice/sess_1", ctx=_default_ctx())
+
+ assert result == {"matches": ["viking://session/alice/sess_1/messages.jsonl"], "count": 1}
+
+
+async def test_glob_uses_path_to_uri_for_non_legacy_namespace(monkeypatch, fs):
+ """中文注释:非 legacy 命名空间必须回落到 _path_to_uri,避免错误沿用请求别名。"""
+ monkeypatch.setattr(
+ fs,
+ "_uri_to_path",
+ lambda _uri, **_kwargs: "/local/test_account/resources/actual-root",
+ )
+ monkeypatch.setattr(
+ fs,
+ "_read_paths",
+ lambda _uri, **_kwargs: ["/local/test_account/resources/actual-root"],
+ )
+ monkeypatch.setattr(
+ fs,
+ "_path_to_uri",
+ lambda path, **_kwargs: path.replace(
+ "/local/test_account/resources/", "viking://resources/"
+ ),
+ )
+
+ async def fake_glob_directory(path, pattern, **kwargs):
+ return {
+ "entries": [
+ {
+ "path": "/local/test_account/resources/actual-root/demo.md",
+ "rel_path": "demo.md",
+ "name": "demo.md",
+ "is_dir": False,
+ }
+ ],
+ "next_token": None,
+ }
+
+ monkeypatch.setattr(fs._async_agfs, "glob_directory", fake_glob_directory)
+
+ result = await fs.glob(
+ "**/*.md",
+ uri="viking://resources/alias-root",
+ ctx=_default_ctx(),
+ )
+
+ assert result == {"matches": ["viking://resources/actual-root/demo.md"], "count": 1}