Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

perf(storage): optimize VikingFS grep implementation (#1731)

baojun-zhangc8ff9e357597
diff --git a/Cargo.lock b/Cargo.lock
index 050bafef1..f272aad47 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -729,6 +729,17 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "bstr"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab"
+dependencies = [
+ "memchr",
+ "regex-automata",
+ "serde",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.2"
@@ -1493,6 +1504,24 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+[[package]]
+name = "encoding_rs"
+version = "0.8.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "encoding_rs_io"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83"
+dependencies = [
+ "encoding_rs",
+]
+
[[package]]
name = "endian-type"
version = "0.1.2"
@@ -1785,6 +1814,98 @@ dependencies = [
"wasip3",
]
+[[package]]
+name = "globset"
+version = "0.4.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52dfc19153a48bde0cbd630453615c8151bce3a5adfac7a0aebfbf0a1e1f57e3"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "log",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "grep"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "309217bc53e2c691c314389c7fa91f9cd1a998cda19e25544ea47d94103880c3"
+dependencies = [
+ "grep-cli",
+ "grep-matcher",
+ "grep-printer",
+ "grep-regex",
+ "grep-searcher",
+]
+
+[[package]]
+name = "grep-cli"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf32d263c5d5cc2a23ce587097f5ddafdb188492ba2e6fb638eaccdc22453631"
+dependencies = [
+ "bstr",
+ "globset",
+ "libc",
+ "log",
+ "termcolor",
+ "winapi-util",
+]
+
+[[package]]
+name = "grep-matcher"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36d7b71093325ab22d780b40d7df3066ae4aebb518ba719d38c697a8228a8023"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "grep-printer"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd76035e87871f51c1ee5b793e32122b3ccf9c692662d9622ef1686ff5321acb"
+dependencies = [
+ "bstr",
+ "grep-matcher",
+ "grep-searcher",
+ "log",
+ "serde",
+ "serde_json",
+ "termcolor",
+]
+
+[[package]]
+name = "grep-regex"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ce0c256c3ad82bcc07b812c15a45ec1d398122e8e15124f96695234db7112ef"
+dependencies = [
+ "bstr",
+ "grep-matcher",
+ "log",
+ "regex-automata",
+ "regex-syntax",
+]
+
+[[package]]
+name = "grep-searcher"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac63295322dc48ebb20a25348147905d816318888e64f531bfc2a2bc0577dc34"
+dependencies = [
+ "bstr",
+ "encoding_rs",
+ "encoding_rs_io",
+ "grep-matcher",
+ "log",
+ "memchr",
+ "memmap2",
+]
+
[[package]]
name = "group"
version = "0.12.1"
@@ -2242,6 +2363,22 @@ dependencies = [
"icu_properties",
]
+[[package]]
+name = "ignore"
+version = "0.4.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3d782a365a015e0f5c04902246139249abf769125006fbe7649e2ee88169b4a"
+dependencies = [
+ "crossbeam-deque",
+ "globset",
+ "log",
+ "memchr",
+ "regex-automata",
+ "same-file",
+ "walkdir",
+ "winapi-util",
+]
+
[[package]]
name = "indexmap"
version = "2.13.0"
@@ -2565,6 +2702,15 @@ version = "2.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
+[[package]]
+name = "memmap2"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "memoffset"
version = "0.9.1"
@@ -3141,10 +3287,16 @@ dependencies = [
"chrono",
"clap",
"criterion",
+ "grep",
+ "grep-matcher",
+ "grep-regex",
+ "grep-searcher",
"hyper 1.8.1",
+ "ignore",
"lru",
"path-clean",
"radix_trie",
+ "regex",
"rusqlite",
"serde",
"serde_json",
@@ -4195,6 +4347,15 @@ dependencies = [
"windows-sys 0.61.2",
]
+[[package]]
+name = "termcolor"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755"
+dependencies = [
+ "winapi-util",
+]
+
[[package]]
name = "termimad"
version = "0.34.1"
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs
index bd239d576..9859de0be 100644
--- a/crates/ragfs-python/src/lib.rs
+++ b/crates/ragfs-python/src/lib.rs
@@ -11,7 +11,9 @@ use std::collections::HashMap;
use std::sync::Arc;
use std::time::UNIX_EPOCH;
-use ragfs::core::{ConfigValue, FileInfo, FileSystem, MountableFS, PluginConfig, WriteFlag};
+use ragfs::core::{
+ ConfigValue, FileInfo, FileSystem, GrepResult, MountableFS, PluginConfig, WriteFlag,
+};
#[cfg(feature = "s3")]
use ragfs::plugins::S3FSPlugin;
use ragfs::plugins::{
@@ -97,6 +99,26 @@ fn file_info_to_py_dict(py: Python<'_>, info: &FileInfo) -> PyResult<Py<PyDict>>
Ok(dict.into())
}
+/// Convert GrepResult to a Python dict matching the Go binding JSON format:
+/// {"matches": [{"file": str, "line": int, "content": str}, ...], "count": int}
+fn grep_result_to_py_dict(py: Python<'_>, result: &GrepResult) -> PyResult<Py<PyDict>> {
+ let dict = PyDict::new(py);
+
+ let matches_list = PyList::empty(py);
+ for m in &result.matches {
+ let match_dict = PyDict::new(py);
+ match_dict.set_item("file", &m.file)?;
+ match_dict.set_item("line", m.line)?;
+ match_dict.set_item("content", &m.content)?;
+ matches_list.append(match_dict)?;
+ }
+
+ dict.set_item("matches", matches_list)?;
+ dict.set_item("count", result.count)?;
+
+ Ok(dict.into())
+}
+
/// Format unix timestamp as RFC3339 string (simplified, UTC)
fn format_rfc3339(secs: u64) -> String {
let s = secs;
@@ -523,8 +545,21 @@ impl RAGFSBindingClient {
))
}
- /// Search for pattern in files (not yet implemented in ragfs).
- #[pyo3(signature = (path, pattern, recursive=false, case_insensitive=false, stream=false, node_limit=None))]
+ /// Search for pattern in files using regular expressions.
+ ///
+ /// Args:
+ /// path: File or directory path to search
+ /// pattern: Regular expression pattern to search for
+ /// recursive: Whether to search recursively in subdirectories (default: false)
+ /// case_insensitive: Whether to perform case-insensitive matching (default: false)
+ /// stream: Not supported in binding mode
+ /// node_limit: Maximum number of matches to return (default: None, no limit)
+ /// exclude_path: Optional path prefix to exclude from search (default: None)
+ /// level_limit: Optional maximum depth relative to query root (default: None)
+ ///
+ /// Returns:
+ /// A dict with "matches" (list of match dicts) and "count" (total matches)
+ #[pyo3(signature = (path, pattern, recursive=false, case_insensitive=false, stream=false, node_limit=None, exclude_path=None, level_limit=None))]
fn grep(
&self,
path: String,
@@ -533,18 +568,39 @@ impl RAGFSBindingClient {
case_insensitive: bool,
stream: bool,
node_limit: Option<i32>,
+ exclude_path: Option<String>,
+ level_limit: Option<i32>,
) -> PyResult<Py<PyAny>> {
- let _ = (
- path,
- pattern,
- recursive,
- case_insensitive,
- stream,
- node_limit,
- );
- Err(PyRuntimeError::new_err(
- "grep not yet implemented in ragfs-python",
- ))
+ if stream {
+ return Err(PyRuntimeError::new_err(
+ "Streaming not supported in binding mode",
+ ));
+ }
+
+ let fs = self.fs.clone();
+ let limit = node_limit.map(|n| if n < 0 { 0 } else { n as usize });
+ let level_limit_usize = level_limit.map(|n| if n < 0 { 0 } else { n as usize });
+
+ let result = self
+ .rt
+ .block_on(async move {
+ fs.grep(
+ &path,
+ &pattern,
+ recursive,
+ case_insensitive,
+ limit,
+ exclude_path.as_deref(),
+ level_limit_usize,
+ )
+ .await
+ })
+ .map_err(to_py_err)?;
+
+ Python::attach(|py| {
+ let dict = grep_result_to_py_dict(py, &result)?;
+ Ok(dict.into())
+ })
}
/// Calculate file digest (not yet implemented in ragfs).
diff --git a/crates/ragfs/Cargo.toml b/crates/ragfs/Cargo.toml
index aecc70454..8c117e70d 100644
--- a/crates/ragfs/Cargo.toml
+++ b/crates/ragfs/Cargo.toml
@@ -76,6 +76,16 @@ aws-types = { version = "1", optional = true }
# Cache
lru = "0.12"
+# Regular expressions for grep
+regex = "1.10"
+
+# Grep acceleration (localfs fallback, rg-like behavior)
+ignore = "0.4.25"
+grep = "0.4.1"
+grep-matcher = "0.1.8"
+grep-regex = "0.1.14"
+grep-searcher = "0.1.16"
+
# Development dependencies
[dev-dependencies]
tempfile = "3.12"
@@ -93,4 +103,4 @@ strip = true
codegen-units = 1
[profile.dev]
-opt-level = 0
+opt-level = 0
\ No newline at end of file
diff --git a/crates/ragfs/src/core/filesystem.rs b/crates/ragfs/src/core/filesystem.rs
index de79ab329..7dee204b6 100644
--- a/crates/ragfs/src/core/filesystem.rs
+++ b/crates/ragfs/src/core/filesystem.rs
@@ -5,9 +5,74 @@
//! different storage backends.
use async_trait::async_trait;
+use regex::Regex;
use super::errors::Result;
-use super::types::{FileInfo, WriteFlag};
+use super::types::{FileInfo, GrepResult, WriteFlag};
+
+/// Normalize a path for prefix comparisons.
+///
+/// - Keeps "/" as-is.
+/// - Strips trailing slashes for non-root paths (so "/a" and "/a/" behave the same).
+fn normalize_prefix_path(path: &str) -> String {
+ if path == "/" {
+ "/".to_string()
+ } else {
+ path.trim_end_matches('/').to_string()
+ }
+}
+
+/// Check whether `path` is under `exclude_path` (including itself).
+fn is_excluded_path(path: &str, exclude_path: &str) -> bool {
+ if exclude_path == "/" {
+ return true;
+ }
+
+ path == exclude_path
+ || path
+ .strip_prefix(exclude_path)
+ .is_some_and(|suffix| suffix.starts_with('/'))
+}
+
+/// Convert an absolute/plugin path to a query-root-relative grep match path.
+///
+/// Contract:
+/// - "." means query root itself.
+/// - otherwise returns a relative path without leading "/".
+fn relative_match_file(query_root: &str, path: &str) -> String {
+ let base = normalize_prefix_path(query_root);
+ if path == base {
+ return ".".to_string();
+ }
+
+ if base == "/" {
+ return path.trim_start_matches('/').to_string();
+ }
+
+ match path.strip_prefix(&base) {
+ Some(rest) => {
+ let rel = rest.trim_start_matches('/');
+ if rel.is_empty() {
+ ".".to_string()
+ } else {
+ rel.to_string()
+ }
+ }
+ None => path.trim_start_matches('/').to_string(),
+ }
+}
+
+/// Compute depth from a query-root-relative path.
+///
+/// - "." and "" => 0
+/// - "a/b" => 2
+fn relative_depth(rel: &str) -> usize {
+ if rel.is_empty() || rel == "." {
+ 0
+ } else {
+ rel.split('/').filter(|p| !p.is_empty()).count()
+ }
+}
/// Core filesystem abstraction trait
///
@@ -160,11 +225,183 @@ pub trait FileSystem: Send + Sync {
async fn exists(&self, path: &str) -> bool {
self.stat(path).await.is_ok()
}
+
+ /// Search for a pattern in files using regular expressions
+ ///
+ /// This is the default implementation that recursively searches files
+ /// and matches lines against the provided pattern. Plugins can override
+ /// this method to provide more efficient implementations.
+ ///
+ /// # Arguments
+ /// * `path` - The path to search (file or directory)
+ /// * `pattern` - The regular expression pattern to search for
+ /// * `recursive` - Whether to search recursively in subdirectories
+ /// * `case_insensitive` - Whether to perform case-insensitive matching
+ /// * `node_limit` - Maximum number of matches to return (None means no limit)
+ /// * `exclude_path` - Optional path prefix to exclude from search
+ /// * `level_limit` - Optional maximum depth relative to query root
+ ///
+ /// # Returns
+ /// A GrepResult containing all matches found
+ ///
+ /// # Errors
+ /// * `Error::NotFound` - If the path doesn't exist
+ /// * `Error::Regex` - If the pattern is invalid
+ async fn grep(
+ &self,
+ path: &str,
+ pattern: &str,
+ recursive: bool,
+ case_insensitive: bool,
+ node_limit: Option<usize>,
+ exclude_path: Option<&str>,
+ level_limit: Option<usize>,
+ ) -> Result<GrepResult> {
+ let regex_pattern = if case_insensitive {
+ format!("(?i){}", pattern)
+ } else {
+ pattern.to_string()
+ };
+
+ let re = Regex::new(&regex_pattern).map_err(|e| {
+ super::errors::Error::invalid_operation(format!("Invalid regex pattern: {}", e))
+ })?;
+
+ let mut result = GrepResult::new();
+ let normalized_path = normalize_prefix_path(path);
+ let normalized_exclude = exclude_path.map(normalize_prefix_path);
+
+ self.grep_internal(
+ normalized_path.as_str(),
+ normalized_path.as_str(),
+ &re,
+ recursive,
+ node_limit,
+ normalized_exclude.as_deref(),
+ level_limit,
+ &mut result,
+ )
+ .await?;
+
+ Ok(result)
+ }
+
+ /// Internal recursive grep helper
+ async fn grep_internal(
+ &self,
+ base_path: &str,
+ current_path: &str,
+ re: &Regex,
+ recursive: bool,
+ node_limit: Option<usize>,
+ exclude_path: Option<&str>,
+ level_limit: Option<usize>,
+ result: &mut GrepResult,
+ ) -> Result<()> {
+ if node_limit.is_some_and(|limit| result.count >= limit) {
+ return Ok(());
+ }
+
+ if let Some(exclude) = exclude_path {
+ if is_excluded_path(current_path, exclude) {
+ return Ok(());
+ }
+ }
+
+ let stat = self.stat(current_path).await?;
+
+ if stat.is_dir {
+ if !recursive && current_path != base_path {
+ return Ok(());
+ }
+
+ if let Some(limit) = level_limit {
+ let rel = relative_match_file(base_path, current_path);
+ let depth = relative_depth(&rel);
+ // Directories at depth >= limit cannot contain files within the limit.
+ if depth >= limit {
+ return Ok(());
+ }
+ }
+
+ let entries = self.read_dir(current_path).await?;
+
+ for entry in entries {
+ if node_limit.is_some_and(|limit| result.count >= limit) {
+ break;
+ }
+
+ let entry_path = if current_path == "/" {
+ format!("/{}", entry.name)
+ } else {
+ format!("{}/{}", current_path, entry.name)
+ };
+
+ self.grep_internal(
+ base_path,
+ &entry_path,
+ re,
+ recursive,
+ node_limit,
+ exclude_path,
+ level_limit,
+ result,
+ )
+ .await?;
+ }
+ } else {
+ if let Some(limit) = level_limit {
+ let rel = relative_match_file(base_path, current_path);
+ let depth = relative_depth(&rel);
+ if depth > limit {
+ return Ok(());
+ }
+ }
+
+ self.grep_file(base_path, current_path, re, node_limit, result)
+ .await?;
+ }
+
+ Ok(())
+ }
+
+ /// Grep a single file
+ async fn grep_file(
+ &self,
+ base_path: &str,
+ path: &str,
+ re: &Regex,
+ node_limit: Option<usize>,
+ result: &mut GrepResult,
+ ) -> Result<()> {
+ if node_limit.is_some_and(|limit| result.count >= limit) {
+ return Ok(());
+ }
+
+ let content = self.read(path, 0, 0).await?;
+
+ let content_str = String::from_utf8_lossy(&content);
+
+ let rel_file = relative_match_file(base_path, path);
+
+ for (line_num, line) in content_str.lines().enumerate() {
+ if node_limit.is_some_and(|limit| result.count >= limit) {
+ break;
+ }
+
+ if re.is_match(line) {
+ result.add_match(rel_file.clone(), (line_num + 1) as u64, line.to_string());
+ }
+ }
+
+ Ok(())
+ }
}
#[cfg(test)]
mod tests {
use super::*;
+ use std::collections::HashMap;
// Mock filesystem for testing
struct MockFS;
@@ -191,7 +428,13 @@ mod tests {
Ok(vec![])
}
- async fn write(&self, _path: &str, _data: &[u8], _offset: u64, _flags: WriteFlag) -> Result<u64> {
+ async fn write(
+ &self,
+ _path: &str,
+ _data: &[u8],
+ _offset: u64,
+ _flags: WriteFlag,
+ ) -> Result<u64> {
Ok(_data.len() as u64)
}
@@ -217,4 +460,158 @@ mod tests {
let fs = MockFS;
assert!(fs.exists("/test").await);
}
+
+ #[derive(Default)]
+ struct TreeFS {
+ /// Map: directory path -> entries (name, is_dir)
+ dirs: HashMap<String, Vec<(String, bool)>>,
+ /// Map: file path -> utf-8 content
+ files: HashMap<String, String>,
+ }
+
+ impl TreeFS {
+ /// Add/update a file and its content.
+ fn with_file(mut self, path: &str, content: &str) -> Self {
+ self.files.insert(path.to_string(), content.to_string());
+ self
+ }
+
+ /// Define directory entries for a directory.
+ fn with_dir_entries(mut self, dir: &str, entries: Vec<(&str, bool)>) -> Self {
+ self.dirs.insert(
+ dir.to_string(),
+ entries
+ .into_iter()
+ .map(|(n, is_dir)| (n.to_string(), is_dir))
+ .collect(),
+ );
+ self
+ }
+
+ /// Helper to build a FileInfo in tests.
+ fn file_info(name: String, is_dir: bool) -> FileInfo {
+ if is_dir {
+ FileInfo::new_dir(name, 0o755)
+ } else {
+ FileInfo::new_file(name, 0, 0o644)
+ }
+ }
+ }
+
+ #[async_trait]
+ impl FileSystem for TreeFS {
+ async fn create(&self, _path: &str) -> Result<()> {
+ Ok(())
+ }
+ async fn mkdir(&self, _path: &str, _mode: u32) -> Result<()> {
+ Ok(())
+ }
+ async fn remove(&self, _path: &str) -> Result<()> {
+ Ok(())
+ }
+ async fn remove_all(&self, _path: &str) -> Result<()> {
+ Ok(())
+ }
+
+ async fn read(&self, path: &str, _offset: u64, _size: u64) -> Result<Vec<u8>> {
+ let s = self.files.get(path).cloned().unwrap_or_default();
+ Ok(s.into_bytes())
+ }
+
+ async fn write(
+ &self,
+ _path: &str,
+ data: &[u8],
+ _offset: u64,
+ _flags: WriteFlag,
+ ) -> Result<u64> {
+ Ok(data.len() as u64)
+ }
+
+ async fn read_dir(&self, path: &str) -> Result<Vec<FileInfo>> {
+ let entries = self.dirs.get(path).cloned().unwrap_or_default();
+ Ok(entries
+ .into_iter()
+ .map(|(name, is_dir)| TreeFS::file_info(name, is_dir))
+ .collect())
+ }
+
+ async fn stat(&self, path: &str) -> Result<FileInfo> {
+ if self.dirs.contains_key(path) {
+ return Ok(TreeFS::file_info(path.to_string(), true));
+ }
+ if self.files.contains_key(path) {
+ return Ok(TreeFS::file_info(path.to_string(), false));
+ }
+ Ok(TreeFS::file_info(path.to_string(), false))
+ }
+
+ async fn rename(&self, _old_path: &str, _new_path: &str) -> Result<()> {
+ Ok(())
+ }
+ async fn chmod(&self, _path: &str, _mode: u32) -> Result<()> {
+ Ok(())
+ }
+ }
+
+ #[tokio::test]
+ async fn test_default_grep_match_file_is_query_root_relative() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/root", vec![("a.txt", false), ("sub", true)])
+ .with_dir_entries("/root/sub", vec![("b.txt", false)])
+ .with_file("/root/a.txt", "hello\n")
+ .with_file("/root/sub/b.txt", "hello\n");
+
+ let out = fs
+ .grep("/root", "hello", true, false, None, None, None)
+ .await
+ .unwrap();
+
+ let files: Vec<String> = out.matches.into_iter().map(|m| m.file).collect();
+ assert!(files.contains(&"a.txt".to_string()));
+ assert!(files.contains(&"sub/b.txt".to_string()));
+ }
+
+ #[tokio::test]
+ async fn test_default_grep_exclude_path_applies_before_node_limit() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/root", vec![("excluded", true), ("ok", true)])
+ .with_dir_entries("/root/excluded", vec![("x.txt", false)])
+ .with_dir_entries("/root/ok", vec![("y.txt", false)])
+ .with_file("/root/excluded/x.txt", "hit\n")
+ .with_file("/root/ok/y.txt", "hit\n");
+
+ let out = fs
+ .grep(
+ "/root",
+ "hit",
+ true,
+ false,
+ Some(1),
+ Some("/root/excluded"),
+ None,
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(out.count, 1);
+ assert_eq!(out.matches[0].file, "ok/y.txt");
+ }
+
+ #[tokio::test]
+ async fn test_default_grep_level_limit_applies_before_node_limit() {
+ let fs = TreeFS::default()
+ .with_dir_entries("/root", vec![("a.txt", false), ("deep", true)])
+ .with_dir_entries("/root/deep", vec![("b.txt", false)])
+ .with_file("/root/a.txt", "hit\n")
+ .with_file("/root/deep/b.txt", "hit\n");
+
+ let out = fs
+ .grep("/root", "hit", true, false, Some(1), None, Some(1))
+ .await
+ .unwrap();
+
+ assert_eq!(out.count, 1);
+ assert_eq!(out.matches[0].file, "a.txt");
+ }
}
diff --git a/crates/ragfs/src/core/mod.rs b/crates/ragfs/src/core/mod.rs
index 9b1e1730e..58ce85cfb 100644
--- a/crates/ragfs/src/core/mod.rs
+++ b/crates/ragfs/src/core/mod.rs
@@ -18,4 +18,4 @@ pub use errors::{Error, Result};
pub use filesystem::FileSystem;
pub use mountable::MountableFS;
pub use plugin::{HealthStatus, PluginRegistry, ServicePlugin};
-pub use types::{ConfigParameter, ConfigValue, FileInfo, PluginConfig, WriteFlag};
+pub use types::{ConfigParameter, ConfigValue, FileInfo, GrepResult, PluginConfig, WriteFlag};
diff --git a/crates/ragfs/src/core/mountable.rs b/crates/ragfs/src/core/mountable.rs
index 7bee90cfd..1bcc6ad2e 100644
--- a/crates/ragfs/src/core/mountable.rs
+++ b/crates/ragfs/src/core/mountable.rs
@@ -13,7 +13,7 @@ use tokio::sync::RwLock;
use super::errors::{Error, Result};
use super::filesystem::FileSystem;
use super::plugin::ServicePlugin;
-use super::types::{FileInfo, PluginConfig, WriteFlag};
+use super::types::{FileInfo, GrepResult, PluginConfig, WriteFlag};
/// Information about a mounted filesystem
#[derive(Clone)]
@@ -279,6 +279,51 @@ impl FileSystem for MountableFS {
let (mount_info, rel_path) = self.find_mount(path).await?;
mount_info.fs.truncate(&rel_path, size).await
}
+
+ async fn grep(
+ &self,
+ path: &str,
+ pattern: &str,
+ recursive: bool,
+ case_insensitive: bool,
+ node_limit: Option<usize>,
+ exclude_path: Option<&str>,
+ level_limit: Option<usize>,
+ ) -> Result<GrepResult> {
+ // Route grep to the mounted plugin so plugin-specific fast paths (e.g. localfs + rg)
+ // can take effect. If a plugin doesn't override grep, it will fall back to the trait
+ // default implementation on that plugin instance (still correct, just slower).
+ let (mount_info, rel_path) = self.find_mount(path).await?;
+
+ // Exclude path only applies when it resolves to the same mount point; otherwise it
+ // should not affect searching under `path`.
+ let exclude_rel: Option<String> = match exclude_path {
+ None => None,
+ Some(excl_abs) => match self.find_mount(excl_abs).await {
+ Ok((exclude_mount, excl_rel)) => {
+ if exclude_mount.path == mount_info.path {
+ Some(excl_rel)
+ } else {
+ None
+ }
+ }
+ Err(_) => None,
+ },
+ };
+
+ mount_info
+ .fs
+ .grep(
+ &rel_path,
+ pattern,
+ recursive,
+ case_insensitive,
+ node_limit,
+ exclude_rel.as_deref(),
+ level_limit,
+ )
+ .await
+ }
}
#[cfg(test)]
@@ -321,7 +366,13 @@ mod tests {
Ok(self.name.as_bytes().to_vec())
}
- async fn write(&self, _path: &str, data: &[u8], _offset: u64, _flags: WriteFlag) -> Result<u64> {
+ async fn write(
+ &self,
+ _path: &str,
+ data: &[u8],
+ _offset: u64,
+ _flags: WriteFlag,
+ ) -> Result<u64> {
Ok(data.len() as u64)
}
@@ -340,6 +391,22 @@ mod tests {
async fn chmod(&self, _path: &str, _mode: u32) -> Result<()> {
Ok(())
}
+
+ async fn grep(
+ &self,
+ path: &str,
+ pattern: &str,
+ _recursive: bool,
+ _case_insensitive: bool,
+ _node_limit: Option<usize>,
+ _exclude_path: Option<&str>,
+ _level_limit: Option<usize>,
+ ) -> Result<GrepResult> {
+ let mut out = GrepResult::new();
+ // Encode the received rel_path into the match so the test can assert routing worked.
+ out.add_match(path.to_string(), 1, pattern.to_string());
+ Ok(out)
+ }
}
// Mock plugin for testing
@@ -471,7 +538,10 @@ mod tests {
assert_eq!(data, b"mock");
// Test write operation
- let written = mfs.write("/mock/test.txt", b"hello", 0, WriteFlag::Create).await.unwrap();
+ let written = mfs
+ .write("/mock/test.txt", b"hello", 0, WriteFlag::Create)
+ .await
+ .unwrap();
assert_eq!(written, 5);
// Test stat operation
@@ -578,7 +648,8 @@ mod tests {
// Register multiple plugins
for i in 0..5 {
- mfs.register_plugin(MockPlugin::new(&format!("mock{}", i))).await;
+ mfs.register_plugin(MockPlugin::new(&format!("mock{}", i)))
+ .await;
}
// Spawn concurrent mount operations
@@ -610,9 +681,8 @@ mod tests {
let mut handles = vec![];
for i in 0..5 {
let mfs_clone = Arc::clone(&mfs);
- let handle = task::spawn(async move {
- mfs_clone.unmount(&format!("/mock{}", i)).await
- });
+ let handle =
+ task::spawn(async move { mfs_clone.unmount(&format!("/mock{}", i)).await });
handles.push(handle);
}
@@ -626,4 +696,28 @@ mod tests {
let mounts = mfs.list_mounts().await;
assert!(mounts.is_empty());
}
+
+ #[tokio::test]
+ async fn test_grep_routes_to_plugin() {
+ let mfs = MountableFS::new();
+ mfs.register_plugin(MockPlugin::new("mock")).await;
+
+ let config = PluginConfig {
+ name: "mock".to_string(),
+ mount_path: "/mock".to_string(),
+ params: HashMap::new(),
+ };
+ mfs.mount(config).await.unwrap();
+
+ let result = mfs
+ .grep("/mock/a.txt", "foo", false, false, None, None, None)
+ .await
+ .unwrap();
+
+ assert_eq!(result.count, 1);
+ assert_eq!(result.matches.len(), 1);
+ assert_eq!(result.matches[0].file, "/a.txt");
+ assert_eq!(result.matches[0].line, 1);
+ assert_eq!(result.matches[0].content, "foo");
+ }
}
diff --git a/crates/ragfs/src/core/types.rs b/crates/ragfs/src/core/types.rs
index 175bd8abf..039409aa5 100644
--- a/crates/ragfs/src/core/types.rs
+++ b/crates/ragfs/src/core/types.rs
@@ -7,6 +7,69 @@ use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::time::SystemTime;
+/// Grep match result
+///
+/// Represents a single match found during a grep operation.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GrepMatch {
+ /// File path where the match was found
+ pub file: String,
+
+ /// Line number (1-based)
+ pub line: u64,
+
+ /// Content of the matched line
+ pub content: String,
+}
+
+/// Grep operation result
+///
+/// Contains all matches found during a grep operation.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct GrepResult {
+ /// List of matches
+ pub matches: Vec<GrepMatch>,
+
+ /// Total number of matches
+ pub count: usize,
+}
+
+impl GrepResult {
+ /// Create a new empty GrepResult
+ pub fn new() -> Self {
+ Self {
+ matches: Vec::new(),
+ count: 0,
+ }
+ }
+
+ /// Create a GrepResult from a list of matches
+ pub fn from_matches(matches: Vec<GrepMatch>) -> Self {
+ let count = matches.len();
+ Self { matches, count }
+ }
+
+ /// Add a match to the result
+ pub fn add_match(&mut self, file: String, line: u64, content: String) {
+ self.matches.push(GrepMatch { file, line, content });
+ self.count += 1;
+ }
+
+ /// Limit the number of matches
+ pub fn limit(&mut self, max_count: usize) {
+ if self.matches.len() > max_count {
+ self.matches.truncate(max_count);
+ self.count = max_count;
+ }
+ }
+}
+
+impl Default for GrepResult {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
/// File metadata information
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct FileInfo {
diff --git a/crates/ragfs/src/plugins/localfs/mod.rs b/crates/ragfs/src/plugins/localfs/mod.rs
index 2249aa2c7..1eb570c7e 100644
--- a/crates/ragfs/src/plugins/localfs/mod.rs
+++ b/crates/ragfs/src/plugins/localfs/mod.rs
@@ -7,15 +7,26 @@ use async_trait::async_trait;
use std::fs;
use std::path::{Path, PathBuf};
+use std::io::{BufRead, BufReader};
+use std::process::{Command, Stdio};
+
+use grep_regex::RegexMatcher;
+use grep_searcher::sinks::UTF8;
+use grep_searcher::{BinaryDetection, SearcherBuilder};
+use ignore::WalkBuilder;
+use serde::Deserialize;
+
use crate::core::errors::{Error, Result};
use crate::core::filesystem::FileSystem;
use crate::core::plugin::ServicePlugin;
-use crate::core::types::{ConfigParameter, FileInfo, PluginConfig, WriteFlag};
+use crate::core::types::{ConfigParameter, FileInfo, GrepResult, PluginConfig, WriteFlag};
/// LocalFS - Local file system implementation
pub struct LocalFileSystem {
/// Base path of the mounted directory
base_path: PathBuf,
+ /// Whether external `rg` is available in current process PATH.
+ has_rg: bool,
}
impl LocalFileSystem {
@@ -45,7 +56,22 @@ impl LocalFileSystem {
)));
}
- Ok(Self { base_path: path })
+ // Canonicalize to stabilize prefix stripping and make rg JSON path mapping more reliable.
+ let canonical = fs::canonicalize(&path)
+ .map_err(|e| Error::plugin(format!("failed to canonicalize base path: {}", e)))?;
+
+ let has_rg = Command::new("rg")
+ .arg("--version")
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status()
+ .map(|s| s.success())
+ .unwrap_or(false);
+
+ Ok(Self {
+ base_path: canonical,
+ has_rg,
+ })
}
/// Resolve a virtual path to actual local path
@@ -60,6 +86,445 @@ impl LocalFileSystem {
self.base_path.join(relative)
}
}
+
+ /// Run blocking grep work on a dedicated thread and normalize join errors.
+ async fn run_blocking_grep<T, F>(job: F) -> Result<T>
+ where
+ T: Send + 'static,
+ F: FnOnce() -> Result<T> + Send + 'static,
+ {
+ tokio::task::spawn_blocking(job)
+ .await
+ .map_err(|e| Error::Internal(format!("spawn_blocking failed: {}", e)))?
+ }
+
+ /// Convert an exclude path (plugin-relative "virtual mount" form, often starts with "/")
+ /// into a query-root-relative path.
+ ///
+ /// Contract:
+ /// - Returns `Some(".")` when the exclude path covers the entire query root (exclude all).
+ /// - Returns `Some(rel)` when the exclude path is under the query root (so it can be
+ /// compared with `file_virtual`, which is always query-root-relative).
+ /// - Returns `None` when the exclude path is outside the query root (no effect).
+ fn exclude_to_query_root_relative_path(
+ base_path: &Path,
+ query_root_local: &Path,
+ exclude_path: &str,
+ ) -> Option<String> {
+ let p = exclude_path.trim_end_matches('/');
+ if p.is_empty() {
+ return Some(".".to_string());
+ }
+
+ let exclude_local = Self::resolve_virtual_path(base_path, p);
+
+ // If the exclude prefix includes the query root itself, exclude everything under the query root.
+ if query_root_local == exclude_local
+ || query_root_local.strip_prefix(&exclude_local).is_ok()
+ {
+ return Some(".".to_string());
+ }
+
+ // If exclude is under query root, convert it to query-root-relative.
+ let rel = exclude_local.strip_prefix(query_root_local).ok()?;
+ let s = rel.to_string_lossy();
+ if s.is_empty() {
+ Some(".".to_string())
+ } else {
+ Some(s.to_string())
+ }
+ }
+
+ /// Check whether a query-root-relative `file_virtual` is excluded by `exclude_rel`.
+ fn is_excluded_virtual(file_virtual: &str, exclude_rel: &str) -> bool {
+ if exclude_rel == "." {
+ return true;
+ }
+ if file_virtual == exclude_rel || exclude_rel.is_empty() {
+ return file_virtual == exclude_rel;
+ }
+
+ file_virtual
+ .strip_prefix(exclude_rel)
+ .is_some_and(|suffix| suffix.starts_with('/'))
+ }
+
+ /// Compute depth of a query-root-relative path.
+ fn virtual_depth(file_virtual: &str) -> usize {
+ if file_virtual.is_empty() || file_virtual == "." {
+ 0
+ } else {
+ file_virtual.split('/').filter(|p| !p.is_empty()).count()
+ }
+ }
+
+ /// Execute grep via external `rg` (ripgrep) and return `GrepResult`.
+ ///
+ /// Returns `Ok(None)` when `rg` is unavailable or cannot be executed (e.g. not in PATH),
+ /// so the caller can fall back to the in-process implementation.
+ fn grep_via_rg(
+ base_path: &Path,
+ target_path: &Path,
+ pattern: &str,
+ recursive: bool,
+ case_insensitive: bool,
+ node_limit: Option<usize>,
+ exclude_path: Option<&str>,
+ level_limit: Option<usize>,
+ ) -> Result<Option<GrepResult>> {
+ if node_limit == Some(0) {
+ return Ok(Some(GrepResult::new()));
+ }
+
+ let mut cmd = Command::new("rg");
+ cmd.arg("--json");
+ cmd.arg("--no-messages"); // Avoid interleaving permission/IO warnings with JSON output
+ cmd.arg("--no-ignore-parent"); // Match fallback `.parents(false)` semantics.
+
+ if case_insensitive {
+ cmd.arg("-i");
+ }
+
+ // Non-recursive directory search: only scan the current directory (no descent).
+ if !recursive && target_path.is_dir() {
+ cmd.arg("--max-depth").arg("1");
+ }
+
+ // NOTE: rg's --max-count is a per-file limit; we enforce a global limit by terminating early while parsing.
+ if let Some(limit) = node_limit {
+ cmd.arg("--max-count").arg(limit.to_string());
+ }
+
+ // Run rg from base_path so we can interpret relative paths in JSON output reliably.
+ let current_dir = if target_path.is_dir() {
+ target_path
+ } else {
+ target_path.parent().unwrap_or(target_path)
+ };
+ cmd.current_dir(current_dir);
+
+ cmd.arg(pattern);
+ // Search relative to the query root so returned paths can be interpreted as query-root relative.
+ if target_path.is_dir() {
+ cmd.arg(".");
+ } else {
+ cmd.arg(
+ target_path
+ .file_name()
+ .ok_or_else(|| Error::InvalidPath(target_path.display().to_string()))?,
+ );
+ }
+ cmd.stdout(Stdio::piped());
+ cmd.stderr(Stdio::piped());
+
+ let mut child = match cmd.spawn() {
+ Ok(c) => c,
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
+ Err(e) => {
+ return Err(Error::InvalidOperation(format!("spawn rg failed: {}", e)));
+ }
+ };
+
+ let stdout = child
+ .stdout
+ .take()
+ .ok_or_else(|| Error::Internal("rg stdout missing".to_string()))?;
+ let mut stderr_pipe = child.stderr.take();
+ let reader = BufReader::new(stdout);
+
+ let mut out = GrepResult::new();
+ let mut killed_for_limit = false;
+ let exclude_rel = exclude_path
+ .and_then(|p| Self::exclude_to_query_root_relative_path(base_path, target_path, p));
+
+ #[derive(Debug, Deserialize)]
+ struct RgEvent {
+ #[serde(rename = "type")]
+ kind: String,
+ #[serde(default)]
+ data: Option<RgMatchData>,
+ }
+
+ #[derive(Debug, Deserialize)]
+ struct RgMatchData {
+ path: RgTextField,
+ #[serde(default)]
+ line_number: u64,
+ #[serde(default)]
+ lines: RgTextField,
+ }
+
+ #[derive(Debug, Deserialize, Default)]
+ struct RgTextField {
+ #[serde(default)]
+ text: String,
+ }
+
+ for line in reader.lines() {
+ let line = match line {
+ Ok(s) => s,
+ Err(_) => break,
+ };
+ if line.is_empty() {
+ continue;
+ }
+
+ let ev: RgEvent = match serde_json::from_str(&line) {
+ Ok(v) => v,
+ Err(_) => continue,
+ };
+
+ // Only handle "match" events.
+ if ev.kind != "match" {
+ continue;
+ }
+
+ let data = match ev.data {
+ Some(d) => d,
+ None => continue,
+ };
+
+ let file_text = data.path.text;
+ if file_text.is_empty() {
+ continue;
+ }
+
+ // `rg --json` may output relative paths depending on invocation. Convert to absolute using base_path.
+ let file_local = {
+ let p = Path::new(&file_text);
+ if p.is_absolute() {
+ p.to_path_buf()
+ } else {
+ current_dir.join(p)
+ }
+ };
+
+ let file_virtual = match Self::local_to_query_relative_path(target_path, &file_local) {
+ Some(p) => p,
+ None => continue,
+ };
+
+ if let Some(excl_rel) = exclude_rel.as_deref() {
+ if Self::is_excluded_virtual(&file_virtual, excl_rel) {
+ continue;
+ }
+ }
+
+ if let Some(limit) = level_limit {
+ if Self::virtual_depth(&file_virtual) > limit {
+ continue;
+ }
+ }
+
+ let line_no = data.line_number;
+
+ // `lines.text` usually contains the full line, possibly including trailing newline.
+ let content = data
+ .lines
+ .text
+ .trim_end_matches(&['\r', '\n'][..])
+ .to_string();
+
+ out.add_match(file_virtual, line_no, content);
+
+ if let Some(limit) = node_limit {
+ if out.count >= limit {
+ // Reached global limit: terminate the rg process to avoid further scanning.
+ let _ = child.kill();
+ killed_for_limit = true;
+ break;
+ }
+ }
+ }
+
+ let status = child
+ .wait()
+ .map_err(|e| Error::InvalidOperation(format!("wait rg failed: {}", e)))?;
+
+ // rg exit code: 0=matches, 1=no matches, 2=error
+ // If we killed it due to node_limit, ignore the exit code.
+ if !killed_for_limit && status.code() == Some(2) {
+ let mut stderr = String::new();
+ if let Some(mut err) = stderr_pipe.take() {
+ use std::io::Read;
+ let _ = err.read_to_string(&mut stderr);
+ }
+ return Err(Error::InvalidOperation(format!("rg failed: {}", stderr)));
+ }
+
+ Ok(Some(out))
+ }
+
+ /// In-process grep fallback using Rust crates, aiming to match rg-like defaults:
+ /// - respect .gitignore/.ignore
+ /// - skip hidden files/dirs by default
+ /// - skip binary files by default
+ fn grep_via_libs(
+ base_path: &Path,
+ virtual_path: &str,
+ pattern: &str,
+ recursive: bool,
+ case_insensitive: bool,
+ node_limit: Option<usize>,
+ exclude_path: Option<&str>,
+ level_limit: Option<usize>,
+ ) -> Result<GrepResult> {
+ let local_root = Self::resolve_virtual_path(base_path, virtual_path);
+ if !local_root.exists() {
+ return Err(Error::NotFound(virtual_path.to_string()));
+ }
+
+ if node_limit == Some(0) {
+ return Ok(GrepResult::new());
+ }
+
+ // Build regex for fallback (uses Rust `regex` semantics).
+ let regex_pattern = if case_insensitive {
+ format!("(?i){}", pattern)
+ } else {
+ pattern.to_string()
+ };
+
+ let matcher = RegexMatcher::new_line_matcher(&regex_pattern)
+ .map_err(|e| Error::InvalidOperation(format!("Invalid regex pattern: {}", e)))?;
+
+ let mut searcher = SearcherBuilder::new()
+ .line_number(true)
+ .binary_detection(BinaryDetection::quit(b'\x00'))
+ .build();
+
+ let mut out = GrepResult::new();
+ let limit = node_limit.unwrap_or(usize::MAX);
+ let exclude_rel = exclude_path
+ .and_then(|p| Self::exclude_to_query_root_relative_path(base_path, &local_root, p));
+
+ // Single file: search directly.
+ if local_root.is_file() {
+ let file_virtual = Self::local_to_query_relative_path(&local_root, &local_root)
+ .ok_or_else(|| Error::InvalidPath(virtual_path.to_string()))?;
+
+ let mut remaining = limit.saturating_sub(out.count);
+ let sink = UTF8(|lnum, line| {
+ if remaining == 0 {
+ return Ok(false);
+ }
+ if let Some(excl_rel) = exclude_rel.as_deref() {
+ if Self::is_excluded_virtual(&file_virtual, excl_rel) {
+ return Ok(true);
+ }
+ }
+ if let Some(max_depth) = level_limit {
+ if Self::virtual_depth(&file_virtual) > max_depth {
+ return Ok(true);
+ }
+ }
+ out.add_match(
+ file_virtual.clone(),
+ lnum as u64,
+ line.trim_end().to_string(),
+ );
+ remaining -= 1;
+ Ok(remaining > 0)
+ });
+
+ // Similar to rg's --no-messages: file-level errors are not fatal here.
+ let _ = searcher.search_path(&matcher, &local_root, sink);
+ return Ok(out);
+ }
+
+ // Directory traversal: use ignore::WalkBuilder and explicitly set rg-like defaults.
+ //
+ // NOTE: We intentionally do NOT inherit ignore rules from parent directories.
+ // In OpenViking, a localfs mount may live under a git repo whose root `.gitignore`
+ // ignores the storage directory (e.g. `data/`). Inheriting parent ignore rules would
+ // cause grep to return empty results unexpectedly.
+ let mut builder = WalkBuilder::new(&local_root);
+ builder
+ // In ignore crate, hidden(true) means "ignore hidden" (enabled by default).
+ // We set it explicitly to match rg default behavior.
+ .hidden(true)
+ .parents(false)
+ .ignore(true)
+ .git_ignore(true)
+ .git_global(true)
+ .git_exclude(true);
+ // Apply git-related ignore rules even if the target directory isn't inside a git repo.
+ builder.require_git(false);
+ if !recursive && local_root.is_dir() {
+ builder.max_depth(Some(1));
+ }
+
+ for dent in builder.build() {
+ if out.count >= limit {
+ break;
+ }
+
+ let dent = match dent {
+ Ok(d) => d,
+ Err(_) => continue,
+ };
+
+ let ft_is_file = dent.file_type().map(|t| t.is_file()).unwrap_or(false);
+ if !ft_is_file {
+ continue;
+ }
+
+ let file_path = dent.path();
+ let file_virtual = match Self::local_to_query_relative_path(&local_root, file_path) {
+ Some(p) => p,
+ None => continue,
+ };
+
+ if let Some(excl_rel) = exclude_rel.as_deref() {
+ if Self::is_excluded_virtual(&file_virtual, excl_rel) {
+ continue;
+ }
+ }
+
+ if let Some(max_depth) = level_limit {
+ if Self::virtual_depth(&file_virtual) > max_depth {
+ continue;
+ }
+ }
+
+ let mut remaining = limit.saturating_sub(out.count);
+ let sink = UTF8(|lnum, line| {
+ if remaining == 0 {
+ return Ok(false);
+ }
+ out.add_match(
+ file_virtual.clone(),
+ lnum as u64,
+ line.trim_end().to_string(),
+ );
+ remaining -= 1;
+ Ok(remaining > 0)
+ });
+
+ // Skip file read/permission errors to better match rg default behavior.
+ let _ = searcher.search_path(&matcher, file_path, sink);
+ }
+
+ Ok(out)
+ }
+ fn local_to_query_relative_path(query_root: &Path, local: &Path) -> Option<String> {
+ let rel = local.strip_prefix(query_root).ok()?;
+ let s = rel.to_string_lossy();
+ if s.is_empty() {
+ Some(".".to_string())
+ } else {
+ Some(s.to_string())
+ }
+ }
+
+ fn resolve_virtual_path(base_path: &Path, path: &str) -> PathBuf {
+ let relative = path.strip_prefix('/').unwrap_or(path);
+ if relative.is_empty() {
+ base_path.to_path_buf()
+ } else {
+ base_path.join(relative)
+ }
+ }
}
#[async_trait]
@@ -153,8 +618,7 @@ impl FileSystem for LocalFileSystem {
let local_path = self.resolve_path(path);
// Check if exists and is not a directory
- let metadata = fs::metadata(&local_path)
- .map_err(|_| Error::NotFound(path.to_string()))?;
+ let metadata = fs::metadata(&local_path).map_err(|_| Error::NotFound(path.to_string()))?;
if metadata.is_dir() {
return Err(Error::plugin(format!("is a directory: {}", path)));
@@ -266,8 +730,7 @@ impl FileSystem for LocalFileSystem {
let local_path = self.resolve_path(path);
// Get file metadata
- let metadata = fs::metadata(&local_path)
- .map_err(|_| Error::NotFound(path.to_string()))?;
+ let metadata = fs::metadata(&local_path).map_err(|_| Error::NotFound(path.to_string()))?;
let name = Path::new(path)
.file_name()
@@ -323,6 +786,342 @@ impl FileSystem for LocalFileSystem {
// For now, just return success
Ok(())
}
+
+ async fn grep(
+ &self,
+ path: &str,
+ pattern: &str,
+ recursive: bool,
+ case_insensitive: bool,
+ node_limit: Option<usize>,
+ exclude_path: Option<&str>,
+ level_limit: Option<usize>,
+ ) -> Result<GrepResult> {
+ // Fast-path: requesting 0 matches should return immediately without touching disk or spawning processes.
+ if node_limit == Some(0) {
+ return Ok(GrepResult::new());
+ }
+
+ let local = self.resolve_path(path);
+ if !local.exists() {
+ return Err(Error::NotFound(path.to_string()));
+ }
+ let pattern_owned = pattern.to_string();
+ let path_owned = path.to_string();
+ let base_path = self.base_path.clone();
+ let exclude_owned = exclude_path.map(|s| s.to_string());
+
+ if self.has_rg {
+ // External rg path: run in a blocking thread to avoid blocking the async runtime.
+ let rg_pattern = pattern_owned.clone();
+ let rg_base_path = base_path.clone();
+ let rg_exclude = exclude_owned.clone();
+ let rg_res = Self::run_blocking_grep(move || {
+ LocalFileSystem::grep_via_rg(
+ rg_base_path.as_path(),
+ local.as_path(),
+ rg_pattern.as_str(),
+ recursive,
+ case_insensitive,
+ node_limit,
+ rg_exclude.as_deref(),
+ level_limit,
+ )
+ })
+ .await?;
+
+ if let Some(out) = rg_res {
+ return Ok(out);
+ }
+ }
+
+ // Fallback: also run in a blocking thread (dir walking + file reads are blocking IO).
+ Self::run_blocking_grep(move || {
+ LocalFileSystem::grep_via_libs(
+ base_path.as_path(),
+ &path_owned,
+ &pattern_owned,
+ recursive,
+ case_insensitive,
+ node_limit,
+ exclude_owned.as_deref(),
+ level_limit,
+ )
+ })
+ .await
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ struct EnvVarGuard {
+ key: &'static str,
+ old: Option<std::ffi::OsString>,
+ }
+
+ impl EnvVarGuard {
+ fn set(key: &'static str, val: &std::ffi::OsStr) -> Self {
+ let old = std::env::var_os(key);
+ std::env::set_var(key, val);
+ Self { key, old }
+ }
+ }
+
+ impl Drop for EnvVarGuard {
+ fn drop(&mut self) {
+ if let Some(v) = self.old.take() {
+ std::env::set_var(self.key, v);
+ } else {
+ std::env::remove_var(self.key);
+ }
+ }
+ }
+
+ #[tokio::test]
+ async fn test_localfs_grep_fallback_respects_gitignore_and_hidden() {
+ use tempfile::TempDir;
+
+ let dir = TempDir::new().unwrap();
+ std::fs::write(dir.path().join(".gitignore"), "ignored.txt\n").unwrap();
+ std::fs::write(dir.path().join("a.txt"), "hello\n").unwrap();
+ std::fs::write(dir.path().join("ignored.txt"), "hello\n").unwrap();
+ std::fs::create_dir_all(dir.path().join(".hidden")).unwrap();
+ std::fs::write(dir.path().join(".hidden/secret.txt"), "hello\n").unwrap();
+
+ let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
+ fs.has_rg = false;
+
+ let result = fs
+ .grep("/", "hello", true, false, None, None, None)
+ .await
+ .unwrap();
+
+ assert_eq!(result.count, 1);
+ assert_eq!(result.matches[0].file, "a.txt");
+ }
+
+ #[tokio::test]
+ async fn test_localfs_grep_case_insensitive() {
+ use tempfile::TempDir;
+
+ let dir = TempDir::new().unwrap();
+ std::fs::write(dir.path().join("a.txt"), "HELLO\n").unwrap();
+
+ let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
+ fs.has_rg = false;
+
+ let result = fs
+ .grep("/", "hello", true, true, None, None, None)
+ .await
+ .unwrap();
+
+ assert_eq!(result.count, 1);
+ }
+
+ #[tokio::test]
+ async fn test_localfs_grep_missing_path_returns_not_found() {
+ use tempfile::TempDir;
+
+ let dir = TempDir::new().unwrap();
+ let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
+ fs.has_rg = false;
+
+ let err = fs
+ .grep("/does-not-exist", "hello", true, false, None, None, None)
+ .await
+ .unwrap_err();
+ assert!(matches!(err, Error::NotFound(_)));
+ }
+
+ #[tokio::test]
+ async fn test_localfs_grep_does_not_inherit_parent_gitignore() {
+ use tempfile::TempDir;
+
+ let dir = TempDir::new().unwrap();
+ let mount_dir = dir.path().join("mount");
+ std::fs::create_dir_all(&mount_dir).unwrap();
+
+ // Parent .gitignore ignores the mounted directory entirely.
+ std::fs::write(dir.path().join(".gitignore"), "mount/\n").unwrap();
+ std::fs::write(mount_dir.join("a.txt"), "hello\n").unwrap();
+
+ let mut fs = LocalFileSystem::new(mount_dir.to_str().unwrap()).unwrap();
+ fs.has_rg = false;
+
+ let result = fs
+ .grep("/", "hello", true, false, None, None, None)
+ .await
+ .unwrap();
+ assert_eq!(result.count, 1);
+ assert_eq!(result.matches[0].file, "a.txt");
+ }
+
+ #[tokio::test]
+ async fn test_localfs_grep_returns_query_root_relative_paths() {
+ use tempfile::TempDir;
+
+ let dir = TempDir::new().unwrap();
+ std::fs::create_dir_all(dir.path().join("sub")).unwrap();
+ std::fs::write(dir.path().join("sub/a.txt"), "hello\n").unwrap();
+
+ let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
+ fs.has_rg = false;
+
+ let result = fs
+ .grep("/sub", "hello", true, false, None, None, None)
+ .await
+ .unwrap();
+ assert_eq!(result.count, 1);
+ assert_eq!(result.matches[0].file, "a.txt");
+
+ let single_file = fs
+ .grep("/sub/a.txt", "hello", true, false, None, None, None)
+ .await
+ .unwrap();
+ assert_eq!(single_file.count, 1);
+ assert_eq!(single_file.matches[0].file, ".");
+ }
+
+ #[cfg(unix)]
+ #[tokio::test]
+ async fn test_localfs_grep_node_limit_zero_does_not_invoke_rg() {
+ use tempfile::TempDir;
+
+ let dir = TempDir::new().unwrap();
+ std::fs::write(dir.path().join("a.txt"), "hello\n").unwrap();
+
+ // Create a fake `rg` that fails if invoked. The expected behavior is to short-circuit
+ // and return empty result when node_limit=0.
+ let bin_dir = dir.path().join("bin");
+ std::fs::create_dir_all(&bin_dir).unwrap();
+ let rg_path = bin_dir.join("rg");
+ std::fs::write(&rg_path, "#!/bin/sh\nexit 2\n").unwrap();
+ use std::os::unix::fs::PermissionsExt;
+ let mut perms = std::fs::metadata(&rg_path).unwrap().permissions();
+ perms.set_mode(0o755);
+ std::fs::set_permissions(&rg_path, perms).unwrap();
+
+ // Prepend fake rg to PATH.
+ let old_path = std::env::var_os("PATH").unwrap_or_default();
+ let mut new_path = std::ffi::OsString::new();
+ new_path.push(bin_dir.as_os_str());
+ new_path.push(std::ffi::OsStr::new(":"));
+ new_path.push(old_path);
+ let _path_guard = EnvVarGuard::set("PATH", &new_path);
+
+ let fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
+ let result = fs
+ .grep("/", "hello", true, false, Some(0), None, None)
+ .await
+ .unwrap();
+ assert_eq!(result.count, 0);
+ assert!(result.matches.is_empty());
+ }
+
+ #[tokio::test]
+ async fn test_localfs_grep_exclude_path_applies_before_node_limit() {
+ use tempfile::TempDir;
+
+ let dir = TempDir::new().unwrap();
+ std::fs::create_dir_all(dir.path().join("excluded")).unwrap();
+ std::fs::create_dir_all(dir.path().join("ok")).unwrap();
+ std::fs::write(dir.path().join("excluded/x.txt"), "hit\n").unwrap();
+ std::fs::write(dir.path().join("ok/y.txt"), "hit\n").unwrap();
+
+ let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
+ fs.has_rg = false;
+
+ let out = fs
+ .grep("/", "hit", true, false, Some(1), Some("/excluded"), None)
+ .await
+ .unwrap();
+
+ assert_eq!(out.count, 1);
+ assert_eq!(out.matches.len(), 1);
+ assert_eq!(out.matches[0].file, "ok/y.txt");
+ }
+
+ #[tokio::test]
+ async fn test_localfs_grep_exclude_path_is_query_root_relative_for_nested_query_root() {
+ use tempfile::TempDir;
+
+ let dir = TempDir::new().unwrap();
+ let nested_root = dir.path().join("acct/resources/docs");
+ std::fs::create_dir_all(nested_root.join("excluded")).unwrap();
+ std::fs::create_dir_all(nested_root.join("ok")).unwrap();
+ std::fs::write(nested_root.join("excluded/x.txt"), "hit\n").unwrap();
+ std::fs::write(nested_root.join("ok/y.txt"), "hit\n").unwrap();
+
+ let mut fs = LocalFileSystem::new(dir.path().to_str().unwrap()).unwrap();
+ fs.has_rg = false;
+
+ // Simulate MountableFS -> LocalFS: query root and exclude path are plugin-relative and nested.
+ let out = fs
+ .grep(
+ "/acct/resources/docs",
+ "hit",
+ true,
+ false,
+ Some(1),
+ Some("/acct/resources/docs/excluded"),
+ None,
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(out.count, 1);
+ assert_eq!(out.matches.len(), 1);
+ assert_eq!(out.matches[0].file, "ok/y.txt");
+ }
+
+ #[cfg(unix)]
+ #[tokio::test]
+ async fn test_localfs_grep_rg_fast_path_disables_parent_ignore_inheritance() {
+ use tempfile::TempDir;
+
+ let dir = TempDir::new().unwrap();
+ std::fs::write(dir.path().join(".gitignore"), "mount/\n").unwrap();
+
+ let mount_dir = dir.path().join("mount");
+ std::fs::create_dir_all(&mount_dir).unwrap();
+ std::fs::write(mount_dir.join("a.txt"), "hello\n").unwrap();
+
+ // Fake `rg` verifies `--no-ignore-parent` is present, then emits one JSON match.
+ let bin_dir = dir.path().join("bin");
+ std::fs::create_dir_all(&bin_dir).unwrap();
+ let rg_path = bin_dir.join("rg");
+ std::fs::write(
+ &rg_path,
+ format!(
+ "#!/bin/sh\nfor arg in \"$@\"; do\n if [ \"$arg\" = \"--no-ignore-parent\" ]; then\n printf '%s\\n' '{{\"type\":\"match\",\"data\":{{\"path\":{{\"text\":\"a.txt\"}},\"line_number\":1,\"lines\":{{\"text\":\"hello\\\\n\"}}}}}}'\n exit 0\n fi\ndone\necho missing --no-ignore-parent 1>&2\nexit 2\n"
+ ),
+ )
+ .unwrap();
+ use std::os::unix::fs::PermissionsExt;
+ let mut perms = std::fs::metadata(&rg_path).unwrap().permissions();
+ perms.set_mode(0o755);
+ std::fs::set_permissions(&rg_path, perms).unwrap();
+
+ let old_path = std::env::var_os("PATH").unwrap_or_default();
+ let mut new_path = std::ffi::OsString::new();
+ new_path.push(bin_dir.as_os_str());
+ new_path.push(std::ffi::OsStr::new(":"));
+ new_path.push(old_path);
+ let _path_guard = EnvVarGuard::set("PATH", &new_path);
+
+ let fs = LocalFileSystem::new(mount_dir.to_str().unwrap()).unwrap();
+ let result = fs
+ .grep("/", "hello", true, false, None, None, None)
+ .await
+ .unwrap();
+
+ assert_eq!(result.count, 1);
+ assert_eq!(result.matches.len(), 1);
+ assert_eq!(result.matches[0].file, "a.txt");
+ assert_eq!(result.matches[0].content, "hello");
+ }
}
/// LocalFS plugin
@@ -334,15 +1133,13 @@ impl LocalFSPlugin {
/// Create a new LocalFS plugin
pub fn new() -> Self {
Self {
- config_params: vec![
- ConfigParameter {
- name: "local_dir".to_string(),
- param_type: "string".to_string(),
- required: true,
- default: None,
- description: "Local directory path to expose (must exist)".to_string(),
- },
- ],
+ config_params: vec![ConfigParameter {
+ name: "local_dir".to_string(),
+ param_type: "string".to_string(),
+ required: true,
+ default: None,
+ description: "Local directory path to expose (must exist)".to_string(),
+ }],
}
}
}
diff --git a/crates/ragfs/src/server/handlers.rs b/crates/ragfs/src/server/handlers.rs
index a64e16f7b..4806681be 100644
--- a/crates/ragfs/src/server/handlers.rs
+++ b/crates/ragfs/src/server/handlers.rs
@@ -11,7 +11,7 @@ use axum::{
use serde::{Deserialize, Serialize};
use std::sync::Arc;
-use crate::core::{FileSystem, MountableFS, PluginConfig, WriteFlag};
+use crate::core::{FileSystem, GrepResult, MountableFS, PluginConfig, WriteFlag};
/// Shared application state
#[derive(Clone)]
@@ -92,6 +92,33 @@ pub struct UnmountRequest {
pub path: String,
}
+/// Request body for grep operation
+#[derive(Debug, Deserialize)]
+pub struct GrepRequest {
+ /// File or directory path to search
+ pub path: String,
+ /// Regular expression pattern
+ pub pattern: String,
+ /// Whether to search recursively
+ #[serde(default)]
+ pub recursive: bool,
+ /// Whether to perform case-insensitive matching
+ #[serde(default)]
+ pub case_insensitive: bool,
+ /// Whether to stream results (currently unsupported by ragfs server)
+ #[serde(default)]
+ pub stream: bool,
+ /// Maximum number of matches to return
+ #[serde(default)]
+ pub node_limit: Option<usize>,
+ /// Optional path prefix to exclude from search
+ #[serde(default)]
+ pub exclude_path: Option<String>,
+ /// Optional maximum depth relative to query root
+ #[serde(default)]
+ pub level_limit: Option<usize>,
+}
+
/// Health check response
#[derive(Debug, Serialize)]
pub struct HealthResponse {
@@ -115,10 +142,7 @@ pub struct MountInfo {
// ============================================================================
/// GET /api/v1/files - Read file
-pub async fn read_file(
- State(state): State<AppState>,
- Query(query): Query<FileQuery>,
-) -> Response {
+pub async fn read_file(State(state): State<AppState>, Query(query): Query<FileQuery>) -> Response {
match state.fs.read(&query.path, query.offset, query.size).await {
Ok(data) => (StatusCode::OK, data).into_response(),
Err(e) => (
@@ -198,10 +222,7 @@ pub async fn delete_file(
}
/// GET /api/v1/stat - Get file metadata
-pub async fn stat_file(
- State(state): State<AppState>,
- Query(query): Query<FileQuery>,
-) -> Response {
+pub async fn stat_file(State(state): State<AppState>, Query(query): Query<FileQuery>) -> Response {
match state.fs.stat(&query.path).await {
Ok(info) => (StatusCode::OK, Json(ApiResponse::success(info))).into_response(),
Err(e) => (
@@ -348,6 +369,40 @@ pub async fn unmount_filesystem(
// Health Check Handler
// ============================================================================
+/// POST /api/v1/grep - Search file content
+pub async fn grep_content(State(state): State<AppState>, Json(req): Json<GrepRequest>) -> Response {
+ if req.stream {
+ return (
+ StatusCode::NOT_IMPLEMENTED,
+ Json(ApiResponse::<()>::error(
+ "streaming grep is not supported by the ragfs HTTP server",
+ )),
+ )
+ .into_response();
+ }
+
+ match state
+ .fs
+ .grep(
+ &req.path,
+ &req.pattern,
+ req.recursive,
+ req.case_insensitive,
+ req.node_limit,
+ req.exclude_path.as_deref(),
+ req.level_limit,
+ )
+ .await
+ {
+ Ok(result) => (StatusCode::OK, Json(ApiResponse::success(result))).into_response(),
+ Err(e) => (
+ StatusCode::INTERNAL_SERVER_ERROR,
+ Json(ApiResponse::<GrepResult>::error(e.to_string())),
+ )
+ .into_response(),
+ }
+}
+
/// GET /api/v1/health - Health check
pub async fn health_check() -> Response {
let response = HealthResponse {
@@ -357,3 +412,59 @@ pub async fn health_check() -> Response {
(StatusCode::OK, Json(ApiResponse::success(response))).into_response()
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use axum::extract::State;
+ use std::collections::HashMap;
+
+ use crate::plugins::LocalFSPlugin;
+
+ #[tokio::test]
+ async fn test_grep_content_forwards_exclude_path_and_level_limit() {
+ let fs = Arc::new(MountableFS::new());
+ fs.register_plugin(LocalFSPlugin::new()).await;
+
+ let temp = tempfile::TempDir::new().unwrap();
+ std::fs::create_dir_all(temp.path().join("excluded")).unwrap();
+ std::fs::create_dir_all(temp.path().join("ok")).unwrap();
+ std::fs::write(temp.path().join("excluded/x.txt"), "hit\n").unwrap();
+ std::fs::write(temp.path().join("ok/y.txt"), "hit\n").unwrap();
+
+ fs.mount(PluginConfig {
+ name: "localfs".to_string(),
+ mount_path: "/local".to_string(),
+ params: HashMap::from([(
+ "local_dir".to_string(),
+ crate::core::ConfigValue::String(temp.path().to_string_lossy().to_string()),
+ )]),
+ })
+ .await
+ .unwrap();
+
+ let response = grep_content(
+ State(AppState { fs: fs.clone() }),
+ Json(GrepRequest {
+ path: "/local".to_string(),
+ pattern: "hit".to_string(),
+ recursive: true,
+ case_insensitive: false,
+ stream: false,
+ node_limit: Some(1),
+ exclude_path: Some("/local/excluded".to_string()),
+ level_limit: Some(5),
+ }),
+ )
+ .await;
+
+ let body = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let parsed: serde_json::Value = serde_json::from_slice(&body).unwrap();
+
+ assert_eq!(parsed["success"], true);
+ assert_eq!(parsed["data"]["count"], 1);
+ assert_eq!(parsed["data"]["matches"][0]["file"], "ok/y.txt");
+ }
+}
diff --git a/crates/ragfs/src/server/main.rs b/crates/ragfs/src/server/main.rs
index 2bd992539..3af9824d2 100644
--- a/crates/ragfs/src/server/main.rs
+++ b/crates/ragfs/src/server/main.rs
@@ -72,6 +72,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing::info!(" GET /api/v1/mounts");
tracing::info!(" POST /api/v1/mount");
tracing::info!(" POST /api/v1/unmount");
+ tracing::info!(" POST /api/v1/grep");
tracing::info!("");
tracing::info!("Example: Mount MemFS");
tracing::info!(" curl -X POST http://{}/api/v1/mount \\", addr);
diff --git a/crates/ragfs/src/server/router.rs b/crates/ragfs/src/server/router.rs
index 2d140dde8..314eec935 100644
--- a/crates/ragfs/src/server/router.rs
+++ b/crates/ragfs/src/server/router.rs
@@ -13,8 +13,8 @@ use tower_http::{
use tracing::Level;
use super::handlers::{
- create_directory, create_file, delete_file, health_check, list_directory, list_mounts,
- mount_filesystem, read_file, stat_file, unmount_filesystem, write_file, AppState,
+ create_directory, create_file, delete_file, grep_content, health_check, list_directory,
+ list_mounts, mount_filesystem, read_file, stat_file, unmount_filesystem, write_file, AppState,
};
/// Create the main application router
@@ -33,12 +33,12 @@ pub fn create_router(state: AppState, enable_cors: bool) -> Router {
.route("/mounts", get(list_mounts))
.route("/mount", post(mount_filesystem))
.route("/unmount", post(unmount_filesystem))
+ // Search operations
+ .route("/grep", post(grep_content))
// Health check
.route("/health", get(health_check));
- let app = Router::new()
- .nest("/api/v1", api_routes)
- .with_state(state);
+ let app = Router::new().nest("/api/v1", api_routes).with_state(state);
// Add tracing middleware
let app = app.layer(
diff --git a/openviking/pyagfs/client.py b/openviking/pyagfs/client.py
index d42817c5c..645368068 100644
--- a/openviking/pyagfs/client.py
+++ b/openviking/pyagfs/client.py
@@ -506,6 +506,8 @@ class AGFSClient:
case_insensitive: bool = False,
stream: bool = False,
node_limit: Optional[int] = None,
+ exclude_path: Optional[str] = None,
+ level_limit: Optional[int] = None,
):
"""Search for a pattern in files using regular expressions
@@ -516,6 +518,8 @@ class AGFSClient:
case_insensitive: Whether to perform case-insensitive matching (default: False)
stream: Whether to stream results as NDJSON (default: False)
node_limit: Maximum number of results to return (default: None)
+ exclude_path: Optional path prefix to exclude from search (default: None)
+ level_limit: Optional maximum depth relative to query root (default: None)
Returns:
If stream=False: Dict with 'matches' (list of match objects) and 'count'
@@ -543,6 +547,10 @@ class AGFSClient:
}
if node_limit is not None:
json_payload["node_limit"] = node_limit
+ if exclude_path is not None:
+ json_payload["exclude_path"] = exclude_path
+ if level_limit is not None:
+ json_payload["level_limit"] = level_limit
response = self.session.post(
f"{self.api_base}/grep",
json=json_payload,
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 112a07341..392e5a0e4 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -30,7 +30,12 @@ from openviking.core.namespace import (
from openviking.core.namespace import (
is_accessible as namespace_is_accessible,
)
-from openviking.pyagfs.exceptions import AGFSClientError, AGFSDirectoryNotEmptyError, AGFSHTTPError
+from openviking.pyagfs.exceptions import (
+ AGFSClientError,
+ AGFSDirectoryNotEmptyError,
+ AGFSHTTPError,
+ AGFSNotSupportedError,
+)
from openviking.resource.watch_storage import is_watch_task_control_uri
from openviking.server.error_mapping import is_not_found_error, map_exception
from openviking.server.identity import RequestContext, Role
@@ -619,7 +624,8 @@ class VikingFS:
) -> Dict:
"""Content search by pattern or keywords.
- Grep search implemented at VikingFS layer, supports encrypted files.
+ Optimized implementation that uses agfs native grep when possible.
+ Falls back to VikingFS layer implementation for encrypted files.
Args:
uri: Viking URI
@@ -629,10 +635,171 @@ class VikingFS:
node_limit: Maximum number of results to return
level_limit: Maximum depth level to traverse (default: 5)
ctx: Request context
+
+ Returns:
+ Dict with matches, count, match_count, files_scanned
"""
self._ensure_access(uri, ctx)
await self.stat(uri, ctx=ctx)
+ if self._encryptor:
+ return await self._grep_encrypted(
+ uri=uri,
+ pattern=pattern,
+ exclude_uri=exclude_uri,
+ case_insensitive=case_insensitive,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ )
+
+ try:
+ return await self._grep_with_agfs(
+ uri=uri,
+ pattern=pattern,
+ exclude_uri=exclude_uri,
+ case_insensitive=case_insensitive,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ )
+ except (AttributeError, AGFSNotSupportedError, NotImplementedError) as e:
+ logger.debug(f"agfs grep unavailable, falling back to VikingFS implementation: {e}")
+
+ return await self._grep_encrypted(
+ uri=uri,
+ pattern=pattern,
+ exclude_uri=exclude_uri,
+ case_insensitive=case_insensitive,
+ node_limit=node_limit,
+ level_limit=level_limit,
+ ctx=ctx,
+ )
+
+ async def _grep_with_agfs(
+ self,
+ uri: str,
+ pattern: str,
+ exclude_uri: Optional[str] = None,
+ case_insensitive: bool = False,
+ node_limit: Optional[int] = None,
+ level_limit: int = 5,
+ ctx: Optional[RequestContext] = None,
+ ) -> Dict:
+ """Grep using agfs native implementation.
+
+ This is the optimized path for non-encrypted files.
+ Uses agfs.grep() which performs matching on the server side.
+
+ Prefer pushing filters down to agfs backend:
+ - exclude_uri -> exclude_path
+ - level_limit -> level_limit
+
+ Args:
+ uri: Viking URI
+ pattern: Regular expression pattern to search for
+ exclude_uri: Optional URI prefix to exclude from search
+ case_insensitive: Whether to perform case-insensitive matching
+ node_limit: Maximum number of results to return
+ level_limit: Maximum depth level to traverse
+ ctx: Request context
+
+ Returns:
+ Dict with matches, count, match_count, files_scanned
+ """
+ path = self._uri_to_path(uri, ctx=ctx)
+
+ excluded_path = None
+ if exclude_uri:
+ normalized_excluded_uri = self._normalize_uri(exclude_uri).rstrip("/")
+ self._ensure_access(normalized_excluded_uri, ctx)
+ excluded_path = self._uri_to_path(normalized_excluded_uri, ctx=ctx)
+
+ try:
+ result = await asyncio.to_thread(
+ self.agfs.grep,
+ path=path,
+ pattern=pattern,
+ recursive=True,
+ case_insensitive=case_insensitive,
+ stream=False,
+ node_limit=node_limit,
+ exclude_path=excluded_path,
+ level_limit=level_limit,
+ )
+ except (AttributeError, AGFSNotSupportedError, NotImplementedError):
+ # Capability missing: let the outer caller fall back to the VikingFS implementation.
+ logger.warning("agfs grep unavailable, falling back to VikingFS implementation")
+ raise
+
+ matches = result.get("matches", [])
+ results = []
+ files_scanned_set = set()
+
+ for match in matches:
+ match_file = match.get("file", "")
+ if not match_file:
+ continue
+
+ agfs_file_path = self._resolve_grep_match_agfs_path(path, match_file)
+
+ file_uri = self._path_to_uri(agfs_file_path, ctx=ctx)
+
+ files_scanned_set.add(file_uri)
+
+ results.append(
+ {
+ "line": match.get("line", match.get("line_number", 0)),
+ "uri": file_uri,
+ "content": match.get("content", ""),
+ }
+ )
+
+ if node_limit and len(results) >= node_limit:
+ break
+
+ # Prefer backend-provided scanned file count if available; otherwise fall back to
+ # counting files that produced at least one match (best-effort).
+ backend_files_scanned = result.get("files_scanned")
+ if isinstance(backend_files_scanned, int) and backend_files_scanned >= 0:
+ files_scanned = backend_files_scanned
+ else:
+ files_scanned = len(files_scanned_set)
+
+ return {
+ "matches": results,
+ "count": len(results),
+ "match_count": len(results),
+ "files_scanned": files_scanned,
+ }
+
+ async def _grep_encrypted(
+ self,
+ uri: str,
+ pattern: str,
+ exclude_uri: Optional[str] = None,
+ case_insensitive: bool = False,
+ node_limit: Optional[int] = None,
+ level_limit: int = 5,
+ ctx: Optional[RequestContext] = None,
+ ) -> Dict:
+ """Grep implementation for encrypted files.
+
+ This implementation decrypts files at VikingFS layer before matching.
+ Used when encryption is enabled or when agfs.grep is not available.
+
+ Args:
+ uri: Viking URI
+ pattern: Regular expression pattern to search for
+ exclude_uri: Optional URI prefix to exclude from search
+ case_insensitive: Whether to perform case-insensitive matching
+ node_limit: Maximum number of results to return
+ level_limit: Maximum depth level to traverse (default: 5)
+ ctx: Request context
+
+ Returns:
+ Dict with matches, count, match_count, files_scanned
+ """
flags = re.IGNORECASE if case_insensitive else 0
compiled_pattern = re.compile(pattern, flags)
excluded_prefix = None
@@ -761,6 +928,18 @@ class VikingFS:
logger.debug(f"Failed to grep {entry_uri}: {e}")
return [], 1
+ def _resolve_grep_match_agfs_path(self, base_path: str, match_file: str) -> str:
+ """Resolve a grep match path (relative to query root) into a full AGFS path."""
+ if match_file == ".":
+ return base_path
+ return f"{base_path.rstrip('/')}/{match_file.lstrip('/')}"
+
+ def _calculate_grep_match_depth(self, match_file: str) -> int:
+ """Calculate relative depth from a grep result path relative to the query root."""
+ if not match_file or match_file == ".":
+ return 0
+ return len([part for part in match_file.split("/") if part])
+
async def stat(self, uri: str, ctx: Optional[RequestContext] = None) -> Dict[str, Any]:
"""
File/directory information.
@@ -2170,6 +2349,7 @@ class VikingFS:
ctx: Optional[RequestContext] = None,
) -> None:
"""Write context to AGFS (L0/L1/L2)."""
+
self._ensure_access(uri, ctx)
path = self._uri_to_path(uri, ctx=ctx)
diff --git a/tests/misc/test_vikingfs_uri_guard.py b/tests/misc/test_vikingfs_uri_guard.py
index 6536cd533..febdba1c2 100644
--- a/tests/misc/test_vikingfs_uri_guard.py
+++ b/tests/misc/test_vikingfs_uri_guard.py
@@ -7,7 +7,9 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
+from openviking.pyagfs.exceptions import AGFSInvalidOperationError
from openviking.storage.viking_fs import VikingFS
+from openviking_cli.exceptions import PermissionDeniedError
def _make_viking_fs() -> VikingFS:
@@ -17,6 +19,7 @@ def _make_viking_fs() -> VikingFS:
fs.query_embedder = None
fs.rerank_config = None
fs.vector_store = None
+ fs._encryptor = None
fs._bound_ctx = contextvars.ContextVar("vikingfs_bound_ctx_test", default=None)
return fs
@@ -24,6 +27,24 @@ def _make_viking_fs() -> VikingFS:
class TestVikingFSURITraversalGuard:
"""Traversal-style URI components should be rejected before any AGFS I/O."""
+ @pytest.mark.parametrize(
+ ("base_path", "match_file", "expected"),
+ [
+ ("/local/default/resources/docs", ".", "/local/default/resources/docs"),
+ (
+ "/local/default/resources/docs",
+ "sub/a.md",
+ "/local/default/resources/docs/sub/a.md",
+ ),
+ ],
+ )
+ def test_resolve_grep_match_agfs_path(
+ self, base_path: str, match_file: str, expected: str
+ ) -> None:
+ fs = _make_viking_fs()
+
+ assert fs._resolve_grep_match_agfs_path(base_path, match_file) == expected
+
@pytest.mark.parametrize(
"uri",
[
@@ -37,14 +58,14 @@ class TestVikingFSURITraversalGuard:
def test_rejects_unsafe_uri_components(self, uri: str) -> None:
fs = _make_viking_fs()
- with pytest.raises(PermissionError, match="Unsafe URI"):
+ with pytest.raises(PermissionDeniedError, match="Unsafe URI"):
fs._normalized_uri_parts(uri)
@pytest.mark.asyncio
async def test_read_file_rejects_traversal_before_agfs_read(self) -> None:
fs = _make_viking_fs()
- with pytest.raises(PermissionError, match="Unsafe URI"):
+ with pytest.raises(PermissionDeniedError, match="Unsafe URI"):
await fs.read_file("viking://resources/../_system/users.json")
fs.agfs.read.assert_not_called()
@@ -53,7 +74,7 @@ class TestVikingFSURITraversalGuard:
async def test_write_rejects_traversal_before_agfs_write(self) -> None:
fs = _make_viking_fs()
- with pytest.raises(PermissionError, match="Unsafe URI"):
+ with pytest.raises(PermissionDeniedError, match="Unsafe URI"):
await fs.write("viking://resources/../../_system/accounts.json", "pwned")
fs.agfs.write.assert_not_called()
@@ -64,7 +85,7 @@ class TestVikingFSURITraversalGuard:
fs._collect_uris = AsyncMock(return_value=[])
fs._delete_from_vector_store = AsyncMock()
- with pytest.raises(PermissionError, match="Unsafe URI"):
+ with pytest.raises(PermissionDeniedError, match="Unsafe URI"):
await fs.rm("viking://resources/../../other_account/_system/users.json")
fs._collect_uris.assert_not_called()
@@ -87,7 +108,7 @@ class TestVikingFSURITraversalGuard:
fs._update_vector_store_uris = AsyncMock()
fs._delete_from_vector_store = AsyncMock()
- with pytest.raises(PermissionError, match="Unsafe URI"):
+ with pytest.raises(PermissionDeniedError, match="Unsafe URI"):
await fs.mv(old_uri, new_uri)
fs._collect_uris.assert_not_called()
@@ -98,9 +119,78 @@ class TestVikingFSURITraversalGuard:
@pytest.mark.asyncio
async def test_read_file_keeps_valid_uri_behavior(self) -> None:
fs = _make_viking_fs()
+ fs.agfs.stat = MagicMock(return_value=MagicMock())
fs.agfs.read = MagicMock(return_value=b"hello")
content = await fs.read_file("viking://resources/docs/guide.md")
-
assert content == "hello"
+ fs.agfs.stat.assert_called_once_with("/local/default/resources/docs/guide.md")
fs.agfs.read.assert_called_once_with("/local/default/resources/docs/guide.md")
+
+ @pytest.mark.asyncio
+ async def test_grep_propagates_agfs_errors_instead_of_falling_back(self) -> None:
+ fs = _make_viking_fs()
+ fs._encryptor = None
+ fs._ensure_access = MagicMock()
+ fs._grep_with_agfs = AsyncMock(side_effect=AGFSInvalidOperationError("invalid regex"))
+ fs._grep_encrypted = AsyncMock(
+ return_value={"matches": [], "count": 0, "match_count": 0, "files_scanned": 0}
+ )
+
+ with pytest.raises(AGFSInvalidOperationError, match="invalid regex"):
+ await fs.grep("viking://resources/docs", "(")
+
+ fs._grep_with_agfs.assert_awaited_once()
+ fs._grep_encrypted.assert_not_awaited()
+
+ @pytest.mark.asyncio
+ async def test_grep_with_agfs_maps_query_root_relative_match_to_uri(self) -> None:
+ """Query-root-relative grep matches should be reconstructed into the final Viking URI."""
+ fs = _make_viking_fs()
+ fs._encryptor = None
+ fs._ensure_access = MagicMock()
+ fs.agfs.grep = MagicMock(
+ return_value={
+ "matches": [
+ {
+ "file": "sub/a.txt",
+ "line_number": 3,
+ "content": "act",
+ }
+ ],
+ "count": 1,
+ }
+ )
+
+ result = await fs._grep_with_agfs("viking://resources/test-root", "act")
+
+ assert result["count"] == 1
+ assert result["matches"][0]["uri"] == "viking://resources/test-root/sub/a.txt"
+ assert result["matches"][0]["line"] == 3
+ assert result["matches"][0]["content"] == "act"
+
+ @pytest.mark.asyncio
+ async def test_grep_with_agfs_maps_dot_match_to_query_root_uri(self) -> None:
+ """A '.' grep match should resolve back to the queried Viking URI itself."""
+ fs = _make_viking_fs()
+ fs._encryptor = None
+ fs._ensure_access = MagicMock()
+ fs.agfs.grep = MagicMock(
+ return_value={
+ "matches": [
+ {
+ "file": ".",
+ "line_number": 1,
+ "content": "act",
+ }
+ ],
+ "count": 1,
+ }
+ )
+
+ result = await fs._grep_with_agfs("viking://resources/test-root", "act")
+
+ assert result["count"] == 1
+ assert result["matches"][0]["uri"] == "viking://resources/test-root"
+ assert result["matches"][0]["line"] == 1
+ assert result["matches"][0]["content"] == "act"
diff --git a/tests/server/test_api_search.py b/tests/server/test_api_search.py
index 3781be13a..888da6827 100644
--- a/tests/server/test_api_search.py
+++ b/tests/server/test_api_search.py
@@ -422,6 +422,49 @@ async def test_grep_missing_uri_returns_not_found(client: httpx.AsyncClient):
assert resp.json()["status"] == "error"
+async def test_grep_level_limit_filters_by_relative_match_path(
+ client: httpx.AsyncClient,
+ upload_temp_dir,
+):
+ root_file = upload_temp_dir / "root_level.md"
+ root_file.write_text("OpenViking on root level\n")
+ deep_file = upload_temp_dir / "deep_level.md"
+ deep_file.write_text("OpenViking in deep level\n")
+
+ await client.post(
+ "/api/v1/resources",
+ json={
+ "temp_file_id": root_file.name,
+ "to": "viking://resources/level-limit/root_level.md",
+ "reason": "test",
+ },
+ )
+ await client.post(
+ "/api/v1/resources",
+ json={
+ "temp_file_id": deep_file.name,
+ "to": "viking://resources/level-limit/nested/deeper/deep_level.md",
+ "reason": "test",
+ },
+ )
+
+ resp = await client.post(
+ "/api/v1/search/grep",
+ json={
+ "uri": "viking://resources/level-limit",
+ "pattern": "OpenViking",
+ "level_limit": 2,
+ },
+ )
+
+ assert resp.status_code == 200
+ body = resp.json()
+ assert body["status"] == "ok"
+ uris = {m["uri"] for m in body["result"]["matches"]}
+ assert "viking://resources/level-limit/root_level.md/root_level.md" in uris
+ assert "viking://resources/level-limit/nested/deeper/deep_level.md/deep_level.md" not in uris
+
+
async def test_grep_exclude_uri_excludes_specific_uri_range(
client: httpx.AsyncClient,
upload_temp_dir,