Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: parallelize grep reads for remote storage (#1761)

Jiahui Zhoub9753385ec2d
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rs
index 18e500564..bd239d576 100644
--- a/crates/ragfs-python/src/lib.rs
+++ b/crates/ragfs-python/src/lib.rs
@@ -18,6 +18,14 @@ use ragfs::plugins::{
KVFSPlugin, LocalFSPlugin, MemFSPlugin, QueueFSPlugin, SQLFSPlugin, ServerInfoFSPlugin,
};
+fn py_detach_blocking<T, F>(py: Python<'_>, f: F) -> T
+where
+ T: Send,
+ F: Send + FnOnce() -> T,
+{
+ py.detach(f)
+}
+
/// Get a Python exception class from the pyagfs module
fn get_exception<'py>(py: Python<'py>, name: &str) -> PyResult<Bound<'py, PyType>> {
let pyagfs = PyModule::import(py, "openviking.pyagfs")?;
@@ -220,10 +228,10 @@ impl RAGFSBindingClient {
/// name, size, mode, modTime, isDir
fn ls(&self, path: String) -> PyResult<Py<PyAny>> {
let fs = self.fs.clone();
- let entries = self
- .rt
- .block_on(async move { fs.read_dir(&path).await })
- .map_err(to_py_err)?;
+ let entries = Python::attach(|py| {
+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.read_dir(&path).await }))
+ })
+ .map_err(to_py_err)?;
Python::attach(|py| {
let list = PyList::empty(py);
@@ -254,10 +262,10 @@ impl RAGFSBindingClient {
let off = if offset < 0 { 0u64 } else { offset as u64 };
let sz = if size < 0 { 0u64 } else { size as u64 };
- let data = self
- .rt
- .block_on(async move { fs.read(&path, off, sz).await })
- .map_err(to_py_err)?;
+ let data = Python::attach(|py| {
+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.read(&path, off, sz).await }))
+ })
+ .map_err(to_py_err)?;
Python::attach(|py| Ok(PyBytes::new(py, &data).into()))
}
@@ -278,9 +286,13 @@ impl RAGFSBindingClient {
let _ = max_retries; // not applicable for local binding
let fs = self.fs.clone();
let len = data.len();
- self.rt
- .block_on(async move { fs.write(&path, &data, 0, WriteFlag::Create).await })
- .map_err(to_py_err)?;
+ Python::attach(|py| {
+ py_detach_blocking(py, move || {
+ self.rt
+ .block_on(async move { fs.write(&path, &data, 0, WriteFlag::Create).await })
+ })
+ })
+ .map_err(to_py_err)?;
Ok(format!("Written {} bytes", len))
}
@@ -288,9 +300,10 @@ impl RAGFSBindingClient {
/// Create a new empty file.
fn create(&self, path: String) -> PyResult<HashMap<String, String>> {
let fs = self.fs.clone();
- self.rt
- .block_on(async move { fs.create(&path).await })
- .map_err(to_py_err)?;
+ Python::attach(|py| {
+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.create(&path).await }))
+ })
+ .map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "created".to_string());
@@ -304,9 +317,10 @@ impl RAGFSBindingClient {
.map_err(|e| PyRuntimeError::new_err(format!("Invalid mode '{}': {}", mode, e)))?;
let fs = self.fs.clone();
- self.rt
- .block_on(async move { fs.mkdir(&path, mode_int).await })
- .map_err(to_py_err)?;
+ Python::attach(|py| {
+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.mkdir(&path, mode_int).await }))
+ })
+ .map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "created".to_string());
@@ -317,15 +331,18 @@ impl RAGFSBindingClient {
#[pyo3(signature = (path, recursive=false))]
fn rm(&self, path: String, recursive: bool) -> PyResult<HashMap<String, String>> {
let fs = self.fs.clone();
- self.rt
- .block_on(async move {
- if recursive {
- fs.remove_all(&path).await
- } else {
- fs.remove(&path).await
- }
+ Python::attach(|py| {
+ py_detach_blocking(py, move || {
+ self.rt.block_on(async move {
+ if recursive {
+ fs.remove_all(&path).await
+ } else {
+ fs.remove(&path).await
+ }
+ })
})
- .map_err(to_py_err)?;
+ })
+ .map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "deleted".to_string());
@@ -335,10 +352,10 @@ impl RAGFSBindingClient {
/// Get file/directory information.
fn stat(&self, path: String) -> PyResult<Py<PyAny>> {
let fs = self.fs.clone();
- let info = self
- .rt
- .block_on(async move { fs.stat(&path).await })
- .map_err(to_py_err)?;
+ let info = Python::attach(|py| {
+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.stat(&path).await }))
+ })
+ .map_err(to_py_err)?;
Python::attach(|py| {
let dict = file_info_to_py_dict(py, &info)?;
@@ -349,9 +366,13 @@ impl RAGFSBindingClient {
/// Rename/move a file or directory.
fn mv(&self, old_path: String, new_path: String) -> PyResult<HashMap<String, String>> {
let fs = self.fs.clone();
- self.rt
- .block_on(async move { fs.rename(&old_path, &new_path).await })
- .map_err(to_py_err)?;
+ Python::attach(|py| {
+ py_detach_blocking(py, move || {
+ self.rt
+ .block_on(async move { fs.rename(&old_path, &new_path).await })
+ })
+ })
+ .map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "renamed".to_string());
@@ -361,9 +382,10 @@ impl RAGFSBindingClient {
/// Change file permissions.
fn chmod(&self, path: String, mode: u32) -> PyResult<HashMap<String, String>> {
let fs = self.fs.clone();
- self.rt
- .block_on(async move { fs.chmod(&path, mode).await })
- .map_err(to_py_err)?;
+ Python::attach(|py| {
+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.chmod(&path, mode).await }))
+ })
+ .map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "chmod ok".to_string());
@@ -373,18 +395,21 @@ impl RAGFSBindingClient {
/// Touch a file (create if not exists, or update timestamp).
fn touch(&self, path: String) -> PyResult<HashMap<String, String>> {
let fs = self.fs.clone();
- self.rt
- .block_on(async move {
- // Try create; if already exists, write empty to update mtime
- match fs.create(&path).await {
- Ok(_) => Ok(()),
- Err(_) => {
- // File exists, write empty bytes to update timestamp
- fs.write(&path, &[], 0, WriteFlag::None).await.map(|_| ())
+ Python::attach(|py| {
+ py_detach_blocking(py, move || {
+ self.rt.block_on(async move {
+ // Try create; if already exists, write empty to update mtime
+ match fs.create(&path).await {
+ Ok(_) => Ok(()),
+ Err(_) => {
+ // File exists, write empty bytes to update timestamp
+ fs.write(&path, &[], 0, WriteFlag::None).await.map(|_| ())
+ }
}
- }
+ })
})
- .map_err(to_py_err)?;
+ })
+ .map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), "touched".to_string());
@@ -394,7 +419,9 @@ impl RAGFSBindingClient {
/// List all mounted plugins.
fn mounts(&self) -> PyResult<Vec<HashMap<String, String>>> {
let fs = self.fs.clone();
- let mount_list = self.rt.block_on(async move { fs.list_mounts().await });
+ let mount_list = Python::attach(|py| {
+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.list_mounts().await }))
+ });
let result: Vec<HashMap<String, String>> = mount_list
.into_iter()
@@ -434,9 +461,10 @@ impl RAGFSBindingClient {
};
let fs = self.fs.clone();
- self.rt
- .block_on(async move { fs.mount(plugin_config).await })
- .map_err(to_py_err)?;
+ Python::attach(|py| {
+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.mount(plugin_config).await }))
+ })
+ .map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert(
@@ -450,9 +478,10 @@ impl RAGFSBindingClient {
fn unmount(&self, path: String) -> PyResult<HashMap<String, String>> {
let fs = self.fs.clone();
let path_clone = path.clone();
- self.rt
- .block_on(async move { fs.unmount(&path_clone).await })
- .map_err(to_py_err)?;
+ Python::attach(|py| {
+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.unmount(&path_clone).await }))
+ })
+ .map_err(to_py_err)?;
let mut m = HashMap::new();
m.insert("message".to_string(), format!("unmounted {}", path));
@@ -534,3 +563,16 @@ fn ragfs_python(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<RAGFSBindingClient>()?;
Ok(())
}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn detach_blocking_helper_runs_without_python_objects() {
+ Python::attach(|py| {
+ let value: i32 = py_detach_blocking(py, || 40 + 2);
+ assert_eq!(value, 42);
+ });
+ }
+}
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index c758eab3a..fe932f322 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -53,6 +53,8 @@ if TYPE_CHECKING:
logger = get_logger(__name__)
+_DEFAULT_GREP_FILE_CONCURRENCY = 32
+
def _ensure_non_empty_search_query(query: str) -> None:
if not query.strip():
@@ -632,14 +634,36 @@ class VikingFS:
if exclude_uri:
excluded_prefix = self._normalize_uri(exclude_uri).rstrip("/")
self._ensure_access(excluded_prefix, ctx)
+ file_uris = await self._collect_grep_files(
+ uri,
+ excluded_prefix=excluded_prefix,
+ level_limit=level_limit,
+ ctx=ctx,
+ )
+ results, files_scanned = await self._grep_files_parallel(
+ file_uris,
+ compiled_pattern=compiled_pattern,
+ node_limit=node_limit,
+ ctx=ctx,
+ )
- results = []
- files_scanned = 0
+ return {
+ "matches": results,
+ "count": len(results),
+ "match_count": len(results),
+ "files_scanned": files_scanned,
+ }
- async def search_recursive(current_uri: str, current_depth: int):
- if node_limit and len(results) >= node_limit:
- return
+ async def _collect_grep_files(
+ self,
+ uri: str,
+ excluded_prefix: Optional[str],
+ level_limit: int,
+ ctx: Optional[RequestContext] = None,
+ ) -> List[str]:
+ file_uris: List[str] = []
+ async def search_recursive(current_uri: str, current_depth: int) -> None:
if current_depth > level_limit:
return
@@ -657,9 +681,6 @@ class VikingFS:
return
for entry in entries:
- if node_limit and len(results) >= node_limit:
- break
-
entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}"
if excluded_prefix and (
entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/")
@@ -670,36 +691,70 @@ class VikingFS:
if entry.get("isDir"):
await search_recursive(entry_uri, current_depth + 1)
else:
- nonlocal files_scanned
- files_scanned += 1
- try:
- content = await self.read(entry_uri, ctx=ctx)
- if isinstance(content, bytes):
- content = content.decode("utf-8", errors="replace")
-
- lines = content.split("\n")
- for line_num, line in enumerate(lines, 1):
- if compiled_pattern.search(line):
- results.append(
- {
- "line": line_num,
- "uri": entry_uri,
- "content": line,
- }
- )
- if node_limit and len(results) >= node_limit:
- break
- except Exception as e:
- logger.debug(f"Failed to grep {entry_uri}: {e}")
+ file_uris.append(entry_uri)
await search_recursive(uri, 0)
+ return file_uris
- return {
- "matches": results,
- "count": len(results),
- "match_count": len(results),
- "files_scanned": files_scanned,
- }
+ async def _grep_files_parallel(
+ self,
+ file_uris: List[str],
+ compiled_pattern: re.Pattern,
+ node_limit: Optional[int],
+ ctx: Optional[RequestContext] = None,
+ ) -> tuple[List[Dict[str, Any]], int]:
+ results: List[Dict[str, Any]] = []
+ files_scanned = 0
+ for start in range(0, len(file_uris), _DEFAULT_GREP_FILE_CONCURRENCY):
+ batch_uris = file_uris[start : start + _DEFAULT_GREP_FILE_CONCURRENCY]
+ batch_jobs = [
+ asyncio.to_thread(self._grep_single_file, entry_uri, compiled_pattern, ctx)
+ for entry_uri in batch_uris
+ ]
+ batch_results = await asyncio.gather(*batch_jobs)
+ for matches, scanned_count in batch_results:
+ files_scanned += scanned_count
+ for match in matches:
+ results.append(match)
+ if node_limit and len(results) >= node_limit:
+ return results, files_scanned
+
+ return results, files_scanned
+
+ def _grep_single_file(
+ self,
+ entry_uri: str,
+ compiled_pattern: re.Pattern,
+ ctx: Optional[RequestContext] = None,
+ ) -> tuple[List[Dict[str, Any]], int]:
+ try:
+ self._ensure_access(entry_uri, ctx)
+ path = self._uri_to_path(entry_uri, ctx=ctx)
+ result = self.agfs.read(path, 0, -1)
+ if isinstance(result, bytes):
+ content = result
+ elif result is not None and hasattr(result, "content"):
+ content = result.content
+ else:
+ content = b""
+ if isinstance(content, bytes):
+ content = content.decode("utf-8", errors="replace")
+
+ matches: List[Dict[str, Any]] = []
+ lines = content.split("\n")
+ for line_num, line in enumerate(lines, 1):
+ if compiled_pattern.search(line):
+ matches.append(
+ {
+ "line": line_num,
+ "uri": entry_uri,
+ "content": line,
+ }
+ )
+ return matches, 1
+ except Exception as e:
+ logger.debug(f"Failed to grep {entry_uri}: {e}")
+ return [], 1
async def stat(self, uri: str, ctx: Optional[RequestContext] = None) -> Dict[str, Any]:
"""
diff --git a/tests/storage/test_viking_fs_grep.py b/tests/storage/test_viking_fs_grep.py
new file mode 100644
index 000000000..d6aed7a13
--- /dev/null
+++ b/tests/storage/test_viking_fs_grep.py
@@ -0,0 +1,194 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+import time
+
+import pytest
+
+import openviking.storage.viking_fs as viking_fs_module
+from openviking.storage.viking_fs import _DEFAULT_GREP_FILE_CONCURRENCY, VikingFS
+
+
+class _DummyAgfs:
+ pass
+
+
+async def test_grep_preserves_dfs_order_and_node_limit(monkeypatch):
+ fs = VikingFS(agfs=_DummyAgfs())
+
+ async def fake_stat(uri, ctx=None):
+ return {"isDir": True}
+
+ async def fake_ls(uri, ctx=None, **kwargs):
+ entries = {
+ "viking://resources": [
+ {"name": "dir_a", "isDir": True},
+ {"name": "dir_b", "isDir": True},
+ ],
+ "viking://resources/dir_a": [
+ {"name": "a1.md", "isDir": False},
+ {"name": "a2.md", "isDir": False},
+ ],
+ "viking://resources/dir_b": [
+ {"name": "b1.md", "isDir": False},
+ ],
+ }
+ return entries.get(uri, [])
+
+ def fake_agfs_read(path, offset, size):
+ contents = {
+ "/resources/dir_a/a1.md": "match a1 line1\nskip\nmatch a1 line3",
+ "/resources/dir_a/a2.md": "match a2 line1",
+ "/resources/dir_b/b1.md": "match b1 line1",
+ }
+ return contents[path].encode()
+
+ monkeypatch.setattr(fs, "stat", fake_stat)
+ monkeypatch.setattr(fs, "ls", fake_ls)
+ monkeypatch.setattr(
+ fs,
+ "_uri_to_path",
+ lambda uri, ctx=None: uri.replace("viking://", "/"),
+ )
+ monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False)
+
+ result = await fs.grep("viking://resources", pattern="match", node_limit=3)
+
+ assert result["count"] == 3
+ assert result["files_scanned"] == 2
+ assert result["matches"] == [
+ {
+ "line": 1,
+ "uri": "viking://resources/dir_a/a1.md",
+ "content": "match a1 line1",
+ },
+ {
+ "line": 3,
+ "uri": "viking://resources/dir_a/a1.md",
+ "content": "match a1 line3",
+ },
+ {
+ "line": 1,
+ "uri": "viking://resources/dir_a/a2.md",
+ "content": "match a2 line1",
+ },
+ ]
+
+
+async def test_grep_parallel_reads_respect_concurrency_limit(monkeypatch):
+ fs = VikingFS(agfs=_DummyAgfs())
+
+ async def fake_stat(uri, ctx=None):
+ return {"isDir": True}
+
+ async def fake_ls(uri, ctx=None, **kwargs):
+ entries = {
+ "viking://resources": [{"name": f"file{i}.md", "isDir": False} for i in range(12)]
+ }
+ return entries.get(uri, [])
+
+ active_reads = 0
+ max_active_reads = 0
+
+ def fake_agfs_read(path, offset, size):
+ nonlocal active_reads, max_active_reads
+ active_reads += 1
+ max_active_reads = max(max_active_reads, active_reads)
+ time.sleep(0.01)
+ active_reads -= 1
+ return f"match from {path}".encode()
+
+ monkeypatch.setattr(fs, "stat", fake_stat)
+ monkeypatch.setattr(fs, "ls", fake_ls)
+ monkeypatch.setattr(
+ fs,
+ "_uri_to_path",
+ lambda uri, ctx=None: uri.replace("viking://", "/"),
+ )
+ monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False)
+
+ result = await fs.grep("viking://resources", pattern="match")
+
+ assert result["count"] == 12
+ assert result["files_scanned"] == 12
+ assert max_active_reads > 1
+ assert max_active_reads <= min(12, _DEFAULT_GREP_FILE_CONCURRENCY)
+
+
+async def test_grep_parallel_reads_work_with_blocking_agfs_read(monkeypatch):
+ fs = VikingFS(agfs=_DummyAgfs())
+
+ async def fake_stat(uri, ctx=None):
+ return {"isDir": True}
+
+ async def fake_ls(uri, ctx=None, **kwargs):
+ if uri == "viking://resources":
+ return [{"name": f"file{i}.md", "isDir": False} for i in range(8)]
+ return []
+
+ def fake_agfs_read(path, offset, size):
+ time.sleep(0.05)
+ return f"match from {path}".encode()
+
+ monkeypatch.setattr(fs, "stat", fake_stat)
+ monkeypatch.setattr(fs, "ls", fake_ls)
+ monkeypatch.setattr(
+ fs,
+ "_uri_to_path",
+ lambda uri, ctx=None: uri.replace("viking://", "/"),
+ )
+ monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False)
+
+ started = time.perf_counter()
+ result = await fs.grep("viking://resources", pattern="match")
+ elapsed = time.perf_counter() - started
+
+ assert result["count"] == 8
+ assert result["files_scanned"] == 8
+ assert elapsed < 0.30
+
+
+async def test_grep_stops_scheduling_later_batches_after_node_limit(monkeypatch):
+ fs = VikingFS(agfs=_DummyAgfs())
+
+ async def fake_stat(uri, ctx=None):
+ return {"isDir": True}
+
+ async def fake_ls(uri, ctx=None, **kwargs):
+ if uri == "viking://resources":
+ return [{"name": f"file{i}.md", "isDir": False} for i in range(6)]
+ return []
+
+ read_paths = []
+
+ def fake_agfs_read(path, offset, size):
+ read_paths.append(path)
+ contents = {
+ "/resources/file0.md": "match file0 line1\nmatch file0 line2",
+ "/resources/file1.md": "match file1 line1",
+ "/resources/file2.md": "match file2 line1",
+ "/resources/file3.md": "match file3 line1",
+ "/resources/file4.md": "match file4 line1",
+ "/resources/file5.md": "match file5 line1",
+ }
+ return contents[path].encode()
+
+ monkeypatch.setattr(fs, "stat", fake_stat)
+ monkeypatch.setattr(fs, "ls", fake_ls)
+ monkeypatch.setattr(
+ fs,
+ "_uri_to_path",
+ lambda uri, ctx=None: uri.replace("viking://", "/"),
+ )
+ monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False)
+ monkeypatch.setattr(viking_fs_module, "_DEFAULT_GREP_FILE_CONCURRENCY", 2)
+
+ result = await fs.grep("viking://resources", pattern="match", node_limit=2)
+
+ assert result["count"] == 2
+ assert result["files_scanned"] == 1
+ assert read_paths == ["/resources/file0.md", "/resources/file1.md"]