Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: parallelize grep reads for remote storage (#1761)
Jiahui Zhoub9753385ec2d
diff --git a/crates/ragfs-python/src/lib.rs b/crates/ragfs-python/src/lib.rsindex 18e500564..bd239d576 100644--- a/crates/ragfs-python/src/lib.rs+++ b/crates/ragfs-python/src/lib.rs@@ -18,6 +18,14 @@ use ragfs::plugins::{KVFSPlugin, LocalFSPlugin, MemFSPlugin, QueueFSPlugin, SQLFSPlugin, ServerInfoFSPlugin,};+fn py_detach_blocking<T, F>(py: Python<'_>, f: F) -> T+where+ T: Send,+ F: Send + FnOnce() -> T,+{+ py.detach(f)+}+/// Get a Python exception class from the pyagfs modulefn get_exception<'py>(py: Python<'py>, name: &str) -> PyResult<Bound<'py, PyType>> {let pyagfs = PyModule::import(py, "openviking.pyagfs")?;@@ -220,10 +228,10 @@ impl RAGFSBindingClient {/// name, size, mode, modTime, isDirfn ls(&self, path: String) -> PyResult<Py<PyAny>> {let fs = self.fs.clone();- let entries = self- .rt- .block_on(async move { fs.read_dir(&path).await })- .map_err(to_py_err)?;+ let entries = Python::attach(|py| {+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.read_dir(&path).await }))+ })+ .map_err(to_py_err)?;Python::attach(|py| {let list = PyList::empty(py);@@ -254,10 +262,10 @@ impl RAGFSBindingClient {let off = if offset < 0 { 0u64 } else { offset as u64 };let sz = if size < 0 { 0u64 } else { size as u64 };- let data = self- .rt- .block_on(async move { fs.read(&path, off, sz).await })- .map_err(to_py_err)?;+ let data = Python::attach(|py| {+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.read(&path, off, sz).await }))+ })+ .map_err(to_py_err)?;Python::attach(|py| Ok(PyBytes::new(py, &data).into()))}@@ -278,9 +286,13 @@ impl RAGFSBindingClient {let _ = max_retries; // not applicable for local bindinglet fs = self.fs.clone();let len = data.len();- self.rt- .block_on(async move { fs.write(&path, &data, 0, WriteFlag::Create).await })- .map_err(to_py_err)?;+ Python::attach(|py| {+ py_detach_blocking(py, move || {+ self.rt+ .block_on(async move { fs.write(&path, &data, 0, WriteFlag::Create).await })+ })+ })+ .map_err(to_py_err)?;Ok(format!("Written {} bytes", len))}@@ -288,9 +300,10 @@ impl RAGFSBindingClient {/// Create a new empty file.fn create(&self, path: String) -> PyResult<HashMap<String, String>> {let fs = self.fs.clone();- self.rt- .block_on(async move { fs.create(&path).await })- .map_err(to_py_err)?;+ Python::attach(|py| {+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.create(&path).await }))+ })+ .map_err(to_py_err)?;let mut m = HashMap::new();m.insert("message".to_string(), "created".to_string());@@ -304,9 +317,10 @@ impl RAGFSBindingClient {.map_err(|e| PyRuntimeError::new_err(format!("Invalid mode '{}': {}", mode, e)))?;let fs = self.fs.clone();- self.rt- .block_on(async move { fs.mkdir(&path, mode_int).await })- .map_err(to_py_err)?;+ Python::attach(|py| {+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.mkdir(&path, mode_int).await }))+ })+ .map_err(to_py_err)?;let mut m = HashMap::new();m.insert("message".to_string(), "created".to_string());@@ -317,15 +331,18 @@ impl RAGFSBindingClient {#[pyo3(signature = (path, recursive=false))]fn rm(&self, path: String, recursive: bool) -> PyResult<HashMap<String, String>> {let fs = self.fs.clone();- self.rt- .block_on(async move {- if recursive {- fs.remove_all(&path).await- } else {- fs.remove(&path).await- }+ Python::attach(|py| {+ py_detach_blocking(py, move || {+ self.rt.block_on(async move {+ if recursive {+ fs.remove_all(&path).await+ } else {+ fs.remove(&path).await+ }+ })})- .map_err(to_py_err)?;+ })+ .map_err(to_py_err)?;let mut m = HashMap::new();m.insert("message".to_string(), "deleted".to_string());@@ -335,10 +352,10 @@ impl RAGFSBindingClient {/// Get file/directory information.fn stat(&self, path: String) -> PyResult<Py<PyAny>> {let fs = self.fs.clone();- let info = self- .rt- .block_on(async move { fs.stat(&path).await })- .map_err(to_py_err)?;+ let info = Python::attach(|py| {+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.stat(&path).await }))+ })+ .map_err(to_py_err)?;Python::attach(|py| {let dict = file_info_to_py_dict(py, &info)?;@@ -349,9 +366,13 @@ impl RAGFSBindingClient {/// Rename/move a file or directory.fn mv(&self, old_path: String, new_path: String) -> PyResult<HashMap<String, String>> {let fs = self.fs.clone();- self.rt- .block_on(async move { fs.rename(&old_path, &new_path).await })- .map_err(to_py_err)?;+ Python::attach(|py| {+ py_detach_blocking(py, move || {+ self.rt+ .block_on(async move { fs.rename(&old_path, &new_path).await })+ })+ })+ .map_err(to_py_err)?;let mut m = HashMap::new();m.insert("message".to_string(), "renamed".to_string());@@ -361,9 +382,10 @@ impl RAGFSBindingClient {/// Change file permissions.fn chmod(&self, path: String, mode: u32) -> PyResult<HashMap<String, String>> {let fs = self.fs.clone();- self.rt- .block_on(async move { fs.chmod(&path, mode).await })- .map_err(to_py_err)?;+ Python::attach(|py| {+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.chmod(&path, mode).await }))+ })+ .map_err(to_py_err)?;let mut m = HashMap::new();m.insert("message".to_string(), "chmod ok".to_string());@@ -373,18 +395,21 @@ impl RAGFSBindingClient {/// Touch a file (create if not exists, or update timestamp).fn touch(&self, path: String) -> PyResult<HashMap<String, String>> {let fs = self.fs.clone();- self.rt- .block_on(async move {- // Try create; if already exists, write empty to update mtime- match fs.create(&path).await {- Ok(_) => Ok(()),- Err(_) => {- // File exists, write empty bytes to update timestamp- fs.write(&path, &[], 0, WriteFlag::None).await.map(|_| ())+ Python::attach(|py| {+ py_detach_blocking(py, move || {+ self.rt.block_on(async move {+ // Try create; if already exists, write empty to update mtime+ match fs.create(&path).await {+ Ok(_) => Ok(()),+ Err(_) => {+ // File exists, write empty bytes to update timestamp+ fs.write(&path, &[], 0, WriteFlag::None).await.map(|_| ())+ }}- }+ })})- .map_err(to_py_err)?;+ })+ .map_err(to_py_err)?;let mut m = HashMap::new();m.insert("message".to_string(), "touched".to_string());@@ -394,7 +419,9 @@ impl RAGFSBindingClient {/// List all mounted plugins.fn mounts(&self) -> PyResult<Vec<HashMap<String, String>>> {let fs = self.fs.clone();- let mount_list = self.rt.block_on(async move { fs.list_mounts().await });+ let mount_list = Python::attach(|py| {+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.list_mounts().await }))+ });let result: Vec<HashMap<String, String>> = mount_list.into_iter()@@ -434,9 +461,10 @@ impl RAGFSBindingClient {};let fs = self.fs.clone();- self.rt- .block_on(async move { fs.mount(plugin_config).await })- .map_err(to_py_err)?;+ Python::attach(|py| {+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.mount(plugin_config).await }))+ })+ .map_err(to_py_err)?;let mut m = HashMap::new();m.insert(@@ -450,9 +478,10 @@ impl RAGFSBindingClient {fn unmount(&self, path: String) -> PyResult<HashMap<String, String>> {let fs = self.fs.clone();let path_clone = path.clone();- self.rt- .block_on(async move { fs.unmount(&path_clone).await })- .map_err(to_py_err)?;+ Python::attach(|py| {+ py_detach_blocking(py, move || self.rt.block_on(async move { fs.unmount(&path_clone).await }))+ })+ .map_err(to_py_err)?;let mut m = HashMap::new();m.insert("message".to_string(), format!("unmounted {}", path));@@ -534,3 +563,16 @@ fn ragfs_python(m: &Bound<'_, PyModule>) -> PyResult<()> {m.add_class::<RAGFSBindingClient>()?;Ok(())}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn detach_blocking_helper_runs_without_python_objects() {+ Python::attach(|py| {+ let value: i32 = py_detach_blocking(py, || 40 + 2);+ assert_eq!(value, 42);+ });+ }+}diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex c758eab3a..fe932f322 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -53,6 +53,8 @@ if TYPE_CHECKING:logger = get_logger(__name__)+_DEFAULT_GREP_FILE_CONCURRENCY = 32+def _ensure_non_empty_search_query(query: str) -> None:if not query.strip():@@ -632,14 +634,36 @@ class VikingFS:if exclude_uri:excluded_prefix = self._normalize_uri(exclude_uri).rstrip("/")self._ensure_access(excluded_prefix, ctx)+ file_uris = await self._collect_grep_files(+ uri,+ excluded_prefix=excluded_prefix,+ level_limit=level_limit,+ ctx=ctx,+ )+ results, files_scanned = await self._grep_files_parallel(+ file_uris,+ compiled_pattern=compiled_pattern,+ node_limit=node_limit,+ ctx=ctx,+ )- results = []- files_scanned = 0+ return {+ "matches": results,+ "count": len(results),+ "match_count": len(results),+ "files_scanned": files_scanned,+ }- async def search_recursive(current_uri: str, current_depth: int):- if node_limit and len(results) >= node_limit:- return+ async def _collect_grep_files(+ self,+ uri: str,+ excluded_prefix: Optional[str],+ level_limit: int,+ ctx: Optional[RequestContext] = None,+ ) -> List[str]:+ file_uris: List[str] = []+ async def search_recursive(current_uri: str, current_depth: int) -> None:if current_depth > level_limit:return@@ -657,9 +681,6 @@ class VikingFS:returnfor entry in entries:- if node_limit and len(results) >= node_limit:- break-entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}"if excluded_prefix and (entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/")@@ -670,36 +691,70 @@ class VikingFS:if entry.get("isDir"):await search_recursive(entry_uri, current_depth + 1)else:- nonlocal files_scanned- files_scanned += 1- try:- content = await self.read(entry_uri, ctx=ctx)- if isinstance(content, bytes):- content = content.decode("utf-8", errors="replace")-- lines = content.split("\n")- for line_num, line in enumerate(lines, 1):- if compiled_pattern.search(line):- results.append(- {- "line": line_num,- "uri": entry_uri,- "content": line,- }- )- if node_limit and len(results) >= node_limit:- break- except Exception as e:- logger.debug(f"Failed to grep {entry_uri}: {e}")+ file_uris.append(entry_uri)await search_recursive(uri, 0)+ return file_uris- return {- "matches": results,- "count": len(results),- "match_count": len(results),- "files_scanned": files_scanned,- }+ async def _grep_files_parallel(+ self,+ file_uris: List[str],+ compiled_pattern: re.Pattern,+ node_limit: Optional[int],+ ctx: Optional[RequestContext] = None,+ ) -> tuple[List[Dict[str, Any]], int]:+ results: List[Dict[str, Any]] = []+ files_scanned = 0+ for start in range(0, len(file_uris), _DEFAULT_GREP_FILE_CONCURRENCY):+ batch_uris = file_uris[start : start + _DEFAULT_GREP_FILE_CONCURRENCY]+ batch_jobs = [+ asyncio.to_thread(self._grep_single_file, entry_uri, compiled_pattern, ctx)+ for entry_uri in batch_uris+ ]+ batch_results = await asyncio.gather(*batch_jobs)+ for matches, scanned_count in batch_results:+ files_scanned += scanned_count+ for match in matches:+ results.append(match)+ if node_limit and len(results) >= node_limit:+ return results, files_scanned++ return results, files_scanned++ def _grep_single_file(+ self,+ entry_uri: str,+ compiled_pattern: re.Pattern,+ ctx: Optional[RequestContext] = None,+ ) -> tuple[List[Dict[str, Any]], int]:+ try:+ self._ensure_access(entry_uri, ctx)+ path = self._uri_to_path(entry_uri, ctx=ctx)+ result = self.agfs.read(path, 0, -1)+ if isinstance(result, bytes):+ content = result+ elif result is not None and hasattr(result, "content"):+ content = result.content+ else:+ content = b""+ if isinstance(content, bytes):+ content = content.decode("utf-8", errors="replace")++ matches: List[Dict[str, Any]] = []+ lines = content.split("\n")+ for line_num, line in enumerate(lines, 1):+ if compiled_pattern.search(line):+ matches.append(+ {+ "line": line_num,+ "uri": entry_uri,+ "content": line,+ }+ )+ return matches, 1+ except Exception as e:+ logger.debug(f"Failed to grep {entry_uri}: {e}")+ return [], 1async def stat(self, uri: str, ctx: Optional[RequestContext] = None) -> Dict[str, Any]:"""diff --git a/tests/storage/test_viking_fs_grep.py b/tests/storage/test_viking_fs_grep.pynew file mode 100644index 000000000..d6aed7a13--- /dev/null+++ b/tests/storage/test_viking_fs_grep.py@@ -0,0 +1,194 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++import time++import pytest++import openviking.storage.viking_fs as viking_fs_module+from openviking.storage.viking_fs import _DEFAULT_GREP_FILE_CONCURRENCY, VikingFS+++class _DummyAgfs:+ pass+++async def test_grep_preserves_dfs_order_and_node_limit(monkeypatch):+ fs = VikingFS(agfs=_DummyAgfs())++ async def fake_stat(uri, ctx=None):+ return {"isDir": True}++ async def fake_ls(uri, ctx=None, **kwargs):+ entries = {+ "viking://resources": [+ {"name": "dir_a", "isDir": True},+ {"name": "dir_b", "isDir": True},+ ],+ "viking://resources/dir_a": [+ {"name": "a1.md", "isDir": False},+ {"name": "a2.md", "isDir": False},+ ],+ "viking://resources/dir_b": [+ {"name": "b1.md", "isDir": False},+ ],+ }+ return entries.get(uri, [])++ def fake_agfs_read(path, offset, size):+ contents = {+ "/resources/dir_a/a1.md": "match a1 line1\nskip\nmatch a1 line3",+ "/resources/dir_a/a2.md": "match a2 line1",+ "/resources/dir_b/b1.md": "match b1 line1",+ }+ return contents[path].encode()++ monkeypatch.setattr(fs, "stat", fake_stat)+ monkeypatch.setattr(fs, "ls", fake_ls)+ monkeypatch.setattr(+ fs,+ "_uri_to_path",+ lambda uri, ctx=None: uri.replace("viking://", "/"),+ )+ monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False)++ result = await fs.grep("viking://resources", pattern="match", node_limit=3)++ assert result["count"] == 3+ assert result["files_scanned"] == 2+ assert result["matches"] == [+ {+ "line": 1,+ "uri": "viking://resources/dir_a/a1.md",+ "content": "match a1 line1",+ },+ {+ "line": 3,+ "uri": "viking://resources/dir_a/a1.md",+ "content": "match a1 line3",+ },+ {+ "line": 1,+ "uri": "viking://resources/dir_a/a2.md",+ "content": "match a2 line1",+ },+ ]+++async def test_grep_parallel_reads_respect_concurrency_limit(monkeypatch):+ fs = VikingFS(agfs=_DummyAgfs())++ async def fake_stat(uri, ctx=None):+ return {"isDir": True}++ async def fake_ls(uri, ctx=None, **kwargs):+ entries = {+ "viking://resources": [{"name": f"file{i}.md", "isDir": False} for i in range(12)]+ }+ return entries.get(uri, [])++ active_reads = 0+ max_active_reads = 0++ def fake_agfs_read(path, offset, size):+ nonlocal active_reads, max_active_reads+ active_reads += 1+ max_active_reads = max(max_active_reads, active_reads)+ time.sleep(0.01)+ active_reads -= 1+ return f"match from {path}".encode()++ monkeypatch.setattr(fs, "stat", fake_stat)+ monkeypatch.setattr(fs, "ls", fake_ls)+ monkeypatch.setattr(+ fs,+ "_uri_to_path",+ lambda uri, ctx=None: uri.replace("viking://", "/"),+ )+ monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False)++ result = await fs.grep("viking://resources", pattern="match")++ assert result["count"] == 12+ assert result["files_scanned"] == 12+ assert max_active_reads > 1+ assert max_active_reads <= min(12, _DEFAULT_GREP_FILE_CONCURRENCY)+++async def test_grep_parallel_reads_work_with_blocking_agfs_read(monkeypatch):+ fs = VikingFS(agfs=_DummyAgfs())++ async def fake_stat(uri, ctx=None):+ return {"isDir": True}++ async def fake_ls(uri, ctx=None, **kwargs):+ if uri == "viking://resources":+ return [{"name": f"file{i}.md", "isDir": False} for i in range(8)]+ return []++ def fake_agfs_read(path, offset, size):+ time.sleep(0.05)+ return f"match from {path}".encode()++ monkeypatch.setattr(fs, "stat", fake_stat)+ monkeypatch.setattr(fs, "ls", fake_ls)+ monkeypatch.setattr(+ fs,+ "_uri_to_path",+ lambda uri, ctx=None: uri.replace("viking://", "/"),+ )+ monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False)++ started = time.perf_counter()+ result = await fs.grep("viking://resources", pattern="match")+ elapsed = time.perf_counter() - started++ assert result["count"] == 8+ assert result["files_scanned"] == 8+ assert elapsed < 0.30+++async def test_grep_stops_scheduling_later_batches_after_node_limit(monkeypatch):+ fs = VikingFS(agfs=_DummyAgfs())++ async def fake_stat(uri, ctx=None):+ return {"isDir": True}++ async def fake_ls(uri, ctx=None, **kwargs):+ if uri == "viking://resources":+ return [{"name": f"file{i}.md", "isDir": False} for i in range(6)]+ return []++ read_paths = []++ def fake_agfs_read(path, offset, size):+ read_paths.append(path)+ contents = {+ "/resources/file0.md": "match file0 line1\nmatch file0 line2",+ "/resources/file1.md": "match file1 line1",+ "/resources/file2.md": "match file2 line1",+ "/resources/file3.md": "match file3 line1",+ "/resources/file4.md": "match file4 line1",+ "/resources/file5.md": "match file5 line1",+ }+ return contents[path].encode()++ monkeypatch.setattr(fs, "stat", fake_stat)+ monkeypatch.setattr(fs, "ls", fake_ls)+ monkeypatch.setattr(+ fs,+ "_uri_to_path",+ lambda uri, ctx=None: uri.replace("viking://", "/"),+ )+ monkeypatch.setattr(fs.agfs, "read", fake_agfs_read, raising=False)+ monkeypatch.setattr(viking_fs_module, "_DEFAULT_GREP_FILE_CONCURRENCY", 2)++ result = await fs.grep("viking://resources", pattern="match", node_limit=2)++ assert result["count"] == 2+ assert result["files_scanned"] == 1+ assert read_paths == ["/resources/file0.md", "/resources/file1.md"]