Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(grep): paginate fallback directory traversal (#5270)
Lusen Dong611f5c469b2b
diff --git a/openviking/storage/viking_fs/_grep.py b/openviking/storage/viking_fs/_grep.pyindex 61ec4bf32..4ae02c403 100644--- a/openviking/storage/viking_fs/_grep.py+++ b/openviking/storage/viking_fs/_grep.py@@ -12,8 +12,11 @@ from openviking.pyagfs.exceptions import AGFSNotSupportedErrorfrom openviking.server.identity import RequestContextfrom openviking.storage.expr import And, PathScope, RawDSLfrom openviking.storage.viking_fs._base import logger+from openviking_cli.exceptions import PermissionDeniedErrorfrom openviking_cli.utils.config.grep_config import GrepEngine+_GREP_LS_PAGE_SIZE = 1000+def _pkg():return sys.modules[__package__]@@ -693,23 +696,42 @@ class _GrepMixin:logger.debug(f"Skipping excluded uri during grep: {normalized_current_uri}")return- try:- entries = await self.ls(normalized_current_uri, ctx=ctx)- except Exception:- return-- for entry in entries:- entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}"- if excluded_prefix and (- entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/")- ):- logger.debug(f"Skipping excluded uri during grep: {entry_uri}")- continue-- if entry.get("isDir"):- await search_recursive(entry_uri, current_depth + 1)- elif allowed_uris is None or entry_uri in allowed_uris:- file_uris.append(entry_uri)+ offset = 0+ while True:+ try:+ entries = await self.ls(+ normalized_current_uri,+ node_limit=_GREP_LS_PAGE_SIZE,+ offset=offset,+ ctx=ctx,+ )+ except PermissionDeniedError:+ if current_depth == 0:+ raise+ logger.debug(+ f"Skipping inaccessible directory during grep: {normalized_current_uri}"+ )+ return++ for entry in entries:+ entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}"+ if excluded_prefix and (+ entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/")+ ):+ logger.debug(f"Skipping excluded uri during grep: {entry_uri}")+ continue+ if entry.get("access") == "denied":+ logger.debug(f"Skipping inaccessible uri during grep: {entry_uri}")+ continue++ if entry.get("isDir"):+ await search_recursive(entry_uri, current_depth + 1)+ elif allowed_uris is None or entry_uri in allowed_uris:+ file_uris.append(entry_uri)++ if len(entries) < _GREP_LS_PAGE_SIZE:+ break+ offset += len(entries)normalized_uri = uriif excluded_prefix and (@@ -717,10 +739,7 @@ class _GrepMixin:):logger.debug(f"Skipping excluded uri during grep: {normalized_uri}")return file_uris- try:- root_stat = await self.stat(normalized_uri, ctx=ctx, skip_count=True)- except Exception:- return file_uris+ root_stat = await self.stat(normalized_uri, ctx=ctx, skip_count=True)if not root_stat.get("isDir", False):if allowed_uris is None or normalized_uri in allowed_uris:file_uris.append(normalized_uri)diff --git a/tests/storage/test_viking_fs_grep.py b/tests/storage/test_viking_fs_grep.pyindex 412bc0db7..c85845533 100644--- a/tests/storage/test_viking_fs_grep.py+++ b/tests/storage/test_viking_fs_grep.py@@ -12,6 +12,8 @@ from openviking.server.identity import RequestContext, Rolefrom openviking.storage.acl import AclEntry, AclLevel, AclMode, DirectAcl, EffectiveAclfrom openviking.storage.expr import And, PathScope, RawDSLfrom openviking.storage.viking_fs import _DEFAULT_GREP_FILE_CONCURRENCY, VikingFS+from openviking.storage.viking_fs import _grep as grep_module+from openviking_cli.exceptions import PermissionDeniedErrorfrom openviking_cli.session.user_id import UserIdentifierfrom openviking_cli.utils.config.grep_config import GrepConfig@@ -83,6 +85,184 @@ async def test_collect_grep_files_skips_directory_vector_count(monkeypatch):stat.assert_awaited_once_with("viking://resources", ctx=None, skip_count=True)[email protected]("entry_count", [999, 1000, 1001, 10000])+async def test_collect_grep_files_paginates_wide_directories(monkeypatch, entry_count):+ viking_fs = VikingFS(agfs=_DummyAgfs())+ entries = [+ {"name": f"archive_{index:05d}.jsonl", "isDir": False} for index in range(entry_count)+ ]+ offsets = []++ async def fake_ls(uri, node_limit, offset, ctx=None):+ assert uri == "viking://user/test/sessions/session-1/history"+ offsets.append(offset)+ return entries[offset : offset + node_limit]++ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))+ monkeypatch.setattr(viking_fs, "ls", fake_ls)++ files = await viking_fs._collect_grep_files(+ "viking://user/test/sessions/session-1/history",+ excluded_prefix=None,+ level_limit=1,+ )++ assert len(files) == entry_count+ assert files[-1].endswith(f"archive_{entry_count - 1:05d}.jsonl")+ assert offsets == list(range(0, entry_count + 1, grep_module._GREP_LS_PAGE_SIZE))+++async def test_grep_fallback_finds_match_after_first_directory_page(monkeypatch):+ viking_fs = VikingFS(agfs=_DummyAgfs())+ entries = [+ {"name": f"archive_{index:04d}.jsonl", "isDir": False} for index in range(1001)+ ]++ async def fake_ls(uri, node_limit, offset, ctx=None):+ return entries[offset : offset + node_limit]++ async def fake_read(uri, ctx=None):+ return b"needle" if uri.endswith("archive_1000.jsonl") else b"haystack"++ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))+ monkeypatch.setattr(viking_fs, "ls", fake_ls)+ monkeypatch.setattr(viking_fs, "read", fake_read)++ result = await viking_fs._grep_encrypted(+ "viking://user/test/sessions/session-1/history",+ pattern="needle",+ level_limit=1,+ )++ assert result["matches"] == [+ {+ "line": 1,+ "uri": (+ "viking://user/test/sessions/session-1/history/archive_1000.jsonl"+ ),+ "content": "needle",+ }+ ]+ assert result["files_scanned"] == 1001+++async def test_collect_grep_files_preserves_dfs_order_across_pages(monkeypatch):+ viking_fs = VikingFS(agfs=_DummyAgfs())+ entries = {+ "viking://resources": [+ {"name": "a.md", "isDir": False},+ {"name": "dir", "isDir": True},+ {"name": "z.md", "isDir": False},+ ],+ "viking://resources/dir": [+ {"name": "nested.md", "isDir": False},+ ],+ }++ async def fake_ls(uri, node_limit, offset, ctx=None):+ return entries.get(uri, [])[offset : offset + node_limit]++ monkeypatch.setattr(grep_module, "_GREP_LS_PAGE_SIZE", 2)+ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))+ monkeypatch.setattr(viking_fs, "ls", fake_ls)++ assert await viking_fs._collect_grep_files(+ "viking://resources",+ excluded_prefix=None,+ level_limit=1,+ ) == [+ "viking://resources/a.md",+ "viking://resources/dir/nested.md",+ "viking://resources/z.md",+ ]+++async def test_collect_grep_files_skips_acl_denied_subtrees(monkeypatch):+ viking_fs = VikingFS(agfs=_DummyAgfs())+ listed = []++ async def fake_ls(uri, node_limit, offset, ctx=None):+ listed.append(uri)+ if uri == "viking://resources":+ return [+ {"name": "public.md", "isDir": False},+ {"name": "private", "isDir": True, "access": "denied"},+ {"name": "revoked", "isDir": True},+ ]+ if uri == "viking://resources/revoked":+ raise PermissionDeniedError("access revoked", resource=uri)+ raise AssertionError(f"grep must not enter {uri}")++ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))+ monkeypatch.setattr(viking_fs, "ls", fake_ls)++ assert await viking_fs._collect_grep_files(+ "viking://resources",+ excluded_prefix=None,+ level_limit=1,+ ) == ["viking://resources/public.md"]+ assert listed == ["viking://resources", "viking://resources/revoked"]+++async def test_collect_grep_files_propagates_root_acl_denial(monkeypatch):+ viking_fs = VikingFS(agfs=_DummyAgfs())+ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))+ monkeypatch.setattr(+ viking_fs,+ "ls",+ AsyncMock(side_effect=PermissionDeniedError("access denied")),+ )++ with pytest.raises(PermissionDeniedError, match="access denied"):+ await viking_fs._collect_grep_files(+ "viking://resources",+ excluded_prefix=None,+ level_limit=1,+ )+++async def test_collect_grep_files_propagates_later_page_failure(monkeypatch):+ viking_fs = VikingFS(agfs=_DummyAgfs())++ async def fake_ls(uri, node_limit, offset, ctx=None):+ if offset:+ raise RuntimeError("page 2 failed")+ return [{"name": f"file_{index:04d}.md", "isDir": False} for index in range(node_limit)]++ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))+ monkeypatch.setattr(viking_fs, "ls", fake_ls)++ with pytest.raises(RuntimeError, match="page 2 failed"):+ await viking_fs._collect_grep_files(+ "viking://resources",+ excluded_prefix=None,+ level_limit=1,+ )+++async def test_collect_grep_files_propagates_root_stat_failure(monkeypatch):+ viking_fs = VikingFS(agfs=_DummyAgfs())+ monkeypatch.setattr(+ viking_fs,+ "stat",+ AsyncMock(side_effect=RuntimeError("root stat failed")),+ )++ with pytest.raises(RuntimeError, match="root stat failed"):+ await viking_fs._collect_grep_files(+ "viking://resources",+ excluded_prefix=None,+ level_limit=1,+ )++def test_grep_config_default_switch_to_remote_threshold_is_10000():assert GrepConfig().switch_to_remote_threshold == 10000