Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(grep): paginate fallback directory traversal (#5270)

Lusen Dong611f5c469b2b
diff --git a/openviking/storage/viking_fs/_grep.py b/openviking/storage/viking_fs/_grep.py
index 61ec4bf32..4ae02c403 100644
--- a/openviking/storage/viking_fs/_grep.py
+++ b/openviking/storage/viking_fs/_grep.py
@@ -12,8 +12,11 @@ from openviking.pyagfs.exceptions import AGFSNotSupportedError
from openviking.server.identity import RequestContext
from openviking.storage.expr import And, PathScope, RawDSL
from openviking.storage.viking_fs._base import logger
+from openviking_cli.exceptions import PermissionDeniedError
from openviking_cli.utils.config.grep_config import GrepEngine
+_GREP_LS_PAGE_SIZE = 1000
+
def _pkg():
return sys.modules[__package__]
@@ -693,23 +696,42 @@ class _GrepMixin:
logger.debug(f"Skipping excluded uri during grep: {normalized_current_uri}")
return
- try:
- entries = await self.ls(normalized_current_uri, ctx=ctx)
- except Exception:
- return
-
- for entry in entries:
- entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}"
- if excluded_prefix and (
- entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/")
- ):
- logger.debug(f"Skipping excluded uri during grep: {entry_uri}")
- continue
-
- if entry.get("isDir"):
- await search_recursive(entry_uri, current_depth + 1)
- elif allowed_uris is None or entry_uri in allowed_uris:
- file_uris.append(entry_uri)
+ offset = 0
+ while True:
+ try:
+ entries = await self.ls(
+ normalized_current_uri,
+ node_limit=_GREP_LS_PAGE_SIZE,
+ offset=offset,
+ ctx=ctx,
+ )
+ except PermissionDeniedError:
+ if current_depth == 0:
+ raise
+ logger.debug(
+ f"Skipping inaccessible directory during grep: {normalized_current_uri}"
+ )
+ return
+
+ for entry in entries:
+ entry_uri = f"{normalized_current_uri.rstrip('/')}/{entry['name']}"
+ if excluded_prefix and (
+ entry_uri == excluded_prefix or entry_uri.startswith(excluded_prefix + "/")
+ ):
+ logger.debug(f"Skipping excluded uri during grep: {entry_uri}")
+ continue
+ if entry.get("access") == "denied":
+ logger.debug(f"Skipping inaccessible uri during grep: {entry_uri}")
+ continue
+
+ if entry.get("isDir"):
+ await search_recursive(entry_uri, current_depth + 1)
+ elif allowed_uris is None or entry_uri in allowed_uris:
+ file_uris.append(entry_uri)
+
+ if len(entries) < _GREP_LS_PAGE_SIZE:
+ break
+ offset += len(entries)
normalized_uri = uri
if excluded_prefix and (
@@ -717,10 +739,7 @@ class _GrepMixin:
):
logger.debug(f"Skipping excluded uri during grep: {normalized_uri}")
return file_uris
- try:
- root_stat = await self.stat(normalized_uri, ctx=ctx, skip_count=True)
- except Exception:
- return file_uris
+ root_stat = await self.stat(normalized_uri, ctx=ctx, skip_count=True)
if not root_stat.get("isDir", False):
if allowed_uris is None or normalized_uri in allowed_uris:
file_uris.append(normalized_uri)
diff --git a/tests/storage/test_viking_fs_grep.py b/tests/storage/test_viking_fs_grep.py
index 412bc0db7..c85845533 100644
--- a/tests/storage/test_viking_fs_grep.py
+++ b/tests/storage/test_viking_fs_grep.py
@@ -12,6 +12,8 @@ from openviking.server.identity import RequestContext, Role
from openviking.storage.acl import AclEntry, AclLevel, AclMode, DirectAcl, EffectiveAcl
from openviking.storage.expr import And, PathScope, RawDSL
from openviking.storage.viking_fs import _DEFAULT_GREP_FILE_CONCURRENCY, VikingFS
+from openviking.storage.viking_fs import _grep as grep_module
+from openviking_cli.exceptions import PermissionDeniedError
from openviking_cli.session.user_id import UserIdentifier
from openviking_cli.utils.config.grep_config import GrepConfig
@@ -83,6 +85,184 @@ async def test_collect_grep_files_skips_directory_vector_count(monkeypatch):
stat.assert_awaited_once_with("viking://resources", ctx=None, skip_count=True)
[email protected]("entry_count", [999, 1000, 1001, 10000])
+async def test_collect_grep_files_paginates_wide_directories(monkeypatch, entry_count):
+ viking_fs = VikingFS(agfs=_DummyAgfs())
+ entries = [
+ {"name": f"archive_{index:05d}.jsonl", "isDir": False} for index in range(entry_count)
+ ]
+ offsets = []
+
+ async def fake_ls(uri, node_limit, offset, ctx=None):
+ assert uri == "viking://user/test/sessions/session-1/history"
+ offsets.append(offset)
+ return entries[offset : offset + node_limit]
+
+ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))
+ monkeypatch.setattr(viking_fs, "ls", fake_ls)
+
+ files = await viking_fs._collect_grep_files(
+ "viking://user/test/sessions/session-1/history",
+ excluded_prefix=None,
+ level_limit=1,
+ )
+
+ assert len(files) == entry_count
+ assert files[-1].endswith(f"archive_{entry_count - 1:05d}.jsonl")
+ assert offsets == list(range(0, entry_count + 1, grep_module._GREP_LS_PAGE_SIZE))
+
+
+async def test_grep_fallback_finds_match_after_first_directory_page(monkeypatch):
+ viking_fs = VikingFS(agfs=_DummyAgfs())
+ entries = [
+ {"name": f"archive_{index:04d}.jsonl", "isDir": False} for index in range(1001)
+ ]
+
+ async def fake_ls(uri, node_limit, offset, ctx=None):
+ return entries[offset : offset + node_limit]
+
+ async def fake_read(uri, ctx=None):
+ return b"needle" if uri.endswith("archive_1000.jsonl") else b"haystack"
+
+ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))
+ monkeypatch.setattr(viking_fs, "ls", fake_ls)
+ monkeypatch.setattr(viking_fs, "read", fake_read)
+
+ result = await viking_fs._grep_encrypted(
+ "viking://user/test/sessions/session-1/history",
+ pattern="needle",
+ level_limit=1,
+ )
+
+ assert result["matches"] == [
+ {
+ "line": 1,
+ "uri": (
+ "viking://user/test/sessions/session-1/history/archive_1000.jsonl"
+ ),
+ "content": "needle",
+ }
+ ]
+ assert result["files_scanned"] == 1001
+
+
+async def test_collect_grep_files_preserves_dfs_order_across_pages(monkeypatch):
+ viking_fs = VikingFS(agfs=_DummyAgfs())
+ entries = {
+ "viking://resources": [
+ {"name": "a.md", "isDir": False},
+ {"name": "dir", "isDir": True},
+ {"name": "z.md", "isDir": False},
+ ],
+ "viking://resources/dir": [
+ {"name": "nested.md", "isDir": False},
+ ],
+ }
+
+ async def fake_ls(uri, node_limit, offset, ctx=None):
+ return entries.get(uri, [])[offset : offset + node_limit]
+
+ monkeypatch.setattr(grep_module, "_GREP_LS_PAGE_SIZE", 2)
+ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))
+ monkeypatch.setattr(viking_fs, "ls", fake_ls)
+
+ assert await viking_fs._collect_grep_files(
+ "viking://resources",
+ excluded_prefix=None,
+ level_limit=1,
+ ) == [
+ "viking://resources/a.md",
+ "viking://resources/dir/nested.md",
+ "viking://resources/z.md",
+ ]
+
+
+async def test_collect_grep_files_skips_acl_denied_subtrees(monkeypatch):
+ viking_fs = VikingFS(agfs=_DummyAgfs())
+ listed = []
+
+ async def fake_ls(uri, node_limit, offset, ctx=None):
+ listed.append(uri)
+ if uri == "viking://resources":
+ return [
+ {"name": "public.md", "isDir": False},
+ {"name": "private", "isDir": True, "access": "denied"},
+ {"name": "revoked", "isDir": True},
+ ]
+ if uri == "viking://resources/revoked":
+ raise PermissionDeniedError("access revoked", resource=uri)
+ raise AssertionError(f"grep must not enter {uri}")
+
+ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))
+ monkeypatch.setattr(viking_fs, "ls", fake_ls)
+
+ assert await viking_fs._collect_grep_files(
+ "viking://resources",
+ excluded_prefix=None,
+ level_limit=1,
+ ) == ["viking://resources/public.md"]
+ assert listed == ["viking://resources", "viking://resources/revoked"]
+
+
+async def test_collect_grep_files_propagates_root_acl_denial(monkeypatch):
+ viking_fs = VikingFS(agfs=_DummyAgfs())
+ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))
+ monkeypatch.setattr(
+ viking_fs,
+ "ls",
+ AsyncMock(side_effect=PermissionDeniedError("access denied")),
+ )
+
+ with pytest.raises(PermissionDeniedError, match="access denied"):
+ await viking_fs._collect_grep_files(
+ "viking://resources",
+ excluded_prefix=None,
+ level_limit=1,
+ )
+
+
+async def test_collect_grep_files_propagates_later_page_failure(monkeypatch):
+ viking_fs = VikingFS(agfs=_DummyAgfs())
+
+ async def fake_ls(uri, node_limit, offset, ctx=None):
+ if offset:
+ raise RuntimeError("page 2 failed")
+ return [{"name": f"file_{index:04d}.md", "isDir": False} for index in range(node_limit)]
+
+ monkeypatch.setattr(viking_fs, "stat", AsyncMock(return_value={"isDir": True}))
+ monkeypatch.setattr(viking_fs, "ls", fake_ls)
+
+ with pytest.raises(RuntimeError, match="page 2 failed"):
+ await viking_fs._collect_grep_files(
+ "viking://resources",
+ excluded_prefix=None,
+ level_limit=1,
+ )
+
+
+async def test_collect_grep_files_propagates_root_stat_failure(monkeypatch):
+ viking_fs = VikingFS(agfs=_DummyAgfs())
+ monkeypatch.setattr(
+ viking_fs,
+ "stat",
+ AsyncMock(side_effect=RuntimeError("root stat failed")),
+ )
+
+ with pytest.raises(RuntimeError, match="root stat failed"):
+ await viking_fs._collect_grep_files(
+ "viking://resources",
+ excluded_prefix=None,
+ level_limit=1,
+ )
+
+
def test_grep_config_default_switch_to_remote_threshold_is_10000():
assert GrepConfig().switch_to_remote_threshold == 10000