Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(ingest): enumerate all children on the ingest path / 入库时枚举目录全部子节点,修复 ls node_limit=1000 静默截断 >1000 文档 (#2518)

Kchen2f214f2e34ab
diff --git a/openviking/parse/parsers/directory.py b/openviking/parse/parsers/directory.py
index 7044d46ce..47ebf1af1 100644
--- a/openviking/parse/parsers/directory.py
+++ b/openviking/parse/parsers/directory.py
@@ -28,6 +28,7 @@ from openviking.parse.base import (
)
from openviking.parse.parsers.base_parser import BaseParser
from openviking.parse.parsers.media.constants import MEDIA_EXTENSIONS
+from openviking.storage.viking_fs import LS_ALL_NODES
from openviking_cli.utils.logger import get_logger
if TYPE_CHECKING:
@@ -461,7 +462,7 @@ class DirectoryParser(BaseParser):
After the move the source temp is deleted.
"""
- entries = await viking_fs.ls(src_temp_uri)
+ entries = await viking_fs.ls(src_temp_uri, node_limit=LS_ALL_NODES)
for entry in entries:
name = entry.get("name", "")
if not name or name in (".", ".."):
@@ -530,7 +531,7 @@ class DirectoryParser(BaseParser):
) -> None:
"""Recursively move a VikingFS directory tree."""
await viking_fs.mkdir(dst_uri, exist_ok=True)
- entries = await viking_fs.ls(src_uri)
+ entries = await viking_fs.ls(src_uri, node_limit=LS_ALL_NODES)
for entry in entries:
name = entry.get("name", "")
if not name or name in (".", ".."):
diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.py
index a5b9921a3..9f35c93e7 100644
--- a/openviking/storage/queuefs/semantic_dag.py
+++ b/openviking/storage/queuefs/semantic_dag.py
@@ -11,7 +11,7 @@ from typing import ClassVar, Dict, List, Optional, Set
from openviking.server.identity import RequestContext
from openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars
from openviking.storage.transaction import NO_LOCK, LockLease
-from openviking.storage.viking_fs import get_viking_fs
+from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking_cli.utils import VikingURI
from openviking_cli.utils.logger import get_logger
@@ -470,7 +470,7 @@ class SemanticDagExecutor:
async def _list_dir(self, uri: str, from_hint: str) -> tuple[list[str], list[str]]:
"""List directory entries and return (child_dirs, file_paths)."""
try:
- entries = await self._viking_fs.ls(uri, ctx=self._ctx)
+ entries = await self._viking_fs.ls(uri, node_limit=LS_ALL_NODES, ctx=self._ctx)
except Exception as e:
logger.warning(
f"[SemanticDagExecutor] Failed to list directory {uri}: {e} from {from_hint}"
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index fa3bf5f28..a4549663f 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -36,7 +36,7 @@ from openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_
from openviking.storage.queuefs.semantic_queue import is_semantic_msg_stale
from openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecars
from openviking.storage.transaction import NO_LOCK, LockLease
-from openviking.storage.viking_fs import get_viking_fs
+from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.telemetry import bind_telemetry, bind_telemetry_stage, resolve_telemetry
from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking.telemetry.span_models import create_root_span_attributes
@@ -538,7 +538,7 @@ class SemanticProcessor(DequeueHandlerBase):
try:
try:
- entries = await viking_fs.ls(dir_uri, ctx=ctx)
+ entries = await viking_fs.ls(dir_uri, node_limit=LS_ALL_NODES, ctx=ctx)
except Exception as e:
raise RuntimeError(f"Failed to list memory directory {dir_uri}: {e}") from e
@@ -749,7 +749,9 @@ class SemanticProcessor(DequeueHandlerBase):
files: Dict[str, str] = {}
dirs: Dict[str, str] = {}
try:
- entries = await viking_fs.ls(dir_uri, show_all_hidden=True, ctx=ctx)
+ entries = await viking_fs.ls(
+ dir_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx
+ )
except Exception as e:
logger.error(f"[SyncDiff] Failed to list {dir_uri}: {e}")
return files, dirs
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 3a46653a5..79643650c 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -63,6 +63,15 @@ if TYPE_CHECKING:
logger = get_logger(__name__)
+# Sentinel node_limit for internal callers that MUST enumerate an entire
+# directory. ``ls()`` defaults to ``node_limit=1000`` to protect agent-facing
+# context from being flooded, but internal system operations (parse merge,
+# temp->final sync, summary DAG, vectorization) must see every child or they
+# silently drop entries beyond the cap — e.g. a >1000-doc directory ingest only
+# materializes its first 1000 subdirectories. Pass this explicitly at those
+# call sites.
+LS_ALL_NODES = 2**31 - 1
+
def _ensure_non_empty_search_query(query: str) -> None:
if not query.strip():
diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.py
index 54b1f46ee..e29a42137 100644
--- a/openviking/utils/embedding_utils.py
+++ b/openviking/utils/embedding_utils.py
@@ -16,7 +16,7 @@ from openviking.core.namespace import context_type_for_uri, owner_space_for_uri
from openviking.server.identity import RequestContext
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
-from openviking.storage.viking_fs import get_viking_fs
+from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.utils.time_utils import parse_iso_datetime
from openviking_cli.utils import VikingURI, get_logger
from openviking_cli.utils.config import get_openviking_config
@@ -507,7 +507,7 @@ async def index_resource(
# 2. Index Files
try:
- files = await viking_fs.ls(uri, ctx=ctx)
+ files = await viking_fs.ls(uri, node_limit=LS_ALL_NODES, ctx=ctx)
for file_info in files:
file_name = file_info["name"]
diff --git a/openviking/utils/summarizer.py b/openviking/utils/summarizer.py
index 744034129..88051a85a 100644
--- a/openviking/utils/summarizer.py
+++ b/openviking/utils/summarizer.py
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
from openviking.core.namespace import context_type_for_uri
from openviking.storage.queuefs import SemanticMsg, get_queue_manager
from openviking.storage.transaction import NO_LOCK, LockLease
-from openviking.storage.viking_fs import get_viking_fs
+from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fs
from openviking.telemetry import get_current_telemetry
from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
from openviking_cli.utils import get_logger
@@ -81,7 +81,9 @@ class Summarizer:
async def list_top_children(temp_uri: str) -> List[Tuple[str, str]]:
viking_fs = get_viking_fs()
- entries = await viking_fs.ls(temp_uri, show_all_hidden=True, ctx=ctx)
+ entries = await viking_fs.ls(
+ temp_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx
+ )
children: List[Tuple[str, str]] = []
for entry in entries:
name = entry.get("name", "")
diff --git a/tests/storage/test_ingest_ls_node_limit.py b/tests/storage/test_ingest_ls_node_limit.py
new file mode 100644
index 000000000..b4c8d4401
--- /dev/null
+++ b/tests/storage/test_ingest_ls_node_limit.py
@@ -0,0 +1,125 @@
+"""Regression: internal directory enumeration during ingest must not be capped
+at ``viking_fs.ls``'s default ``node_limit=1000``.
+
+When a resource namespace already exists, re-ingesting a directory takes the
+incremental sync path (``_sync_topdown_recursive`` -> ``sync_dir`` ->
+``list_children``), which lists the temp root's children via ``viking_fs.ls``.
+``ls`` defaults ``node_limit=1000``, so an import of >1000 docs silently
+materialized only the first 1000 subdirectories into the target; the rest were
+dropped when temp was deleted.
+
+Observed in the wild: a 6,221-doc WixQA re-ingest (the namespace already held an
+earlier 209-doc import, so the atomic-move fast path was skipped) produced
+exactly 1000 directories under ``resources/wixqa``.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from openviking.storage.queuefs.semantic_processor import SemanticProcessor
+from openviking.storage.transaction import NO_LOCK
+
+
+class _TruncatingVikingFS:
+ """Fake whose ``ls`` truncates at ``node_limit``, exactly like the real one.
+
+ The temp root holds ``n_children`` doc subdirectories; the target namespace
+ pre-exists but is empty (forcing the incremental ``sync_dir`` path instead of
+ the atomic whole-directory move).
+ """
+
+ TEMP = "viking://temp/import"
+ TARGET = "viking://resources/wixqa"
+
+ def __init__(self, n_children: int):
+ self._children = [
+ {"name": f"doc{i:05d}", "isDir": True} for i in range(n_children)
+ ]
+ self.moved: list[tuple[str, str]] = []
+
+ async def exists(self, uri, ctx=None):
+ return uri in (self.TEMP, self.TARGET)
+
+ async def ls(self, uri, show_all_hidden=False, node_limit=1000, ctx=None):
+ entries = self._children if uri == self.TEMP else []
+ return entries[:node_limit] # the real ls truncates here
+
+ async def mv(self, src, dst, ctx=None, lock_handle=None):
+ self.moved.append((src, dst))
+
+ async def rm(self, uri, recursive=False, ctx=None, lock_handle=None):
+ pass
+
+ async def mkdir(self, uri, exist_ok=False, ctx=None):
+ pass
+
+ async def delete_temp(self, uri, ctx=None):
+ pass
+
+
+async def test_sync_materializes_all_children_above_default_node_limit(monkeypatch):
+ n_children = 1500 # > the default node_limit of 1000
+ fake = _TruncatingVikingFS(n_children)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_processor.get_viking_fs",
+ lambda: fake,
+ )
+
+ diff = await SemanticProcessor()._sync_topdown_recursive(
+ _TruncatingVikingFS.TEMP,
+ _TruncatingVikingFS.TARGET,
+ lock=NO_LOCK,
+ )
+
+ assert len(fake.moved) == n_children, (
+ f"only {len(fake.moved)}/{n_children} doc dirs materialized — internal "
+ "sync was truncated at ls node_limit"
+ )
+ assert len(diff.added_dirs) == n_children
+
+
+class _TruncatingLsFS:
+ """Minimal fake exposing only a node_limit-truncating ``ls`` over a fixed
+ set of child directories."""
+
+ def __init__(self, dir_uri: str, n_children: int):
+ self._dir_uri = dir_uri
+ self._children = [
+ {"name": f"doc{i:05d}", "isDir": True} for i in range(n_children)
+ ]
+
+ async def ls(self, uri, node_limit=1000, ctx=None):
+ entries = self._children if uri == self._dir_uri else []
+ return entries[:node_limit] # the real ls truncates here
+
+
+async def test_semantic_dag_list_dir_enumerates_all_children(monkeypatch):
+ """The summary DAG dispatches one summary/recursion task per child it lists.
+ If ``_list_dir`` is capped at node_limit, children beyond 1000 never get an
+ ``.abstract.md``/``.overview.md`` (and are never recursed into), so a large
+ namespace silently loses its L0/L1 layers."""
+ from openviking.storage.queuefs.semantic_dag import SemanticDagExecutor
+
+ dir_uri = "viking://resources/wixqa"
+ n_children = 1500 # > the default node_limit of 1000
+ fake = _TruncatingLsFS(dir_uri, n_children)
+ monkeypatch.setattr(
+ "openviking.storage.queuefs.semantic_dag.get_viking_fs",
+ lambda: fake,
+ )
+
+ executor = SemanticDagExecutor(
+ processor=None,
+ context_type="resource",
+ max_concurrent_llm=1,
+ ctx=None,
+ )
+ children_dirs, file_paths = await executor._list_dir(dir_uri, from_hint="test")
+
+ assert len(children_dirs) == n_children, (
+ f"only {len(children_dirs)}/{n_children} children listed — summary DAG "
+ "was truncated at ls node_limit"
+ )
diff --git a/tests/storage/test_semantic_dag_incremental.py b/tests/storage/test_semantic_dag_incremental.py
index ee45577c1..60d165599 100644
--- a/tests/storage/test_semantic_dag_incremental.py
+++ b/tests/storage/test_semantic_dag_incremental.py
@@ -40,7 +40,7 @@ class _FakeVikingFS:
rest = re.sub(r"/{2,}", "/", rest)
return f"{scheme}://{rest}"
- async def ls(self, uri, ctx=None):
+ async def ls(self, uri, node_limit=None, ctx=None):
return self._tree.get(self._norm(uri), [])
async def stat(self, uri, ctx=None):
diff --git a/tests/storage/test_semantic_dag_skip_files.py b/tests/storage/test_semantic_dag_skip_files.py
index 7f2fd155e..70bae748d 100644
--- a/tests/storage/test_semantic_dag_skip_files.py
+++ b/tests/storage/test_semantic_dag_skip_files.py
@@ -32,7 +32,7 @@ class _FakeVikingFS:
self._tree = tree
self.writes = []
- async def ls(self, uri, ctx=None):
+ async def ls(self, uri, node_limit=None, ctx=None):
return self._tree.get(uri, [])
async def write_file(self, path, content, ctx=None):
diff --git a/tests/storage/test_semantic_processor_target_preexisting.py b/tests/storage/test_semantic_processor_target_preexisting.py
index 4bbe08110..2546509a3 100644
--- a/tests/storage/test_semantic_processor_target_preexisting.py
+++ b/tests/storage/test_semantic_processor_target_preexisting.py
@@ -40,7 +40,7 @@ class _SyncVikingFS:
async def exists(self, uri, ctx=None):
return uri in self.entries
- async def ls(self, uri, show_all_hidden=False, ctx=None):
+ async def ls(self, uri, show_all_hidden=False, node_limit=None, ctx=None):
return self.entries.get(uri, [])
async def stat(self, uri, ctx=None):