Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(ingest): enumerate all children on the ingest path / 入库时枚举目录全部子节点,修复 ls node_limit=1000 静默截断 >1000 文档 (#2518)
Kchen2f214f2e34ab
diff --git a/openviking/parse/parsers/directory.py b/openviking/parse/parsers/directory.pyindex 7044d46ce..47ebf1af1 100644--- a/openviking/parse/parsers/directory.py+++ b/openviking/parse/parsers/directory.py@@ -28,6 +28,7 @@ from openviking.parse.base import ()from openviking.parse.parsers.base_parser import BaseParserfrom openviking.parse.parsers.media.constants import MEDIA_EXTENSIONS+from openviking.storage.viking_fs import LS_ALL_NODESfrom openviking_cli.utils.logger import get_loggerif TYPE_CHECKING:@@ -461,7 +462,7 @@ class DirectoryParser(BaseParser):After the move the source temp is deleted."""- entries = await viking_fs.ls(src_temp_uri)+ entries = await viking_fs.ls(src_temp_uri, node_limit=LS_ALL_NODES)for entry in entries:name = entry.get("name", "")if not name or name in (".", ".."):@@ -530,7 +531,7 @@ class DirectoryParser(BaseParser):) -> None:"""Recursively move a VikingFS directory tree."""await viking_fs.mkdir(dst_uri, exist_ok=True)- entries = await viking_fs.ls(src_uri)+ entries = await viking_fs.ls(src_uri, node_limit=LS_ALL_NODES)for entry in entries:name = entry.get("name", "")if not name or name in (".", ".."):diff --git a/openviking/storage/queuefs/semantic_dag.py b/openviking/storage/queuefs/semantic_dag.pyindex a5b9921a3..9f35c93e7 100644--- a/openviking/storage/queuefs/semantic_dag.py+++ b/openviking/storage/queuefs/semantic_dag.py@@ -11,7 +11,7 @@ from typing import ClassVar, Dict, List, Optional, Setfrom openviking.server.identity import RequestContextfrom openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecarsfrom openviking.storage.transaction import NO_LOCK, LockLease-from openviking.storage.viking_fs import get_viking_fs+from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fsfrom openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking_cli.utils import VikingURIfrom openviking_cli.utils.logger import get_logger@@ -470,7 +470,7 @@ class SemanticDagExecutor:async def _list_dir(self, uri: str, from_hint: str) -> tuple[list[str], list[str]]:"""List directory entries and return (child_dirs, file_paths)."""try:- entries = await self._viking_fs.ls(uri, ctx=self._ctx)+ entries = await self._viking_fs.ls(uri, node_limit=LS_ALL_NODES, ctx=self._ctx)except Exception as e:logger.warning(f"[SemanticDagExecutor] Failed to list directory {uri}: {e} from {from_hint}"diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex fa3bf5f28..a4549663f 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -36,7 +36,7 @@ from openviking.storage.queuefs.semantic_msg import SemanticMsg, build_semantic_from openviking.storage.queuefs.semantic_queue import is_semantic_msg_stalefrom openviking.storage.queuefs.semantic_sidecar import write_semantic_sidecarsfrom openviking.storage.transaction import NO_LOCK, LockLease-from openviking.storage.viking_fs import get_viking_fs+from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fsfrom openviking.telemetry import bind_telemetry, bind_telemetry_stage, resolve_telemetryfrom openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking.telemetry.span_models import create_root_span_attributes@@ -538,7 +538,7 @@ class SemanticProcessor(DequeueHandlerBase):try:try:- entries = await viking_fs.ls(dir_uri, ctx=ctx)+ entries = await viking_fs.ls(dir_uri, node_limit=LS_ALL_NODES, ctx=ctx)except Exception as e:raise RuntimeError(f"Failed to list memory directory {dir_uri}: {e}") from e@@ -749,7 +749,9 @@ class SemanticProcessor(DequeueHandlerBase):files: Dict[str, str] = {}dirs: Dict[str, str] = {}try:- entries = await viking_fs.ls(dir_uri, show_all_hidden=True, ctx=ctx)+ entries = await viking_fs.ls(+ dir_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx+ )except Exception as e:logger.error(f"[SyncDiff] Failed to list {dir_uri}: {e}")return files, dirsdiff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex 3a46653a5..79643650c 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -63,6 +63,15 @@ if TYPE_CHECKING:logger = get_logger(__name__)+# Sentinel node_limit for internal callers that MUST enumerate an entire+# directory. ``ls()`` defaults to ``node_limit=1000`` to protect agent-facing+# context from being flooded, but internal system operations (parse merge,+# temp->final sync, summary DAG, vectorization) must see every child or they+# silently drop entries beyond the cap — e.g. a >1000-doc directory ingest only+# materializes its first 1000 subdirectories. Pass this explicitly at those+# call sites.+LS_ALL_NODES = 2**31 - 1+def _ensure_non_empty_search_query(query: str) -> None:if not query.strip():diff --git a/openviking/utils/embedding_utils.py b/openviking/utils/embedding_utils.pyindex 54b1f46ee..e29a42137 100644--- a/openviking/utils/embedding_utils.py+++ b/openviking/utils/embedding_utils.py@@ -16,7 +16,7 @@ from openviking.core.namespace import context_type_for_uri, owner_space_for_urifrom openviking.server.identity import RequestContextfrom openviking.storage.queuefs import get_queue_managerfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter-from openviking.storage.viking_fs import get_viking_fs+from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fsfrom openviking.utils.time_utils import parse_iso_datetimefrom openviking_cli.utils import VikingURI, get_loggerfrom openviking_cli.utils.config import get_openviking_config@@ -507,7 +507,7 @@ async def index_resource(# 2. Index Filestry:- files = await viking_fs.ls(uri, ctx=ctx)+ files = await viking_fs.ls(uri, node_limit=LS_ALL_NODES, ctx=ctx)for file_info in files:file_name = file_info["name"]diff --git a/openviking/utils/summarizer.py b/openviking/utils/summarizer.pyindex 744034129..88051a85a 100644--- a/openviking/utils/summarizer.py+++ b/openviking/utils/summarizer.py@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuplefrom openviking.core.namespace import context_type_for_urifrom openviking.storage.queuefs import SemanticMsg, get_queue_managerfrom openviking.storage.transaction import NO_LOCK, LockLease-from openviking.storage.viking_fs import get_viking_fs+from openviking.storage.viking_fs import LS_ALL_NODES, get_viking_fsfrom openviking.telemetry import get_current_telemetryfrom openviking.telemetry.request_wait_tracker import get_request_wait_trackerfrom openviking_cli.utils import get_logger@@ -81,7 +81,9 @@ class Summarizer:async def list_top_children(temp_uri: str) -> List[Tuple[str, str]]:viking_fs = get_viking_fs()- entries = await viking_fs.ls(temp_uri, show_all_hidden=True, ctx=ctx)+ entries = await viking_fs.ls(+ temp_uri, show_all_hidden=True, node_limit=LS_ALL_NODES, ctx=ctx+ )children: List[Tuple[str, str]] = []for entry in entries:name = entry.get("name", "")diff --git a/tests/storage/test_ingest_ls_node_limit.py b/tests/storage/test_ingest_ls_node_limit.pynew file mode 100644index 000000000..b4c8d4401--- /dev/null+++ b/tests/storage/test_ingest_ls_node_limit.py@@ -0,0 +1,125 @@+"""Regression: internal directory enumeration during ingest must not be capped+at ``viking_fs.ls``'s default ``node_limit=1000``.++When a resource namespace already exists, re-ingesting a directory takes the+incremental sync path (``_sync_topdown_recursive`` -> ``sync_dir`` ->+``list_children``), which lists the temp root's children via ``viking_fs.ls``.+``ls`` defaults ``node_limit=1000``, so an import of >1000 docs silently+materialized only the first 1000 subdirectories into the target; the rest were+dropped when temp was deleted.++Observed in the wild: a 6,221-doc WixQA re-ingest (the namespace already held an+earlier 209-doc import, so the atomic-move fast path was skipped) produced+exactly 1000 directories under ``resources/wixqa``.+"""++from __future__ import annotations++import pytest++from openviking.storage.queuefs.semantic_processor import SemanticProcessor+from openviking.storage.transaction import NO_LOCK+++class _TruncatingVikingFS:+ """Fake whose ``ls`` truncates at ``node_limit``, exactly like the real one.++ The temp root holds ``n_children`` doc subdirectories; the target namespace+ pre-exists but is empty (forcing the incremental ``sync_dir`` path instead of+ the atomic whole-directory move).+ """++ TEMP = "viking://temp/import"+ TARGET = "viking://resources/wixqa"++ def __init__(self, n_children: int):+ self._children = [+ {"name": f"doc{i:05d}", "isDir": True} for i in range(n_children)+ ]+ self.moved: list[tuple[str, str]] = []++ async def exists(self, uri, ctx=None):+ return uri in (self.TEMP, self.TARGET)++ async def ls(self, uri, show_all_hidden=False, node_limit=1000, ctx=None):+ entries = self._children if uri == self.TEMP else []+ return entries[:node_limit] # the real ls truncates here++ async def mv(self, src, dst, ctx=None, lock_handle=None):+ self.moved.append((src, dst))++ async def rm(self, uri, recursive=False, ctx=None, lock_handle=None):+ pass++ async def mkdir(self, uri, exist_ok=False, ctx=None):+ pass++ async def delete_temp(self, uri, ctx=None):+ pass+++async def test_sync_materializes_all_children_above_default_node_limit(monkeypatch):+ n_children = 1500 # > the default node_limit of 1000+ fake = _TruncatingVikingFS(n_children)+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_processor.get_viking_fs",+ lambda: fake,+ )++ diff = await SemanticProcessor()._sync_topdown_recursive(+ _TruncatingVikingFS.TEMP,+ _TruncatingVikingFS.TARGET,+ lock=NO_LOCK,+ )++ assert len(fake.moved) == n_children, (+ f"only {len(fake.moved)}/{n_children} doc dirs materialized — internal "+ "sync was truncated at ls node_limit"+ )+ assert len(diff.added_dirs) == n_children+++class _TruncatingLsFS:+ """Minimal fake exposing only a node_limit-truncating ``ls`` over a fixed+ set of child directories."""++ def __init__(self, dir_uri: str, n_children: int):+ self._dir_uri = dir_uri+ self._children = [+ {"name": f"doc{i:05d}", "isDir": True} for i in range(n_children)+ ]++ async def ls(self, uri, node_limit=1000, ctx=None):+ entries = self._children if uri == self._dir_uri else []+ return entries[:node_limit] # the real ls truncates here+++async def test_semantic_dag_list_dir_enumerates_all_children(monkeypatch):+ """The summary DAG dispatches one summary/recursion task per child it lists.+ If ``_list_dir`` is capped at node_limit, children beyond 1000 never get an+ ``.abstract.md``/``.overview.md`` (and are never recursed into), so a large+ namespace silently loses its L0/L1 layers."""+ from openviking.storage.queuefs.semantic_dag import SemanticDagExecutor++ dir_uri = "viking://resources/wixqa"+ n_children = 1500 # > the default node_limit of 1000+ fake = _TruncatingLsFS(dir_uri, n_children)+ monkeypatch.setattr(+ "openviking.storage.queuefs.semantic_dag.get_viking_fs",+ lambda: fake,+ )++ executor = SemanticDagExecutor(+ processor=None,+ context_type="resource",+ max_concurrent_llm=1,+ ctx=None,+ )+ children_dirs, file_paths = await executor._list_dir(dir_uri, from_hint="test")++ assert len(children_dirs) == n_children, (+ f"only {len(children_dirs)}/{n_children} children listed — summary DAG "+ "was truncated at ls node_limit"+ )diff --git a/tests/storage/test_semantic_dag_incremental.py b/tests/storage/test_semantic_dag_incremental.pyindex ee45577c1..60d165599 100644--- a/tests/storage/test_semantic_dag_incremental.py+++ b/tests/storage/test_semantic_dag_incremental.py@@ -40,7 +40,7 @@ class _FakeVikingFS:rest = re.sub(r"/{2,}", "/", rest)return f"{scheme}://{rest}"- async def ls(self, uri, ctx=None):+ async def ls(self, uri, node_limit=None, ctx=None):return self._tree.get(self._norm(uri), [])async def stat(self, uri, ctx=None):diff --git a/tests/storage/test_semantic_dag_skip_files.py b/tests/storage/test_semantic_dag_skip_files.pyindex 7f2fd155e..70bae748d 100644--- a/tests/storage/test_semantic_dag_skip_files.py+++ b/tests/storage/test_semantic_dag_skip_files.py@@ -32,7 +32,7 @@ class _FakeVikingFS:self._tree = treeself.writes = []- async def ls(self, uri, ctx=None):+ async def ls(self, uri, node_limit=None, ctx=None):return self._tree.get(uri, [])async def write_file(self, path, content, ctx=None):diff --git a/tests/storage/test_semantic_processor_target_preexisting.py b/tests/storage/test_semantic_processor_target_preexisting.pyindex 4bbe08110..2546509a3 100644--- a/tests/storage/test_semantic_processor_target_preexisting.py+++ b/tests/storage/test_semantic_processor_target_preexisting.py@@ -40,7 +40,7 @@ class _SyncVikingFS:async def exists(self, uri, ctx=None):return uri in self.entries- async def ls(self, uri, show_all_hidden=False, ctx=None):+ async def ls(self, uri, show_all_hidden=False, node_limit=None, ctx=None):return self.entries.get(uri, [])async def stat(self, uri, ctx=None):