Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: VikingDB grep fallback, exclude URI filtering, and text field byte limit (#2825)
ByteDanceLiuYang39e4180a36eb
diff --git a/openviking/storage/vectordb_adapters/base.py b/openviking/storage/vectordb_adapters/base.pyindex c12843332..126a82468 100644--- a/openviking/storage/vectordb_adapters/base.py+++ b/openviking/storage/vectordb_adapters/base.py@@ -37,7 +37,7 @@ logger = get_logger(__name__)# VikingDB rejects upsert when any text field exceeds this byte length.# Truncation is applied at a valid UTF-8 character boundary so that# multi-byte sequences are never split in the middle.-VIKINGDB_TEXT_FIELD_BYTE_LIMIT: int = 65535+VIKINGDB_TEXT_FIELD_BYTE_LIMIT: int = 1024 * 1024def _truncate_text_field(text: str, byte_limit: int = VIKINGDB_TEXT_FIELD_BYTE_LIMIT) -> str:diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex 0b9e58aca..4de90be41 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -25,13 +25,12 @@ from datetime import datetime, timezonefrom pathlib import PurePathfrom typing import TYPE_CHECKING, Any, Dict, List, Optional, Union+from openviking.core.context import ContextLevelfrom openviking.core.namespace import (- canonical_user_root,canonicalize_uri,is_hidden_by_actor_peer_view,may_include_hidden_actor_peers,)-from openviking.core.context import ContextLevelfrom openviking.core.namespace import (is_accessible as namespace_is_accessible,)@@ -46,7 +45,7 @@ from openviking.pyagfs.exceptions import (from openviking.resource.watch_storage import is_watch_task_control_urifrom openviking.server.error_mapping import is_not_found_error, map_exceptionfrom openviking.server.identity import RequestContext, Role-from openviking.storage.expr import PathScope+from openviking.storage.expr import And, PathScope, RawDSLfrom openviking.storage.internal_names import (MULTIWRITE_PATH_LOCK_FILE,STORAGE_INTERNAL_ENTRY_NAMES,@@ -374,7 +373,7 @@ class VikingFS:if parts and parts[0] == "agent":if len(parts) >= 2 and parts[1] not in {"skills", "endpoints", "tools", "payments"}:raise PermissionDeniedError(- f"viking://agent/{{agent_id}} is deprecated. Use viking://user/.../peers/{{agent_id}} instead.",+ "viking://agent/{agent_id} is deprecated. Use viking://user/.../peers/{agent_id} instead.",resource=normalized_uri,)if len(parts) < 2:@@ -969,6 +968,23 @@ class VikingFS:# will handle the tokenization of the query string.query = " ".join(kw.strip() for kw in pattern.split("|") if kw.strip())filter_expr = PathScope("uri", uri, depth=level_limit)+ excluded_prefix = None+ if exclude_uri:+ excluded_prefix = self._normalize_uri(exclude_uri).rstrip("/")+ self._ensure_access(excluded_prefix, ctx)+ filter_expr = And(+ [+ filter_expr,+ RawDSL(+ {+ "op": "must_not",+ "field": "uri",+ "conds": [excluded_prefix],+ "para": "-d=-1",+ }+ ),+ ]+ )# Auto-adapt bm25 recall limit: recall up to 5x requested matches# while capping at VikingDB's max limit. If node_limit is unset,@@ -977,6 +993,14 @@ class VikingFS:# Step 1: vikingdb recall candidate filestry:+ logger.debug(+ "grep vikingdb search_by_keywords request: query=%r limit=%s filter=%r "+ "output_fields=%s",+ query,+ remote_return_limit,+ filter_expr,+ ["uri"],+ )result = await vector_store.search_by_keywords(query=query,limit=remote_return_limit,@@ -997,8 +1021,12 @@ class VikingFS:)candidate_uris = [r["uri"] for r in result if r.get("uri")]- if exclude_uri:- candidate_uris = [u for u in candidate_uris if not u.startswith(exclude_uri)]+ if excluded_prefix:+ candidate_uris = [+ u+ for u in candidate_uris+ if u != excluded_prefix and not u.startswith(excluded_prefix + "/")+ ]if not candidate_uris:# BM25 returned no candidates — the index confirms no matching contentreturn {"matches": [], "count": 0, "match_count": 0, "files_scanned": 0}@@ -2294,7 +2322,8 @@ class VikingFS:def _is_legacy_agent_id_uri(self, uri: str) -> bool:_, parts = self._normalized_uri_parts(uri)return bool(- parts and parts[0] == "agent"+ parts+ and parts[0] == "agent"and len(parts) >= 2and parts[1] not in {"skills", "endpoints", "tools", "payments"})@@ -2337,15 +2366,9 @@ class VikingFS:# Only legacy namespaces need alias remapping:# - Old format viking://agent/{agent_id}/...# - viking://session/...- is_legacy_namespace = (- request_parts- and (- request_parts[0] == "session"- or (- request_parts[0] == "agent"- and self._is_legacy_agent_id_uri(request_uri)- )- )+ is_legacy_namespace = request_parts and (+ request_parts[0] == "session"+ or (request_parts[0] == "agent" and self._is_legacy_agent_id_uri(request_uri)))if not is_legacy_namespace:return self._path_to_uri(entry_path, ctx=ctx)@@ -3351,14 +3374,10 @@ class VikingFS:canonical = canonicalize_uri(uri, real_ctx)_, parts = self._normalized_uri_parts(canonical)if not parts:- raise ValueError(- f"git tree path cannot be the account root: {uri!r}"- )+ raise ValueError(f"git tree path cannot be the account root: {uri!r}")first = parts[0]if first in self._GIT_INTERNAL_FIRST_SEGMENTS:- raise ValueError(- f"git tree path rejects internal scope/segment {first!r}: {uri!r}"- )+ raise ValueError(f"git tree path rejects internal scope/segment {first!r}: {uri!r}")return "/".join(parts)def _tree_path_to_uri(self, tree_path: str) -> str:@@ -3377,9 +3396,7 @@ class VikingFS:}_NO_VECTOR_DERIVED = frozenset({".relations.json"})- def _classify_restore_path(- self, tree_path: str, *, deleted: bool- ) -> Optional[tuple]:+ def _classify_restore_path(self, tree_path: str, *, deleted: bool) -> Optional[tuple]:"""Classify a restore-affected tree path into a vector maintenance task.Returns a ``(op, uri, level)`` triple, or ``None`` when the path has no@@ -3531,9 +3548,9 @@ class VikingFS:if dry_run:return await self._async_agfs.run("git_restore", **kwargs)+ from openviking.pyagfs.exceptions import GitRestoreWritebackPartialErrorfrom openviking.storage.errors import LockAcquisitionError, ResourceBusyErrorfrom openviking.storage.transaction import LockContext, get_lock_manager- from openviking.pyagfs.exceptions import GitRestoreWritebackPartialError# Serialize the writeback against concurrent VFS mutations on the same# subtree. A scoped restore tree-locks project_dir; a full restore@@ -3603,9 +3620,7 @@ class VikingFS:"[VikingFS] git restore reindex task creation failed; ""falling back to fire-and-forget rebuild")- self._schedule_vector_rebuild(- written=written, deleted=deleted, ctx=real_ctx- )+ self._schedule_vector_rebuild(written=written, deleted=deleted, ctx=real_ctx)return resultasync def _schedule_restore_reindex_for_paths(@@ -3722,14 +3737,20 @@ class VikingFS:real_ctx = self._ctx_or_default(ctx)account = real_ctx.account_idhead = await self._async_agfs.run(- "git_show", account=account, target_ref=branch, path=None,+ "git_show",+ account=account,+ target_ref=branch,+ path=None,)results: List[Dict[str, Any]] = [head]parents = head.get("parents") or []while parents and len(results) < limit:parent_oid = parents[0]commit = await self._async_agfs.run(- "git_show", account=account, target_ref=parent_oid, path=None,+ "git_show",+ account=account,+ target_ref=parent_oid,+ path=None,)results.append(commit)parents = commit.get("parents") or []@@ -3792,9 +3813,7 @@ class VikingFS:try:loop = asyncio.get_running_loop()except RuntimeError:- logger.warning(- "[VikingFS] git restore vector rebuild skipped: no running event loop"- )+ logger.warning("[VikingFS] git restore vector rebuild skipped: no running event loop")returntasks = self._collect_restore_vector_tasks(written, deleted)@@ -3865,18 +3884,12 @@ class VikingFS:"""Wrapper coroutine: dispatch one vector task and swallow errors."""try:if op == "reindex_marker":- await executor.reindex_directory_marker(- dir_uri=uri, level=level, ctx=ctx- )+ await executor.reindex_directory_marker(dir_uri=uri, level=level, ctx=ctx)elif op == "reindex_file":- await executor.execute(- uri=uri, mode="vectors_only", wait=True, ctx=ctx- )+ await executor.execute(uri=uri, mode="vectors_only", wait=True, ctx=ctx)elif op == "delete":await executor.delete_uri_level(uri=uri, level=level, ctx=ctx)else: # pragma: no cover - defensivelogger.warning("[VikingFS] unknown vector rebuild op %r for %s", op, uri)except Exception:- logger.exception(- "[VikingFS] git restore vector task %s failed for %s", op, uri- )+ logger.exception("[VikingFS] git restore vector task %s failed for %s", op, uri)diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex 9e411e650..2b6921fe2 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -610,7 +610,7 @@ class _SingleAccountBackend:)except Exception as e:logger.error("Error searching by keywords: %s", e)- return []+ raiseasync def close(self) -> None:try:diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex 0da2711c3..3023231dd 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -22,6 +22,10 @@ from openviking.storage.expr import Eqfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsgfrom openviking.storage.vectordb import engine as vectordb_enginefrom openviking.storage.vectordb.collection.result import UpsertDataResult+from openviking.storage.vectordb_adapters.base import (+ VIKINGDB_TEXT_FIELD_BYTE_LIMIT,+ _truncate_text_field,+)from openviking.storage.vectordb_adapters.local_adapter import LocalCollectionAdapterfrom openviking.storage.viking_vector_index_backend import (VIKINGDB_CONTENT_MAX_SIZE,@@ -845,6 +849,16 @@ async def test_single_account_backend_truncates_content_only_at_vector_write():assert captured["data"]["content"] == full_content[:VIKINGDB_CONTENT_MAX_SIZE]+def test_vikingdb_text_field_byte_limit_is_one_mb_and_utf8_safe():+ text = "a" * (1024 * 1024) + "😀"++ truncated = _truncate_text_field(text)++ assert VIKINGDB_TEXT_FIELD_BYTE_LIMIT == 1024 * 1024+ assert len(truncated.encode("utf-8")) == VIKINGDB_TEXT_FIELD_BYTE_LIMIT+ assert truncated == "a" * VIKINGDB_TEXT_FIELD_BYTE_LIMIT++@pytest.mark.asyncioasync def test_single_account_backend_collection_exists_runs_in_threadpool(monkeypatch):called = {}diff --git a/tests/storage/test_viking_fs_grep.py b/tests/storage/test_viking_fs_grep.pyindex 062d46297..47aa71ad5 100644--- a/tests/storage/test_viking_fs_grep.py+++ b/tests/storage/test_viking_fs_grep.py@@ -6,6 +6,7 @@ import timeimport pytestimport openviking.storage.viking_fs as viking_fs_module+from openviking.storage.expr import And, PathScope, RawDSLfrom openviking.storage.viking_fs import _DEFAULT_GREP_FILE_CONCURRENCY, VikingFSfrom openviking_cli.utils.config.grep_config import GrepConfig@@ -15,12 +16,18 @@ class _DummyAgfs:class _DummyVectorStore:- def __init__(self):+ def __init__(self, results=None):self.calls = []+ self.results = results or []async def search_by_keywords(self, **kwargs):self.calls.append(kwargs)- return []+ return self.results+++class _FailingVectorStore:+ async def search_by_keywords(self, **kwargs):+ raise RuntimeError("remote keyword search failed")@pytest.fixture@@ -98,6 +105,111 @@ async def test_grep_vikingdb_auto_remote_limit_uses_five_times_node_limit(assert vector_store.calls[0]["limit"] == expected_remote_limit+async def test_grep_vikingdb_remote_error_falls_back_to_fs(monkeypatch):+ fs = VikingFS(agfs=_DummyAgfs())+ monkeypatch.setattr(fs, "_get_vector_store", lambda: _FailingVectorStore())++ calls = []++ async def fake_grep_fs(**kwargs):+ calls.append(kwargs)+ return {+ "matches": [{"uri": "viking://resources/a.md"}],+ "count": 1,+ "match_count": 1,+ "files_scanned": 1,+ }++ monkeypatch.setattr(fs, "_grep_fs", fake_grep_fs)++ result = await fs._grep_vikingdb_then_fs(+ uri="viking://resources",+ pattern="needle",+ exclude_uri="viking://resources/archive",+ case_insensitive=True,+ node_limit=10,+ level_limit=3,+ ctx=None,+ )++ assert result["count"] == 1+ assert calls == [+ {+ "uri": "viking://resources",+ "pattern": "needle",+ "exclude_uri": "viking://resources/archive",+ "case_insensitive": True,+ "node_limit": 10,+ "level_limit": 3,+ "ctx": None,+ }+ ]+++async def test_grep_vikingdb_pushes_exclude_uri_to_filter(monkeypatch):+ fs = VikingFS(agfs=_DummyAgfs())+ vector_store = _DummyVectorStore()+ monkeypatch.setattr(fs, "_get_vector_store", lambda: vector_store)+ monkeypatch.setattr(fs, "_ensure_access", lambda uri, ctx=None: None)++ result = await fs._grep_vikingdb_then_fs(+ uri="viking://resources",+ pattern="needle",+ exclude_uri="viking://resources/archive",+ case_insensitive=False,+ node_limit=10,+ level_limit=3,+ ctx=None,+ )++ assert result == {"matches": [], "count": 0, "match_count": 0, "files_scanned": 0}+ filter_expr = vector_store.calls[0]["filter"]+ assert isinstance(filter_expr, And)+ assert filter_expr.conds[0] == PathScope("uri", "viking://resources", depth=3)+ assert isinstance(filter_expr.conds[1], RawDSL)+ assert filter_expr.conds[1].payload == {+ "op": "must_not",+ "field": "uri",+ "conds": ["viking://resources/archive"],+ "para": "-d=-1",+ }+++async def test_grep_vikingdb_keeps_local_exclude_uri_guard(monkeypatch):+ fs = VikingFS(agfs=_DummyAgfs())+ vector_store = _DummyVectorStore(+ results=[+ {"uri": "viking://resources/archive/a.md"},+ {"uri": "viking://resources/keep.md"},+ ]+ )+ monkeypatch.setattr(fs, "_get_vector_store", lambda: vector_store)+ monkeypatch.setattr(fs, "_ensure_access", lambda uri, ctx=None: None)++ grep_in_files_calls = []++ async def fake_grep_in_files(file_uris, pattern, case_insensitive, node_limit, ctx):+ grep_in_files_calls.append(file_uris)+ return {"matches": [], "count": 0, "match_count": 0, "files_scanned": len(file_uris)}++ monkeypatch.setattr(fs, "_grep_in_files", fake_grep_in_files)++ await fs._grep_vikingdb_then_fs(+ uri="viking://resources",+ pattern="needle",+ exclude_uri="viking://resources/archive",+ case_insensitive=False,+ node_limit=10,+ level_limit=3,+ ctx=None,+ )++ assert grep_in_files_calls == [["viking://resources/keep.md"]]++@pytest.mark.asyncioasync def test_grep_preserves_dfs_order_and_node_limit(monkeypatch):fs = VikingFS(agfs=_DummyAgfs())