Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: VikingDB grep fallback, exclude URI filtering, and text field byte limit (#2825)

ByteDanceLiuYang39e4180a36eb
diff --git a/openviking/storage/vectordb_adapters/base.py b/openviking/storage/vectordb_adapters/base.py
index c12843332..126a82468 100644
--- a/openviking/storage/vectordb_adapters/base.py
+++ b/openviking/storage/vectordb_adapters/base.py
@@ -37,7 +37,7 @@ logger = get_logger(__name__)
# VikingDB rejects upsert when any text field exceeds this byte length.
# Truncation is applied at a valid UTF-8 character boundary so that
# multi-byte sequences are never split in the middle.
-VIKINGDB_TEXT_FIELD_BYTE_LIMIT: int = 65535
+VIKINGDB_TEXT_FIELD_BYTE_LIMIT: int = 1024 * 1024
def _truncate_text_field(text: str, byte_limit: int = VIKINGDB_TEXT_FIELD_BYTE_LIMIT) -> str:
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 0b9e58aca..4de90be41 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -25,13 +25,12 @@ from datetime import datetime, timezone
from pathlib import PurePath
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
+from openviking.core.context import ContextLevel
from openviking.core.namespace import (
- canonical_user_root,
canonicalize_uri,
is_hidden_by_actor_peer_view,
may_include_hidden_actor_peers,
)
-from openviking.core.context import ContextLevel
from openviking.core.namespace import (
is_accessible as namespace_is_accessible,
)
@@ -46,7 +45,7 @@ from openviking.pyagfs.exceptions import (
from openviking.resource.watch_storage import is_watch_task_control_uri
from openviking.server.error_mapping import is_not_found_error, map_exception
from openviking.server.identity import RequestContext, Role
-from openviking.storage.expr import PathScope
+from openviking.storage.expr import And, PathScope, RawDSL
from openviking.storage.internal_names import (
MULTIWRITE_PATH_LOCK_FILE,
STORAGE_INTERNAL_ENTRY_NAMES,
@@ -374,7 +373,7 @@ class VikingFS:
if parts and parts[0] == "agent":
if len(parts) >= 2 and parts[1] not in {"skills", "endpoints", "tools", "payments"}:
raise PermissionDeniedError(
- f"viking://agent/{{agent_id}} is deprecated. Use viking://user/.../peers/{{agent_id}} instead.",
+ "viking://agent/{agent_id} is deprecated. Use viking://user/.../peers/{agent_id} instead.",
resource=normalized_uri,
)
if len(parts) < 2:
@@ -969,6 +968,23 @@ class VikingFS:
# will handle the tokenization of the query string.
query = " ".join(kw.strip() for kw in pattern.split("|") if kw.strip())
filter_expr = PathScope("uri", uri, depth=level_limit)
+ excluded_prefix = None
+ if exclude_uri:
+ excluded_prefix = self._normalize_uri(exclude_uri).rstrip("/")
+ self._ensure_access(excluded_prefix, ctx)
+ filter_expr = And(
+ [
+ filter_expr,
+ RawDSL(
+ {
+ "op": "must_not",
+ "field": "uri",
+ "conds": [excluded_prefix],
+ "para": "-d=-1",
+ }
+ ),
+ ]
+ )
# Auto-adapt bm25 recall limit: recall up to 5x requested matches
# while capping at VikingDB's max limit. If node_limit is unset,
@@ -977,6 +993,14 @@ class VikingFS:
# Step 1: vikingdb recall candidate files
try:
+ logger.debug(
+ "grep vikingdb search_by_keywords request: query=%r limit=%s filter=%r "
+ "output_fields=%s",
+ query,
+ remote_return_limit,
+ filter_expr,
+ ["uri"],
+ )
result = await vector_store.search_by_keywords(
query=query,
limit=remote_return_limit,
@@ -997,8 +1021,12 @@ class VikingFS:
)
candidate_uris = [r["uri"] for r in result if r.get("uri")]
- if exclude_uri:
- candidate_uris = [u for u in candidate_uris if not u.startswith(exclude_uri)]
+ if excluded_prefix:
+ candidate_uris = [
+ u
+ for u in candidate_uris
+ if u != excluded_prefix and not u.startswith(excluded_prefix + "/")
+ ]
if not candidate_uris:
# BM25 returned no candidates — the index confirms no matching content
return {"matches": [], "count": 0, "match_count": 0, "files_scanned": 0}
@@ -2294,7 +2322,8 @@ class VikingFS:
def _is_legacy_agent_id_uri(self, uri: str) -> bool:
_, parts = self._normalized_uri_parts(uri)
return bool(
- parts and parts[0] == "agent"
+ parts
+ and parts[0] == "agent"
and len(parts) >= 2
and parts[1] not in {"skills", "endpoints", "tools", "payments"}
)
@@ -2337,15 +2366,9 @@ class VikingFS:
# Only legacy namespaces need alias remapping:
# - Old format viking://agent/{agent_id}/...
# - viking://session/...
- is_legacy_namespace = (
- request_parts
- and (
- request_parts[0] == "session"
- or (
- request_parts[0] == "agent"
- and self._is_legacy_agent_id_uri(request_uri)
- )
- )
+ is_legacy_namespace = request_parts and (
+ request_parts[0] == "session"
+ or (request_parts[0] == "agent" and self._is_legacy_agent_id_uri(request_uri))
)
if not is_legacy_namespace:
return self._path_to_uri(entry_path, ctx=ctx)
@@ -3351,14 +3374,10 @@ class VikingFS:
canonical = canonicalize_uri(uri, real_ctx)
_, parts = self._normalized_uri_parts(canonical)
if not parts:
- raise ValueError(
- f"git tree path cannot be the account root: {uri!r}"
- )
+ raise ValueError(f"git tree path cannot be the account root: {uri!r}")
first = parts[0]
if first in self._GIT_INTERNAL_FIRST_SEGMENTS:
- raise ValueError(
- f"git tree path rejects internal scope/segment {first!r}: {uri!r}"
- )
+ raise ValueError(f"git tree path rejects internal scope/segment {first!r}: {uri!r}")
return "/".join(parts)
def _tree_path_to_uri(self, tree_path: str) -> str:
@@ -3377,9 +3396,7 @@ class VikingFS:
}
_NO_VECTOR_DERIVED = frozenset({".relations.json"})
- def _classify_restore_path(
- self, tree_path: str, *, deleted: bool
- ) -> Optional[tuple]:
+ def _classify_restore_path(self, tree_path: str, *, deleted: bool) -> Optional[tuple]:
"""Classify a restore-affected tree path into a vector maintenance task.
Returns a ``(op, uri, level)`` triple, or ``None`` when the path has no
@@ -3531,9 +3548,9 @@ class VikingFS:
if dry_run:
return await self._async_agfs.run("git_restore", **kwargs)
+ from openviking.pyagfs.exceptions import GitRestoreWritebackPartialError
from openviking.storage.errors import LockAcquisitionError, ResourceBusyError
from openviking.storage.transaction import LockContext, get_lock_manager
- from openviking.pyagfs.exceptions import GitRestoreWritebackPartialError
# Serialize the writeback against concurrent VFS mutations on the same
# subtree. A scoped restore tree-locks project_dir; a full restore
@@ -3603,9 +3620,7 @@ class VikingFS:
"[VikingFS] git restore reindex task creation failed; "
"falling back to fire-and-forget rebuild"
)
- self._schedule_vector_rebuild(
- written=written, deleted=deleted, ctx=real_ctx
- )
+ self._schedule_vector_rebuild(written=written, deleted=deleted, ctx=real_ctx)
return result
async def _schedule_restore_reindex_for_paths(
@@ -3722,14 +3737,20 @@ class VikingFS:
real_ctx = self._ctx_or_default(ctx)
account = real_ctx.account_id
head = await self._async_agfs.run(
- "git_show", account=account, target_ref=branch, path=None,
+ "git_show",
+ account=account,
+ target_ref=branch,
+ path=None,
)
results: List[Dict[str, Any]] = [head]
parents = head.get("parents") or []
while parents and len(results) < limit:
parent_oid = parents[0]
commit = await self._async_agfs.run(
- "git_show", account=account, target_ref=parent_oid, path=None,
+ "git_show",
+ account=account,
+ target_ref=parent_oid,
+ path=None,
)
results.append(commit)
parents = commit.get("parents") or []
@@ -3792,9 +3813,7 @@ class VikingFS:
try:
loop = asyncio.get_running_loop()
except RuntimeError:
- logger.warning(
- "[VikingFS] git restore vector rebuild skipped: no running event loop"
- )
+ logger.warning("[VikingFS] git restore vector rebuild skipped: no running event loop")
return
tasks = self._collect_restore_vector_tasks(written, deleted)
@@ -3865,18 +3884,12 @@ class VikingFS:
"""Wrapper coroutine: dispatch one vector task and swallow errors."""
try:
if op == "reindex_marker":
- await executor.reindex_directory_marker(
- dir_uri=uri, level=level, ctx=ctx
- )
+ await executor.reindex_directory_marker(dir_uri=uri, level=level, ctx=ctx)
elif op == "reindex_file":
- await executor.execute(
- uri=uri, mode="vectors_only", wait=True, ctx=ctx
- )
+ await executor.execute(uri=uri, mode="vectors_only", wait=True, ctx=ctx)
elif op == "delete":
await executor.delete_uri_level(uri=uri, level=level, ctx=ctx)
else: # pragma: no cover - defensive
logger.warning("[VikingFS] unknown vector rebuild op %r for %s", op, uri)
except Exception:
- logger.exception(
- "[VikingFS] git restore vector task %s failed for %s", op, uri
- )
+ logger.exception("[VikingFS] git restore vector task %s failed for %s", op, uri)
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index 9e411e650..2b6921fe2 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -610,7 +610,7 @@ class _SingleAccountBackend:
)
except Exception as e:
logger.error("Error searching by keywords: %s", e)
- return []
+ raise
async def close(self) -> None:
try:
diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py
index 0da2711c3..3023231dd 100644
--- a/tests/storage/test_collection_schemas.py
+++ b/tests/storage/test_collection_schemas.py
@@ -22,6 +22,10 @@ from openviking.storage.expr import Eq
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
from openviking.storage.vectordb import engine as vectordb_engine
from openviking.storage.vectordb.collection.result import UpsertDataResult
+from openviking.storage.vectordb_adapters.base import (
+ VIKINGDB_TEXT_FIELD_BYTE_LIMIT,
+ _truncate_text_field,
+)
from openviking.storage.vectordb_adapters.local_adapter import LocalCollectionAdapter
from openviking.storage.viking_vector_index_backend import (
VIKINGDB_CONTENT_MAX_SIZE,
@@ -845,6 +849,16 @@ async def test_single_account_backend_truncates_content_only_at_vector_write():
assert captured["data"]["content"] == full_content[:VIKINGDB_CONTENT_MAX_SIZE]
+def test_vikingdb_text_field_byte_limit_is_one_mb_and_utf8_safe():
+ text = "a" * (1024 * 1024) + "😀"
+
+ truncated = _truncate_text_field(text)
+
+ assert VIKINGDB_TEXT_FIELD_BYTE_LIMIT == 1024 * 1024
+ assert len(truncated.encode("utf-8")) == VIKINGDB_TEXT_FIELD_BYTE_LIMIT
+ assert truncated == "a" * VIKINGDB_TEXT_FIELD_BYTE_LIMIT
+
+
@pytest.mark.asyncio
async def test_single_account_backend_collection_exists_runs_in_threadpool(monkeypatch):
called = {}
diff --git a/tests/storage/test_viking_fs_grep.py b/tests/storage/test_viking_fs_grep.py
index 062d46297..47aa71ad5 100644
--- a/tests/storage/test_viking_fs_grep.py
+++ b/tests/storage/test_viking_fs_grep.py
@@ -6,6 +6,7 @@ import time
import pytest
import openviking.storage.viking_fs as viking_fs_module
+from openviking.storage.expr import And, PathScope, RawDSL
from openviking.storage.viking_fs import _DEFAULT_GREP_FILE_CONCURRENCY, VikingFS
from openviking_cli.utils.config.grep_config import GrepConfig
@@ -15,12 +16,18 @@ class _DummyAgfs:
class _DummyVectorStore:
- def __init__(self):
+ def __init__(self, results=None):
self.calls = []
+ self.results = results or []
async def search_by_keywords(self, **kwargs):
self.calls.append(kwargs)
- return []
+ return self.results
+
+
+class _FailingVectorStore:
+ async def search_by_keywords(self, **kwargs):
+ raise RuntimeError("remote keyword search failed")
@pytest.fixture
@@ -98,6 +105,111 @@ async def test_grep_vikingdb_auto_remote_limit_uses_five_times_node_limit(
assert vector_store.calls[0]["limit"] == expected_remote_limit
+async def test_grep_vikingdb_remote_error_falls_back_to_fs(monkeypatch):
+ fs = VikingFS(agfs=_DummyAgfs())
+ monkeypatch.setattr(fs, "_get_vector_store", lambda: _FailingVectorStore())
+
+ calls = []
+
+ async def fake_grep_fs(**kwargs):
+ calls.append(kwargs)
+ return {
+ "matches": [{"uri": "viking://resources/a.md"}],
+ "count": 1,
+ "match_count": 1,
+ "files_scanned": 1,
+ }
+
+ monkeypatch.setattr(fs, "_grep_fs", fake_grep_fs)
+
+ result = await fs._grep_vikingdb_then_fs(
+ uri="viking://resources",
+ pattern="needle",
+ exclude_uri="viking://resources/archive",
+ case_insensitive=True,
+ node_limit=10,
+ level_limit=3,
+ ctx=None,
+ )
+
+ assert result["count"] == 1
+ assert calls == [
+ {
+ "uri": "viking://resources",
+ "pattern": "needle",
+ "exclude_uri": "viking://resources/archive",
+ "case_insensitive": True,
+ "node_limit": 10,
+ "level_limit": 3,
+ "ctx": None,
+ }
+ ]
+
+
+async def test_grep_vikingdb_pushes_exclude_uri_to_filter(monkeypatch):
+ fs = VikingFS(agfs=_DummyAgfs())
+ vector_store = _DummyVectorStore()
+ monkeypatch.setattr(fs, "_get_vector_store", lambda: vector_store)
+ monkeypatch.setattr(fs, "_ensure_access", lambda uri, ctx=None: None)
+
+ result = await fs._grep_vikingdb_then_fs(
+ uri="viking://resources",
+ pattern="needle",
+ exclude_uri="viking://resources/archive",
+ case_insensitive=False,
+ node_limit=10,
+ level_limit=3,
+ ctx=None,
+ )
+
+ assert result == {"matches": [], "count": 0, "match_count": 0, "files_scanned": 0}
+ filter_expr = vector_store.calls[0]["filter"]
+ assert isinstance(filter_expr, And)
+ assert filter_expr.conds[0] == PathScope("uri", "viking://resources", depth=3)
+ assert isinstance(filter_expr.conds[1], RawDSL)
+ assert filter_expr.conds[1].payload == {
+ "op": "must_not",
+ "field": "uri",
+ "conds": ["viking://resources/archive"],
+ "para": "-d=-1",
+ }
+
+
+async def test_grep_vikingdb_keeps_local_exclude_uri_guard(monkeypatch):
+ fs = VikingFS(agfs=_DummyAgfs())
+ vector_store = _DummyVectorStore(
+ results=[
+ {"uri": "viking://resources/archive/a.md"},
+ {"uri": "viking://resources/keep.md"},
+ ]
+ )
+ monkeypatch.setattr(fs, "_get_vector_store", lambda: vector_store)
+ monkeypatch.setattr(fs, "_ensure_access", lambda uri, ctx=None: None)
+
+ grep_in_files_calls = []
+
+ async def fake_grep_in_files(file_uris, pattern, case_insensitive, node_limit, ctx):
+ grep_in_files_calls.append(file_uris)
+ return {"matches": [], "count": 0, "match_count": 0, "files_scanned": len(file_uris)}
+
+ monkeypatch.setattr(fs, "_grep_in_files", fake_grep_in_files)
+
+ await fs._grep_vikingdb_then_fs(
+ uri="viking://resources",
+ pattern="needle",
+ exclude_uri="viking://resources/archive",
+ case_insensitive=False,
+ node_limit=10,
+ level_limit=3,
+ ctx=None,
+ )
+
+ assert grep_in_files_calls == [["viking://resources/keep.md"]]
+
+
@pytest.mark.asyncio
async def test_grep_preserves_dfs_order_and_node_limit(monkeypatch):
fs = VikingFS(agfs=_DummyAgfs())