Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(storage): enforce remote VikingDB field limits (#5101)
ByteDanceLiuYang25a17fa228f7
diff --git a/openviking/storage/vectordb_adapters/base.py b/openviking/storage/vectordb_adapters/base.pyindex bfb79e2b0..b0a42dd56 100644--- a/openviking/storage/vectordb_adapters/base.py+++ b/openviking/storage/vectordb_adapters/base.py@@ -35,11 +35,12 @@ from openviking_cli.utils.config.vectordb_config import DEFAULT_INDEX_NAMElogger = get_logger(__name__)# ----------------------------------------------------------------------------# VikingDB text field byte limit+# VikingDB field byte limits# ----------------------------------------------------------------------------# VikingDB rejects upsert when any text field exceeds this byte length.+# VikingDB string fields use a uint16 byte length, while text fields allow 1 MiB.# Truncation is applied at a valid UTF-8 character boundary so that# multi-byte sequences are never split in the middle.+VIKINGDB_STRING_FIELD_BYTE_LIMIT: int = 64 * 1024VIKINGDB_TEXT_FIELD_BYTE_LIMIT: int = 1024 * 1024@@ -102,11 +103,15 @@ class CollectionAdapter(ABC):mode: str_URI_FIELD_NAMES = {"uri", "parent_uri"}- # Text fields subject to byte-limit truncation before upsert.- _TRUNCATABLE_TEXT_FIELDS: tuple[str, ...] = ("content", "abstract")+ # Only derived fields may be shortened silently. An oversized abstract is+ # stored as a prefix, so an exact-match filter using the original full+ # abstract will not match the stored value.+ _TRUNCATABLE_STRING_FIELDS: tuple[str, ...] = ("abstract",)+ _TRUNCATABLE_TEXT_FIELDS: tuple[str, ...] = ("content",)- # Per-backend byte limit for text fields. ``None`` means no truncation.- # Subclasses backed by VikingDB should set this to ``VIKINGDB_TEXT_FIELD_BYTE_LIMIT``.+ # Per-backend byte limits. ``None`` means no truncation. VikingDB-backed+ # adapters set both limits; local adapters keep the complete values.+ _STRING_FIELD_BYTE_LIMIT: int | None = None_TEXT_FIELD_BYTE_LIMIT: int | None = None# Whether this backend actually stores the ``content`` (full text) field.@@ -281,6 +286,11 @@ class CollectionAdapter(ABC):value = normalized.get(field)if isinstance(value, str):normalized[field] = _truncate_text_field(value, self._TEXT_FIELD_BYTE_LIMIT)+ if self._STRING_FIELD_BYTE_LIMIT is not None:+ for field in self._TRUNCATABLE_STRING_FIELDS:+ value = normalized.get(field)+ if isinstance(value, str):+ normalized[field] = _truncate_text_field(value, self._STRING_FIELD_BYTE_LIMIT)return normalized@staticmethoddiff --git a/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py b/openviking/storage/vectordb_adapters/vikingdb_private_adapter.pyindex cc3cfc703..60d3a54da 100644--- a/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py+++ b/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py@@ -4,19 +4,24 @@from __future__ import annotations-from typing import Any, Dict, Optional+from typing import Any, Dict, List, Optionalfrom openviking.storage.vectordb.collection.collection import Collectionfrom openviking.storage.vectordb.collection.vikingdb_clients import VIKINGDB_APIS, VikingDBClientfrom openviking.storage.vectordb.collection.vikingdb_collection import VikingDBCollection-from .base import VIKINGDB_TEXT_FIELD_BYTE_LIMIT, CollectionAdapter+from .base import (+ VIKINGDB_STRING_FIELD_BYTE_LIMIT,+ VIKINGDB_TEXT_FIELD_BYTE_LIMIT,+ CollectionAdapter,+)class VikingDBPrivateCollectionAdapter(CollectionAdapter):"""Adapter for private VikingDB deployment."""_DATA_BATCH_SIZE = 100+ _STRING_FIELD_BYTE_LIMIT = VIKINGDB_STRING_FIELD_BYTE_LIMIT_TEXT_FIELD_BYTE_LIMIT = VIKINGDB_TEXT_FIELD_BYTE_LIMITUSE_CONTENT_FIELD = True@@ -119,3 +124,17 @@ class VikingDBPrivateCollectionAdapter(CollectionAdapter):def _normalize_record_for_read(self, record: Dict[str, Any]) -> Dict[str, Any]:return super()._normalize_record_for_read(record)++ def update_data(self, data_list: List[Dict[str, Any]]):+ data_list = [self._normalize_record_for_write(item) for item in data_list]+ result = self.get_collection().update_data(data_list)+ if isinstance(result, dict):+ for key in ("primary_keys", "ids"):+ values = result.get(key)+ if isinstance(values, list):+ return [str(item) for item in values if item is not None]+ if result.get("updated") == 0:+ return []+ if isinstance(result, list):+ return [str(item) for item in result if item is not None]+ return [str(item["id"]) for item in data_list if item.get("id") is not None]diff --git a/openviking/storage/vectordb_adapters/volcengine_adapter.py b/openviking/storage/vectordb_adapters/volcengine_adapter.pyindex 56dfed1c0..9c93a4b4a 100644--- a/openviking/storage/vectordb_adapters/volcengine_adapter.py+++ b/openviking/storage/vectordb_adapters/volcengine_adapter.py@@ -15,13 +15,18 @@ from openviking.storage.vectordb.collection.volcengine_collection import (get_or_create_volcengine_collection,)-from .base import VIKINGDB_TEXT_FIELD_BYTE_LIMIT, CollectionAdapter+from .base import (+ VIKINGDB_STRING_FIELD_BYTE_LIMIT,+ VIKINGDB_TEXT_FIELD_BYTE_LIMIT,+ CollectionAdapter,+)class VolcengineCollectionAdapter(CollectionAdapter):"""Adapter for Volcengine-hosted VikingDB."""_DATA_BATCH_SIZE = 100+ _STRING_FIELD_BYTE_LIMIT = VIKINGDB_STRING_FIELD_BYTE_LIMIT_TEXT_FIELD_BYTE_LIMIT = VIKINGDB_TEXT_FIELD_BYTE_LIMITUSE_CONTENT_FIELD = True@@ -181,6 +186,7 @@ class VolcengineCollectionAdapter(CollectionAdapter):return super()._normalize_record_for_read(record)def update_data(self, data_list: List[Dict[str, Any]]):+ data_list = [self._normalize_record_for_write(item) for item in data_list]collection = self.get_collection()result = collection.update_data(data_list)if isinstance(result, dict):diff --git a/tests/storage/test_volcengine_clients.py b/tests/storage/test_volcengine_clients.pyindex 1b33dd0b6..d715784f1 100644--- a/tests/storage/test_volcengine_clients.py+++ b/tests/storage/test_volcengine_clients.py@@ -1,3 +1,4 @@+import pytestimport requestsfrom volcengine.base.Request import Request@@ -7,6 +8,11 @@ from openviking.storage.vectordb.collection.volcengine_clients import (ClientForDataApiWithApiKey,)from openviking.storage.vectordb.collection.volcengine_collection import VolcengineCollection+from openviking.storage.vectordb_adapters.base import VIKINGDB_STRING_FIELD_BYTE_LIMIT+from openviking.storage.vectordb_adapters.local_adapter import LocalCollectionAdapter+from openviking.storage.vectordb_adapters.vikingdb_private_adapter import (+ VikingDBPrivateCollectionAdapter,+)from openviking.storage.vectordb_adapters.volcengine_adapter import VolcengineCollectionAdapterfrom openviking_cli.utils.config.vectordb_config import (VectorDBBackendConfig,@@ -660,6 +666,84 @@ def test_volcengine_adapter_update_data_returns_ids():assert result == ["doc-1"][email protected]("operation", ["upsert", "update_data"])+def test_volcengine_adapter_truncates_abstract_at_remote_string_limit(operation):+ adapter = VolcengineCollectionAdapter(+ ak="test-ak",+ sk="test-sk",+ region="cn-beijing",+ session_token=None,+ api_key=None,+ host=None,+ project_name="default",+ collection_name="context",+ index_name="default",+ )+ captured = {}++ class _Collection:+ def upsert_data(self, data_list, ttl=0):+ captured["data"] = data_list+ return None++ def update_data(self, data_list):+ captured["data"] = data_list+ return {"updated": 1, "primary_keys": ["doc-1"]}++ adapter._collection = _Collection()+ abstract = "a" * (VIKINGDB_STRING_FIELD_BYTE_LIMIT - 3) + "你好"++ getattr(adapter, operation)([{"id": "doc-1", "abstract": abstract}])++ stored = captured["data"][0]["abstract"]+ assert len(stored.encode("utf-8")) <= VIKINGDB_STRING_FIELD_BYTE_LIMIT+ assert abstract.startswith(stored)+ assert stored.endswith("你")+++def test_private_vikingdb_update_normalizes_remote_fields():+ adapter = VikingDBPrivateCollectionAdapter(+ host="unused.invalid",+ headers=None,+ project_name="default",+ collection_name="context",+ index_name="default",+ )+ captured = {}++ class _Collection:+ def update_data(self, data_list):+ captured["data"] = data_list+ return {"updated": 1, "primary_keys": ["doc-1"]}++ adapter._collection = _Collection()+ result = adapter.update_data(+ [+ {+ "id": "doc-1",+ "uri": "viking://resources/sample",+ "abstract": "😀" * VIKINGDB_STRING_FIELD_BYTE_LIMIT,+ }+ ]+ )++ stored = captured["data"][0]+ assert stored["uri"] == "/resources/sample"+ assert len(stored["abstract"].encode("utf-8")) <= VIKINGDB_STRING_FIELD_BYTE_LIMIT+ assert result == ["doc-1"]+++def test_local_adapter_does_not_apply_remote_abstract_limit():+ adapter = LocalCollectionAdapter(+ collection_name="context", project_path="", index_name="default"+ )+ abstract = "😀" * VIKINGDB_STRING_FIELD_BYTE_LIMIT++ normalized = adapter._normalize_record_for_write({"abstract": abstract})++ assert normalized["abstract"] == abstract++def test_volcengine_adapter_update_data_returns_batch_primary_keys():adapter = VolcengineCollectionAdapter(ak="test-ak",