Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(storage): enforce remote VikingDB field limits (#5101)

ByteDanceLiuYang25a17fa228f7
diff --git a/openviking/storage/vectordb_adapters/base.py b/openviking/storage/vectordb_adapters/base.py
index bfb79e2b0..b0a42dd56 100644
--- a/openviking/storage/vectordb_adapters/base.py
+++ b/openviking/storage/vectordb_adapters/base.py
@@ -35,11 +35,12 @@ from openviking_cli.utils.config.vectordb_config import DEFAULT_INDEX_NAME
logger = get_logger(__name__)
# ---------------------------------------------------------------------------
-# VikingDB text field byte limit
+# VikingDB field byte limits
# ---------------------------------------------------------------------------
-# VikingDB rejects upsert when any text field exceeds this byte length.
+# VikingDB string fields use a uint16 byte length, while text fields allow 1 MiB.
# Truncation is applied at a valid UTF-8 character boundary so that
# multi-byte sequences are never split in the middle.
+VIKINGDB_STRING_FIELD_BYTE_LIMIT: int = 64 * 1024
VIKINGDB_TEXT_FIELD_BYTE_LIMIT: int = 1024 * 1024
@@ -102,11 +103,15 @@ class CollectionAdapter(ABC):
mode: str
_URI_FIELD_NAMES = {"uri", "parent_uri"}
- # Text fields subject to byte-limit truncation before upsert.
- _TRUNCATABLE_TEXT_FIELDS: tuple[str, ...] = ("content", "abstract")
+ # Only derived fields may be shortened silently. An oversized abstract is
+ # stored as a prefix, so an exact-match filter using the original full
+ # abstract will not match the stored value.
+ _TRUNCATABLE_STRING_FIELDS: tuple[str, ...] = ("abstract",)
+ _TRUNCATABLE_TEXT_FIELDS: tuple[str, ...] = ("content",)
- # Per-backend byte limit for text fields. ``None`` means no truncation.
- # Subclasses backed by VikingDB should set this to ``VIKINGDB_TEXT_FIELD_BYTE_LIMIT``.
+ # Per-backend byte limits. ``None`` means no truncation. VikingDB-backed
+ # adapters set both limits; local adapters keep the complete values.
+ _STRING_FIELD_BYTE_LIMIT: int | None = None
_TEXT_FIELD_BYTE_LIMIT: int | None = None
# Whether this backend actually stores the ``content`` (full text) field.
@@ -281,6 +286,11 @@ class CollectionAdapter(ABC):
value = normalized.get(field)
if isinstance(value, str):
normalized[field] = _truncate_text_field(value, self._TEXT_FIELD_BYTE_LIMIT)
+ if self._STRING_FIELD_BYTE_LIMIT is not None:
+ for field in self._TRUNCATABLE_STRING_FIELDS:
+ value = normalized.get(field)
+ if isinstance(value, str):
+ normalized[field] = _truncate_text_field(value, self._STRING_FIELD_BYTE_LIMIT)
return normalized
@staticmethod
diff --git a/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py b/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py
index cc3cfc703..60d3a54da 100644
--- a/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py
+++ b/openviking/storage/vectordb_adapters/vikingdb_private_adapter.py
@@ -4,19 +4,24 @@
from __future__ import annotations
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
from openviking.storage.vectordb.collection.collection import Collection
from openviking.storage.vectordb.collection.vikingdb_clients import VIKINGDB_APIS, VikingDBClient
from openviking.storage.vectordb.collection.vikingdb_collection import VikingDBCollection
-from .base import VIKINGDB_TEXT_FIELD_BYTE_LIMIT, CollectionAdapter
+from .base import (
+ VIKINGDB_STRING_FIELD_BYTE_LIMIT,
+ VIKINGDB_TEXT_FIELD_BYTE_LIMIT,
+ CollectionAdapter,
+)
class VikingDBPrivateCollectionAdapter(CollectionAdapter):
"""Adapter for private VikingDB deployment."""
_DATA_BATCH_SIZE = 100
+ _STRING_FIELD_BYTE_LIMIT = VIKINGDB_STRING_FIELD_BYTE_LIMIT
_TEXT_FIELD_BYTE_LIMIT = VIKINGDB_TEXT_FIELD_BYTE_LIMIT
USE_CONTENT_FIELD = True
@@ -119,3 +124,17 @@ class VikingDBPrivateCollectionAdapter(CollectionAdapter):
def _normalize_record_for_read(self, record: Dict[str, Any]) -> Dict[str, Any]:
return super()._normalize_record_for_read(record)
+
+ def update_data(self, data_list: List[Dict[str, Any]]):
+ data_list = [self._normalize_record_for_write(item) for item in data_list]
+ result = self.get_collection().update_data(data_list)
+ if isinstance(result, dict):
+ for key in ("primary_keys", "ids"):
+ values = result.get(key)
+ if isinstance(values, list):
+ return [str(item) for item in values if item is not None]
+ if result.get("updated") == 0:
+ return []
+ if isinstance(result, list):
+ return [str(item) for item in result if item is not None]
+ return [str(item["id"]) for item in data_list if item.get("id") is not None]
diff --git a/openviking/storage/vectordb_adapters/volcengine_adapter.py b/openviking/storage/vectordb_adapters/volcengine_adapter.py
index 56dfed1c0..9c93a4b4a 100644
--- a/openviking/storage/vectordb_adapters/volcengine_adapter.py
+++ b/openviking/storage/vectordb_adapters/volcengine_adapter.py
@@ -15,13 +15,18 @@ from openviking.storage.vectordb.collection.volcengine_collection import (
get_or_create_volcengine_collection,
)
-from .base import VIKINGDB_TEXT_FIELD_BYTE_LIMIT, CollectionAdapter
+from .base import (
+ VIKINGDB_STRING_FIELD_BYTE_LIMIT,
+ VIKINGDB_TEXT_FIELD_BYTE_LIMIT,
+ CollectionAdapter,
+)
class VolcengineCollectionAdapter(CollectionAdapter):
"""Adapter for Volcengine-hosted VikingDB."""
_DATA_BATCH_SIZE = 100
+ _STRING_FIELD_BYTE_LIMIT = VIKINGDB_STRING_FIELD_BYTE_LIMIT
_TEXT_FIELD_BYTE_LIMIT = VIKINGDB_TEXT_FIELD_BYTE_LIMIT
USE_CONTENT_FIELD = True
@@ -181,6 +186,7 @@ class VolcengineCollectionAdapter(CollectionAdapter):
return super()._normalize_record_for_read(record)
def update_data(self, data_list: List[Dict[str, Any]]):
+ data_list = [self._normalize_record_for_write(item) for item in data_list]
collection = self.get_collection()
result = collection.update_data(data_list)
if isinstance(result, dict):
diff --git a/tests/storage/test_volcengine_clients.py b/tests/storage/test_volcengine_clients.py
index 1b33dd0b6..d715784f1 100644
--- a/tests/storage/test_volcengine_clients.py
+++ b/tests/storage/test_volcengine_clients.py
@@ -1,3 +1,4 @@
+import pytest
import requests
from volcengine.base.Request import Request
@@ -7,6 +8,11 @@ from openviking.storage.vectordb.collection.volcengine_clients import (
ClientForDataApiWithApiKey,
)
from openviking.storage.vectordb.collection.volcengine_collection import VolcengineCollection
+from openviking.storage.vectordb_adapters.base import VIKINGDB_STRING_FIELD_BYTE_LIMIT
+from openviking.storage.vectordb_adapters.local_adapter import LocalCollectionAdapter
+from openviking.storage.vectordb_adapters.vikingdb_private_adapter import (
+ VikingDBPrivateCollectionAdapter,
+)
from openviking.storage.vectordb_adapters.volcengine_adapter import VolcengineCollectionAdapter
from openviking_cli.utils.config.vectordb_config import (
VectorDBBackendConfig,
@@ -660,6 +666,84 @@ def test_volcengine_adapter_update_data_returns_ids():
assert result == ["doc-1"]
[email protected]("operation", ["upsert", "update_data"])
+def test_volcengine_adapter_truncates_abstract_at_remote_string_limit(operation):
+ adapter = VolcengineCollectionAdapter(
+ ak="test-ak",
+ sk="test-sk",
+ region="cn-beijing",
+ session_token=None,
+ api_key=None,
+ host=None,
+ project_name="default",
+ collection_name="context",
+ index_name="default",
+ )
+ captured = {}
+
+ class _Collection:
+ def upsert_data(self, data_list, ttl=0):
+ captured["data"] = data_list
+ return None
+
+ def update_data(self, data_list):
+ captured["data"] = data_list
+ return {"updated": 1, "primary_keys": ["doc-1"]}
+
+ adapter._collection = _Collection()
+ abstract = "a" * (VIKINGDB_STRING_FIELD_BYTE_LIMIT - 3) + "你好"
+
+ getattr(adapter, operation)([{"id": "doc-1", "abstract": abstract}])
+
+ stored = captured["data"][0]["abstract"]
+ assert len(stored.encode("utf-8")) <= VIKINGDB_STRING_FIELD_BYTE_LIMIT
+ assert abstract.startswith(stored)
+ assert stored.endswith("你")
+
+
+def test_private_vikingdb_update_normalizes_remote_fields():
+ adapter = VikingDBPrivateCollectionAdapter(
+ host="unused.invalid",
+ headers=None,
+ project_name="default",
+ collection_name="context",
+ index_name="default",
+ )
+ captured = {}
+
+ class _Collection:
+ def update_data(self, data_list):
+ captured["data"] = data_list
+ return {"updated": 1, "primary_keys": ["doc-1"]}
+
+ adapter._collection = _Collection()
+ result = adapter.update_data(
+ [
+ {
+ "id": "doc-1",
+ "uri": "viking://resources/sample",
+ "abstract": "😀" * VIKINGDB_STRING_FIELD_BYTE_LIMIT,
+ }
+ ]
+ )
+
+ stored = captured["data"][0]
+ assert stored["uri"] == "/resources/sample"
+ assert len(stored["abstract"].encode("utf-8")) <= VIKINGDB_STRING_FIELD_BYTE_LIMIT
+ assert result == ["doc-1"]
+
+
+def test_local_adapter_does_not_apply_remote_abstract_limit():
+ adapter = LocalCollectionAdapter(
+ collection_name="context", project_path="", index_name="default"
+ )
+ abstract = "😀" * VIKINGDB_STRING_FIELD_BYTE_LIMIT
+
+ normalized = adapter._normalize_record_for_write({"abstract": abstract})
+
+ assert normalized["abstract"] == abstract
+
+
def test_volcengine_adapter_update_data_returns_batch_primary_keys():
adapter = VolcengineCollectionAdapter(
ak="test-ak",