Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
import json
import random
import string
import unittest
from dataclasses import dataclass, field
from typing import List
from openviking.storage.vectordb import engine
from openviking.storage.vectordb.store.bytes_row import (
FieldType,
_PyBytesRow,
_PyFieldType,
_PySchema,
)
from openviking.storage.vectordb.store.serializable import serializable
# Define a complex data structure for testing consistency
@serializable
@dataclass
class ComplexData:
label: int = 0
vector: List[float] = field(default_factory=list)
sparse_raw_terms: List[str] = field(default_factory=list)
sparse_values: List[float] = field(default_factory=list)
fields: str = ""
expire_ns_ts: int = 0
is_deleted: bool = False
class TestBytesRow(unittest.TestCase):
def test_basic_serialization(self):
@serializable
@dataclass
class BasicData:
id: int = field(default=0, metadata={"field_type": FieldType.int64})
score: float = 0.0
active: bool = False
name: str = ""
data = BasicData(id=1234567890, score=0.95, active=True, name="viking_db")
# Serialize
serialized = data.serialize()
self.assertIsInstance(serialized, bytes)
# Deserialize whole row
deserialized = BasicData.from_bytes(serialized)
self.assertEqual(deserialized.id, 1234567890)
self.assertAlmostEqual(deserialized.score, 0.95, places=5)
self.assertEqual(deserialized.active, True)
self.assertEqual(deserialized.name, "viking_db")
# Deserialize single field
val_id = BasicData.bytes_row.deserialize_field(serialized, "id")
self.assertEqual(val_id, 1234567890)
val_name = BasicData.bytes_row.deserialize_field(serialized, "name")
self.assertEqual(val_name, "viking_db")
def test_list_types(self):
@serializable
@dataclass
class ListData:
tags: List[str] = field(default_factory=list)
embedding: List[float] = field(default_factory=list)
counts: List[int] = field(default_factory=list)
data = ListData(
tags=["AI", "Vector", "Search"], embedding=[0.1, 0.2, 0.3, 0.4], counts=[1, 10, 100]
)
serialized = data.serialize()
deserialized = ListData.from_bytes(serialized)
self.assertEqual(deserialized.tags, ["AI", "Vector", "Search"])
self.assertEqual(len(deserialized.embedding), 4)
for i, v in enumerate([0.1, 0.2, 0.3, 0.4]):
self.assertAlmostEqual(deserialized.embedding[i], v, places=5)
self.assertEqual(deserialized.counts, [1, 10, 100])
def test_default_values(self):
@serializable
@dataclass
class DefaultData:
id: int = field(default=999, metadata={"field_type": FieldType.int64})
desc: str = "default"
# Empty data, should use defaults
data = DefaultData()
serialized = data.serialize()
deserialized = DefaultData.from_bytes(serialized)
self.assertEqual(deserialized.id, 999)
self.assertEqual(deserialized.desc, "default")
def test_unicode_strings(self):
@serializable
@dataclass
class UnicodeData:
text: str = ""
text = "你好,世界!🌍"
data = UnicodeData(text=text)
serialized = data.serialize()
val = UnicodeData.bytes_row.deserialize_field(serialized, "text")
self.assertEqual(val, text)
def test_oversized_string_serialization_raises(self):
@serializable
@dataclass
class LargeStringData:
text: str = ""
text = "x" * 65536
with self.assertRaisesRegex(
(RuntimeError, ValueError), "text.*exceeds 65535 bytes"
):
LargeStringData(text=text).serialize()
py_schema = _PySchema([{"name": "text", "data_type": _PyFieldType.string, "id": 0}])
py_row = _PyBytesRow(py_schema)
with self.assertRaisesRegex(ValueError, "text.*exceeds 65535 bytes"):
py_row.serialize({"text": text})
def test_oversized_list_string_element_serialization_raises(self):
# A list<string> element uses the same UINT16 length prefix as a scalar
# string, so it must reject >65535-byte elements with the same clean,
# field-attributed ValueError — not a raw struct.error from deep inside
# struct.pack_into.
py_schema = _PySchema(
[{"name": "tags", "data_type": _PyFieldType.list_string, "id": 0}]
)
py_row = _PyBytesRow(py_schema)
with self.assertRaisesRegex(ValueError, "tags.*exceeds 65535 bytes"):
py_row.serialize({"tags": ["x" * 65536, "ok"]})
# A list of in-bounds strings (including multibyte) still round-trips.
blob = py_row.serialize({"tags": ["hello", "世界"]})
self.assertEqual(py_row.deserialize_field(blob, "tags"), ["hello", "世界"])
def test_binary_data(self):
@serializable
@dataclass
class BinaryData:
raw: bytes = b""
blob = b"\x00\x01\x02\xff\xfe"
data = BinaryData(raw=blob)
serialized = data.serialize()
val = BinaryData.bytes_row.deserialize_field(serialized, "raw")
self.assertEqual(val, blob)
def test_schema_id_validation(self):
with self.assertRaises(ValueError):
engine.Schema(
[
{"name": "id", "data_type": engine.FieldType.int64, "id": 0},
{"name": "name", "data_type": engine.FieldType.string, "id": 2},
]
)
with self.assertRaises(ValueError):
engine.Schema(
[
{"name": "id", "data_type": engine.FieldType.int64, "id": 0},
{"name": "dup", "data_type": engine.FieldType.string, "id": 0},
]
)
def test_missing_fields_use_defaults(self):
schema = engine.Schema(
[
{
"name": "id",
"data_type": engine.FieldType.int64,
"id": 0,
"default_value": 7,
},
{
"name": "name",
"data_type": engine.FieldType.string,
"id": 1,
"default_value": "fallback",
},
{
"name": "tags",
"data_type": engine.FieldType.list_string,
"id": 2,
"default_value": ["a", "b"],
},
{
"name": "score",
"data_type": engine.FieldType.float32,
"id": 3,
},
]
)
row = engine.BytesRow(schema)
serialized = row.serialize({"id": 5})
self.assertEqual(row.deserialize_field(serialized, "id"), 5)
self.assertEqual(row.deserialize_field(serialized, "name"), "fallback")
self.assertEqual(row.deserialize_field(serialized, "tags"), ["a", "b"])
self.assertAlmostEqual(row.deserialize_field(serialized, "score"), 0.0, places=5)
class TestBytesRowConsistency(unittest.TestCase):
def setUp(self):
# Create C++ Schema equivalent to ComplexData
# Note: IDs must match the order in ComplexData (0-indexed)
self.cpp_fields = [
{"name": "label", "data_type": engine.FieldType.int64, "id": 0},
{"name": "vector", "data_type": engine.FieldType.list_float32, "id": 1},
{"name": "sparse_raw_terms", "data_type": engine.FieldType.list_string, "id": 2},
{"name": "sparse_values", "data_type": engine.FieldType.list_float32, "id": 3},
{"name": "fields", "data_type": engine.FieldType.string, "id": 4},
{"name": "expire_ns_ts", "data_type": engine.FieldType.int64, "id": 5},
{"name": "is_deleted", "data_type": engine.FieldType.boolean, "id": 6},
]
self.cpp_schema = engine.Schema(self.cpp_fields)
self.cpp_row = engine.BytesRow(self.cpp_schema)
# Create Python Schema equivalent to ComplexData
self.py_fields = [
{"name": "label", "data_type": _PyFieldType.int64, "id": 0},
{"name": "vector", "data_type": _PyFieldType.list_float32, "id": 1},
{"name": "sparse_raw_terms", "data_type": _PyFieldType.list_string, "id": 2},
{"name": "sparse_values", "data_type": _PyFieldType.list_float32, "id": 3},
{"name": "fields", "data_type": _PyFieldType.string, "id": 4},
{"name": "expire_ns_ts", "data_type": _PyFieldType.int64, "id": 5},
{"name": "is_deleted", "data_type": _PyFieldType.boolean, "id": 6},
]
self.py_schema = _PySchema(self.py_fields)
self.py_row = _PyBytesRow(self.py_schema)
def generate_random_data(self):
dim = 128
sparse_dim = 10
return {
"label": random.randint(0, 1000000),
"vector": [random.random() for _ in range(dim)],
"sparse_raw_terms": [
"".join(random.choices(string.ascii_letters, k=5)) for _ in range(sparse_dim)
],
"sparse_values": [random.random() for _ in range(sparse_dim)],
"fields": json.dumps(
{"key": "value", "data": "".join(random.choices(string.ascii_letters, k=50))}
),
"expire_ns_ts": 1234567890,
"is_deleted": random.choice([True, False]),
}
def test_py_write_cpp_read(self):
"""Test Python serialization -> C++ deserialization"""
data_dict = self.generate_random_data()
# Python Serialize (using pure Python impl)
py_bytes = self.py_row.serialize(data_dict)
# C++ Deserialize (using ComplexData via serializable or direct engine usage)
# Here we use direct engine usage to be explicit
cpp_res = self.cpp_row.deserialize(py_bytes)
# Verify
self.assertEqual(cpp_res["label"], data_dict["label"])
self.assertEqual(len(cpp_res["vector"]), len(data_dict["vector"]))
for a, b in zip(cpp_res["vector"], data_dict["vector"]):
self.assertAlmostEqual(a, b, places=5)
self.assertEqual(cpp_res["sparse_raw_terms"], data_dict["sparse_raw_terms"])
for a, b in zip(cpp_res["sparse_values"], data_dict["sparse_values"]):
self.assertAlmostEqual(a, b, places=5)
self.assertEqual(cpp_res["fields"], data_dict["fields"])
self.assertEqual(cpp_res["expire_ns_ts"], data_dict["expire_ns_ts"])
self.assertEqual(cpp_res["is_deleted"], data_dict["is_deleted"])
def test_cpp_write_py_read(self):
"""Test C++ serialization -> Python deserialization"""
data_dict = self.generate_random_data()
# C++ Serialize
cpp_bytes = self.cpp_row.serialize(data_dict)
# Python Deserialize
py_res = self.py_row.deserialize(cpp_bytes)
# Verify
self.assertEqual(py_res["label"], data_dict["label"])
# Check vector with almost equal
for a, b in zip(py_res["vector"], data_dict["vector"]):
self.assertAlmostEqual(a, b, places=5)
self.assertEqual(py_res["sparse_raw_terms"], data_dict["sparse_raw_terms"])
for a, b in zip(py_res["sparse_values"], data_dict["sparse_values"]):
self.assertAlmostEqual(a, b, places=5)
self.assertEqual(py_res["fields"], data_dict["fields"])
self.assertEqual(py_res["expire_ns_ts"], data_dict["expire_ns_ts"])
self.assertEqual(py_res["is_deleted"], data_dict["is_deleted"])
def test_binary_consistency(self):
"""Test that C++ and Python produce identical binary output"""
data_dict = self.generate_random_data()
py_bytes = self.py_row.serialize(data_dict)
cpp_bytes = self.cpp_row.serialize(data_dict)
self.assertEqual(len(py_bytes), len(cpp_bytes), "Binary length mismatch")
self.assertEqual(py_bytes, cpp_bytes, "Binary content mismatch")
class TestTextFieldType(unittest.TestCase):
"""The `text` field type mirrors STRING but uses a uint32 length prefix,
so it supports values larger than the 65535-byte STRING limit."""
def setUp(self):
self.cpp_fields = [
{"name": "label", "data_type": engine.FieldType.int64, "id": 0},
{"name": "body", "data_type": engine.FieldType.text, "id": 1},
]
self.cpp_schema = engine.Schema(self.cpp_fields)
self.cpp_row = engine.BytesRow(self.cpp_schema)
self.py_fields = [
{"name": "label", "data_type": _PyFieldType.int64, "id": 0},
{"name": "body", "data_type": _PyFieldType.text, "id": 1},
]
self.py_schema = _PySchema(self.py_fields)
self.py_row = _PyBytesRow(self.py_schema)
def _make_data(self):
# Well beyond the 65535-byte STRING limit, plus multibyte content.
return {"label": 42, "body": ("你好x" * 40000)}
def test_large_text_roundtrip_cpp(self):
data = self._make_data()
serialized = self.cpp_row.serialize(data)
self.assertEqual(self.cpp_row.deserialize_field(serialized, "body"), data["body"])
self.assertEqual(self.cpp_row.deserialize_field(serialized, "label"), 42)
def test_large_text_roundtrip_py(self):
data = self._make_data()
serialized = self.py_row.serialize(data)
self.assertEqual(self.py_row.deserialize_field(serialized, "body"), data["body"])
self.assertEqual(self.py_row.deserialize_field(serialized, "label"), 42)
def test_py_write_cpp_read(self):
data = self._make_data()
py_bytes = self.py_row.serialize(data)
self.assertEqual(self.cpp_row.deserialize_field(py_bytes, "body"), data["body"])
def test_cpp_write_py_read(self):
data = self._make_data()
cpp_bytes = self.cpp_row.serialize(data)
self.assertEqual(self.py_row.deserialize_field(cpp_bytes, "body"), data["body"])
def test_binary_consistency(self):
legacy_data = {"label": 42, "body": "升级前的记录"}
readers = []
for fields, field_type, schema, row in (
(self.cpp_fields, engine.FieldType, engine.Schema, engine.BytesRow),
(self.py_fields, _PyFieldType, _PySchema, _PyBytesRow),
):
legacy_schema = schema([fields[0], {**fields[1], "data_type": field_type.string}])
legacy_bytes = row(legacy_schema).serialize(legacy_data)
reader = row(schema([fields[0], {**fields[1], "legacy_data_type": field_type.string}]))
self.assertEqual(reader.deserialize(legacy_bytes), legacy_data)
readers.append(reader)
data = self._make_data()
cpp_bytes, py_bytes = [reader.serialize(data) for reader in readers]
self.assertEqual(cpp_bytes, py_bytes, "Binary content mismatch")
for reader in readers:
self.assertEqual(reader.deserialize(cpp_bytes), data)
self.assertEqual(reader.deserialize_field(cpp_bytes, "label"), 42)
for invalid in (b"\x00\x01", b"\x00\x02\x02"):
with self.assertRaisesRegex((ValueError, RuntimeError), "record version"):
reader.deserialize(invalid)
def test_text_declared_via_metadata(self):
@serializable
@dataclass
class TextData:
body: str = field(default="", metadata={"field_type": FieldType.text})
text = "y" * 70000
data = TextData(body=text)
serialized = data.serialize()
self.assertEqual(TextData.from_bytes(serialized).body, text)
if __name__ == "__main__":
unittest.main()