Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
refactor: cpp bytes rows (#105)
kkkwjxc763ab125a4d
diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.pyindex 61eb178df..5455621c1 100644--- a/openviking/storage/vectordb/store/bytes_row.py+++ b/openviking/storage/vectordb/store/bytes_row.py@@ -19,13 +19,13 @@ class FieldMeta:"""Field metadata: records field encoding rules and position."""name: str- data_type: "FieldType"+ data_type: "_PyFieldType"offset: int # Start offset (calculated from the beginning of row data)id: intdefault_value: Any = None-class FieldType(Enum):+class _PyFieldType(Enum):int64 = 0uint64 = 1float32 = 2@@ -37,7 +37,7 @@ class FieldType(Enum):list_float32 = 8-class Schema:+class _PySchema:"""Row data schema: manages metadata for all fields and calculates offsets."""def __init__(self, fields):@@ -45,10 +45,10 @@ class Schema:Initialize schema.fields example:[- {"name": "id", "data_type": FieldType.int64, "id": 0},- {"name": "score", "data_type": FieldType.float32, "id": 1},- {"name": "name", "data_type": FieldType.string, "id": 2},- {"name": "is_pass", "data_type": FieldType.boolean, "id": 3}+ {"name": "id", "data_type": _PyFieldType.int64, "id": 0},+ {"name": "score", "data_type": _PyFieldType.float32, "id": 1},+ {"name": "name", "data_type": _PyFieldType.string, "id": 2},+ {"name": "is_pass", "data_type": _PyFieldType.boolean, "id": 3}]"""self.field_metas: Dict[str, FieldMeta] = {}@@ -57,15 +57,15 @@ class Schema:# Type to size and default value mappingTYPE_INFO = {- FieldType.int64: (INT64_SIZE, 0),- FieldType.uint64: (UINT64_SIZE, 0),- FieldType.float32: (FLOAT32_SIZE, 0.0),- FieldType.string: (UINT32_SIZE, "default"),- FieldType.binary: (UINT32_SIZE, b""),- FieldType.boolean: (BOOL_SIZE, False),- FieldType.list_int64: (UINT32_SIZE, [0]),- FieldType.list_string: (UINT32_SIZE, ["default"]),- FieldType.list_float32: (UINT32_SIZE, [0.0]),+ _PyFieldType.int64: (INT64_SIZE, 0),+ _PyFieldType.uint64: (UINT64_SIZE, 0),+ _PyFieldType.float32: (FLOAT32_SIZE, 0.0),+ _PyFieldType.string: (UINT32_SIZE, "default"),+ _PyFieldType.binary: (UINT32_SIZE, b""),+ _PyFieldType.boolean: (BOOL_SIZE, False),+ _PyFieldType.list_int64: (UINT32_SIZE, [0]),+ _PyFieldType.list_string: (UINT32_SIZE, ["default"]),+ _PyFieldType.list_float32: (UINT32_SIZE, [0.0]),}for field in fields:@@ -107,8 +107,8 @@ class Schema:return self.field_orders-class BytesRow:- def __init__(self, schema: Schema):+class _PyBytesRow:+ def __init__(self, schema: _PySchema):self.schema = schemaself.field_order = schema.get_field_order()@@ -124,23 +124,23 @@ class BytesRow:field_name = field_meta.namevalue = row_data[field_name] if field_name in row_data else field_meta.default_value- if field_meta.data_type == FieldType.int64:+ if field_meta.data_type == _PyFieldType.int64:fix_fmt_list.append("q")fix_val_list.append(value)fix_region_offset += INT64_SIZE- elif field_meta.data_type == FieldType.uint64:+ elif field_meta.data_type == _PyFieldType.uint64:fix_fmt_list.append("Q")fix_val_list.append(value)fix_region_offset += UINT64_SIZE- elif field_meta.data_type == FieldType.float32:+ elif field_meta.data_type == _PyFieldType.float32:fix_fmt_list.append("f")fix_val_list.append(value)fix_region_offset += FLOAT32_SIZE- elif field_meta.data_type == FieldType.boolean:+ elif field_meta.data_type == _PyFieldType.boolean:fix_fmt_list.append("B")fix_val_list.append(int(value))fix_region_offset += BOOL_SIZE- elif field_meta.data_type == FieldType.string:+ elif field_meta.data_type == _PyFieldType.string:fix_fmt_list.append("I")fix_val_list.append(variable_region_offset)fix_region_offset += UINT32_SIZE@@ -152,7 +152,7 @@ class BytesRow:var_fmt_list.append(f"{bytes_item_len}s")var_val_list.append(bytes_item)variable_region_offset += bytes_item_len- elif field_meta.data_type == FieldType.binary:+ elif field_meta.data_type == _PyFieldType.binary:fix_fmt_list.append("I")fix_val_list.append(variable_region_offset)fix_region_offset += UINT32_SIZE@@ -162,7 +162,7 @@ class BytesRow:var_fmt_list.append(f"{len(value)}s")var_val_list.append(value)variable_region_offset += len(value)- elif field_meta.data_type == FieldType.list_int64:+ elif field_meta.data_type == _PyFieldType.list_int64:fix_fmt_list.append("I")fix_val_list.append(variable_region_offset)fix_region_offset += UINT32_SIZE@@ -172,7 +172,7 @@ class BytesRow:var_fmt_list.append(f"{value_len}q")var_val_list.extend(value)variable_region_offset += UINT16_SIZE + len(value) * INT64_SIZE- elif field_meta.data_type == FieldType.list_float32:+ elif field_meta.data_type == _PyFieldType.list_float32:fix_fmt_list.append("I")fix_val_list.append(variable_region_offset)fix_region_offset += UINT32_SIZE@@ -183,7 +183,7 @@ class BytesRow:var_val_list.extend(value)variable_region_offset += UINT16_SIZE + len(value) * FLOAT32_SIZE- elif field_meta.data_type == FieldType.list_string:+ elif field_meta.data_type == _PyFieldType.list_string:fix_fmt_list.append("I")fix_val_list.append(variable_region_offset)fix_region_offset += UINT32_SIZE@@ -207,32 +207,35 @@ class BytesRow:struct.pack_into(fmt, buffer, 1, *(fix_val_list + var_val_list))return bytes(buffer)+ def serialize_batch(self, rows_data) -> List[bytes]:+ return [self.serialize(row_data) for row_data in rows_data]+def deserialize_field(self, serialized_data, field_name):field_meta = self.schema.get_field_meta(field_name)if field_meta.id >= serialized_data[0]:return field_meta.default_value# Use '<' for little-endian in all unpack operations- if field_meta.data_type == FieldType.int64:+ if field_meta.data_type == _PyFieldType.int64:return struct.unpack_from("<q", serialized_data, field_meta.offset)[0]- elif field_meta.data_type == FieldType.uint64:+ elif field_meta.data_type == _PyFieldType.uint64:return struct.unpack_from("<Q", serialized_data, field_meta.offset)[0]- elif field_meta.data_type == FieldType.float32:+ elif field_meta.data_type == _PyFieldType.float32:return struct.unpack_from("<f", serialized_data, field_meta.offset)[0]- elif field_meta.data_type == FieldType.boolean:+ elif field_meta.data_type == _PyFieldType.boolean:# B is 1 byte, endianness doesn't matter, but consistent stylereturn bool(serialized_data[field_meta.offset])- elif field_meta.data_type == FieldType.string:+ elif field_meta.data_type == _PyFieldType.string:str_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]str_len = struct.unpack_from("<H", serialized_data, str_offset)[0]str_offset += UINT16_SIZEreturn serialized_data[str_offset : str_offset + str_len].decode("utf-8")- elif field_meta.data_type == FieldType.binary:+ elif field_meta.data_type == _PyFieldType.binary:binary_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]binary_len = struct.unpack_from("<I", serialized_data, binary_offset)[0]binary_offset += UINT32_SIZEreturn serialized_data[binary_offset : binary_offset + binary_len]- elif field_meta.data_type == FieldType.list_string:+ elif field_meta.data_type == _PyFieldType.list_string:list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]list_offset += UINT16_SIZE@@ -243,13 +246,13 @@ class BytesRow:str_list[i] = serialized_data[list_offset : list_offset + str_len].decode("utf-8")list_offset += str_lenreturn str_list- elif field_meta.data_type == FieldType.list_int64:+ elif field_meta.data_type == _PyFieldType.list_int64:list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]list_offset += UINT16_SIZEreturn list(struct.unpack_from(f"<{list_len}q", serialized_data, list_offset))- elif field_meta.data_type == FieldType.list_float32:+ elif field_meta.data_type == _PyFieldType.list_float32:list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]list_offset += UINT16_SIZE@@ -266,3 +269,17 @@ class BytesRow:data_dict[field_name] = valuereturn data_dict+++try:+ import openviking.storage.vectordb.engine as engine++ # Use C++ implementation if available+ BytesRow = engine.BytesRow+ Schema = engine.Schema+ FieldType = engine.FieldType+except ImportError:+ # Fallback to Python implementation+ BytesRow = _PyBytesRow+ Schema = _PySchema+ FieldType = _PyFieldTypediff --git a/openviking/storage/vectordb/store/serializable.py b/openviking/storage/vectordb/store/serializable.pyindex 2e4b7376b..1a8a1ae08 100644--- a/openviking/storage/vectordb/store/serializable.py+++ b/openviking/storage/vectordb/store/serializable.py@@ -81,10 +81,12 @@ def serializable(cls):# Get custom type from metadata, otherwise auto-inferif f.metadata and "field_type" in f.metadata:field_type = f.metadata["field_type"]- if not isinstance(field_type, FieldType):- raise ValueError(- f"Field '{field_name}': field_type must be a FieldType, got {type(field_type).__name__}"- )+ # Check if it's the correct C++ enum type+ if hasattr(field_type, "value") and isinstance(field_type.value, int):+ # Try to find corresponding FieldType+ # Assuming enum names match+ if hasattr(FieldType, field_type.name):+ field_type = getattr(FieldType, field_type.name)else:field_type = _python_type_to_field_type(f.type, field_name)@@ -101,6 +103,7 @@ def serializable(cls):field_list.append(field_def)# Create schema and bytes_row+ # Pass field_list (list of dicts) to C++ Schema constructorcls.schema = Schema(field_list)cls.bytes_row = BytesRow(cls.schema)@@ -112,7 +115,9 @@ def serializable(cls):def deserialize(self, data: bytes):data_dict = self.__class__.bytes_row.deserialize(data)for key, value in data_dict.items():- setattr(self, key, value)+ # Handle potential None values if C++ returns std::monostate+ if value is not None:+ setattr(self, key, value)# Automatically generate from_bytes class method@classmethod@@ -123,9 +128,18 @@ def serializable(cls):inst.deserialize(data)return inst+ # Automatically generate serialize_list class method+ @classmethod+ def serialize_list(cls_method, objects: list) -> list[bytes]:+ """Batch serialization for a list of objects"""+ if not objects:+ return []+ return cls_method.bytes_row.serialize_batch(objects)+# Inject methods into classcls.serialize = serializecls.deserialize = deserializecls.from_bytes = from_bytes+ cls.serialize_list = serialize_listreturn clsdiff --git a/openviking/storage/vectordb/store/store_manager.py b/openviking/storage/vectordb/store/store_manager.pyindex 6fa5c0923..7a31b6ac8 100644--- a/openviking/storage/vectordb/store/store_manager.py+++ b/openviking/storage/vectordb/store/store_manager.py@@ -95,7 +95,7 @@ class StoreManager:StoreManager.DeltaTable,[OpType.PUT] * len(delta_list),[str(base_ts + i) for i in range(len(delta_list))],- [data.serialize() for data in delta_list],+ DeltaRecord.serialize_list(delta_list),))@@ -109,7 +109,7 @@ class StoreManager:StoreManager.CandsTable,[OpType.PUT] * len(cands_list),[str(data.label) for data in cands_list],- [data.serialize() for data in cands_list],+ CandidateData.serialize_list(cands_list),))@@ -164,7 +164,7 @@ class StoreManager:StoreManager.DeltaTable,[OpType.PUT] * len(delta_list),[str(base_ts + i) for i in range(len(delta_list))],- [data.serialize() for data in delta_list],+ DeltaRecord.serialize_list(delta_list),))@@ -307,7 +307,7 @@ class StoreManager:StoreManager.DeltaTable,[OpType.PUT] * len(delta_list),[str(base_ts + i) for i in range(len(delta_list))],- [data.serialize() for data in delta_list],+ DeltaRecord.serialize_list(delta_list),))diff --git a/src/py_accessors.h b/src/py_accessors.hnew file mode 100644index 000000000..7712263a9--- /dev/null+++ b/src/py_accessors.h@@ -0,0 +1,276 @@+// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+// SPDX-License-Identifier: Apache-2.0++#pragma once++#include <pybind11/pybind11.h>+#include <pybind11/stl.h>+#include <cstring>+#include "store/bytes_row.h"++namespace py = pybind11;+namespace vdb = vectordb;++// Accessor for Python Dict+class PyDictAccessor {+ public:+ PyDictAccessor(const vdb::Schema& schema)+ : field_order_(schema.get_field_order()) {+ }++ bool has_value(const py::dict& row, int field_idx) const {+ const auto& name = field_order_[field_idx].name;+ if (!row.contains(name.c_str()))+ return false;+ return !row[name.c_str()].is_none();+ }++ int64_t get_int64(const py::dict& row, int field_idx) const {+ return row[field_order_[field_idx].name.c_str()].cast<int64_t>();+ }++ uint64_t get_uint64(const py::dict& row, int field_idx) const {+ return row[field_order_[field_idx].name.c_str()].cast<uint64_t>();+ }++ float get_float(const py::dict& row, int field_idx) const {+ return row[field_order_[field_idx].name.c_str()].cast<float>();+ }++ bool get_bool(const py::dict& row, int field_idx) const {+ return row[field_order_[field_idx].name.c_str()].cast<bool>();+ }++ int get_string_len(const py::dict& row, int field_idx) const {+ py::object val = row[field_order_[field_idx].name.c_str()];+ // Assume it's string or bytes+ if (py::isinstance<py::bytes>(val)) {+ return PyBytes_Size(val.ptr());+ }+ return val.cast<std::string>().length();+ }++ int get_binary_len(const py::dict& row, int field_idx) const {+ // Same as string for length+ return get_string_len(row, field_idx);+ }++ int get_list_len(const py::dict& row, int field_idx) const {+ py::list l = row[field_order_[field_idx].name.c_str()].cast<py::list>();+ return static_cast<int>(l.size());+ }++ int get_list_string_content_len(const py::dict& row, int field_idx) const {+ py::list l = row[field_order_[field_idx].name.c_str()].cast<py::list>();+ int total = 0;+ for (auto item : l) {+ total += item.cast<std::string>().length();+ }+ return total;+ }++ // Writers+ void write_string(const py::dict& row, int field_idx, char* dest) const {+ std::string s =+ row[field_order_[field_idx].name.c_str()].cast<std::string>();+ uint16_t len = static_cast<uint16_t>(s.length());+ std::memcpy(dest, &len, 2);+ if (len > 0)+ std::memcpy(dest + 2, s.data(), len);+ }++ void write_binary(const py::dict& row, int field_idx, char* dest) const {+ std::string s =+ row[field_order_[field_idx].name.c_str()].cast<std::string>();+ uint32_t len = static_cast<uint32_t>(s.length());+ std::memcpy(dest, &len, 4);+ if (len > 0)+ std::memcpy(dest + 4, s.data(), len);+ }++ void write_list_int64(const py::dict& row, int field_idx, char* dest) const {+ py::list l = row[field_order_[field_idx].name.c_str()].cast<py::list>();+ uint16_t len = static_cast<uint16_t>(l.size());+ std::memcpy(dest, &len, 2);+ int64_t* data_ptr = reinterpret_cast<int64_t*>(dest + 2);+ for (size_t i = 0; i < len; ++i) {+ data_ptr[i] = l[i].cast<int64_t>();+ }+ }++ void write_list_float32(const py::dict& row, int field_idx,+ char* dest) const {+ py::list l = row[field_order_[field_idx].name.c_str()].cast<py::list>();+ uint16_t len = static_cast<uint16_t>(l.size());+ std::memcpy(dest, &len, 2);+ float* data_ptr = reinterpret_cast<float*>(dest + 2);+ for (size_t i = 0; i < len; ++i) {+ data_ptr[i] = l[i].cast<float>();+ }+ }++ void write_list_string(const py::dict& row, int field_idx, char* dest) const {+ py::list l = row[field_order_[field_idx].name.c_str()].cast<py::list>();+ uint16_t len = static_cast<uint16_t>(l.size());+ std::memcpy(dest, &len, 2);+ char* cur = dest + 2;+ for (size_t i = 0; i < len; ++i) {+ std::string s = l[i].cast<std::string>();+ uint16_t slen = static_cast<uint16_t>(s.length());+ std::memcpy(cur, &slen, 2);+ cur += 2;+ if (slen > 0)+ std::memcpy(cur, s.data(), slen);+ cur += slen;+ }+ }++ private:+ const std::vector<vdb::FieldMeta>& field_order_;+};++// Accessor for Python Object+class PyObjectAccessor {+ public:+ PyObjectAccessor(const vdb::Schema& schema)+ : field_order_(schema.get_field_order()) {+ }++ bool has_value(const py::handle& row, int field_idx) const {+ const char* name = field_order_[field_idx].name.c_str();+ if (!py::hasattr(row, name))+ return false;+ return !row.attr(name).is_none();+ }++ int64_t get_int64(const py::handle& row, int field_idx) const {+ return row.attr(field_order_[field_idx].name.c_str()).cast<int64_t>();+ }++ uint64_t get_uint64(const py::handle& row, int field_idx) const {+ return row.attr(field_order_[field_idx].name.c_str()).cast<uint64_t>();+ }++ float get_float(const py::handle& row, int field_idx) const {+ return row.attr(field_order_[field_idx].name.c_str()).cast<float>();+ }++ bool get_bool(const py::handle& row, int field_idx) const {+ return row.attr(field_order_[field_idx].name.c_str()).cast<bool>();+ }++ int get_string_len(const py::handle& row, int field_idx) const {+ // See comments in PyDictAccessor about encoding efficiency+ return row.attr(field_order_[field_idx].name.c_str())+ .cast<std::string>()+ .length();+ }++ int get_binary_len(const py::handle& row, int field_idx) const {+ return get_string_len(row, field_idx);+ }++ int get_list_len(const py::handle& row, int field_idx) const {+ py::list l =+ row.attr(field_order_[field_idx].name.c_str()).cast<py::list>();+ return static_cast<int>(l.size());+ }++ int get_list_string_content_len(const py::handle& row, int field_idx) const {+ py::list l =+ row.attr(field_order_[field_idx].name.c_str()).cast<py::list>();+ int total = 0;+ for (auto item : l) {+ total += item.cast<std::string>().length();+ }+ return total;+ }++ void write_string(const py::handle& row, int field_idx, char* dest) const {+ std::string s =+ row.attr(field_order_[field_idx].name.c_str()).cast<std::string>();+ uint16_t len = static_cast<uint16_t>(s.length());+ std::memcpy(dest, &len, 2);+ if (len > 0)+ std::memcpy(dest + 2, s.data(), len);+ }++ void write_binary(const py::handle& row, int field_idx, char* dest) const {+ std::string s =+ row.attr(field_order_[field_idx].name.c_str()).cast<std::string>();+ uint32_t len = static_cast<uint32_t>(s.length());+ std::memcpy(dest, &len, 4);+ if (len > 0)+ std::memcpy(dest + 4, s.data(), len);+ }++ void write_list_int64(const py::handle& row, int field_idx,+ char* dest) const {+ py::list l =+ row.attr(field_order_[field_idx].name.c_str()).cast<py::list>();+ uint16_t len = static_cast<uint16_t>(l.size());+ std::memcpy(dest, &len, 2);+ int64_t* data_ptr = reinterpret_cast<int64_t*>(dest + 2);+ for (size_t i = 0; i < len; ++i) {+ data_ptr[i] = l[i].cast<int64_t>();+ }+ }++ void write_list_float32(const py::handle& row, int field_idx,+ char* dest) const {+ py::list l =+ row.attr(field_order_[field_idx].name.c_str()).cast<py::list>();+ uint16_t len = static_cast<uint16_t>(l.size());+ std::memcpy(dest, &len, 2);+ float* data_ptr = reinterpret_cast<float*>(dest + 2);+ for (size_t i = 0; i < len; ++i) {+ data_ptr[i] = l[i].cast<float>();+ }+ }++ void write_list_string(const py::handle& row, int field_idx,+ char* dest) const {+ py::list l =+ row.attr(field_order_[field_idx].name.c_str()).cast<py::list>();+ uint16_t len = static_cast<uint16_t>(l.size());+ std::memcpy(dest, &len, 2);+ char* cur = dest + 2;+ for (size_t i = 0; i < len; ++i) {+ std::string s = l[i].cast<std::string>();+ uint16_t slen = static_cast<uint16_t>(s.length());+ std::memcpy(cur, &slen, 2);+ cur += 2;+ if (slen > 0)+ std::memcpy(cur, s.data(), slen);+ cur += slen;+ }+ }++ private:+ const std::vector<vdb::FieldMeta>& field_order_;+};++// Helper to convert C++ Value to Python object+inline py::object value_to_py(const vdb::Value& val) {+ if (std::holds_alternative<std::monostate>(val))+ return py::none();+ if (std::holds_alternative<int64_t>(val))+ return py::cast(std::get<int64_t>(val));+ if (std::holds_alternative<uint64_t>(val))+ return py::cast(std::get<uint64_t>(val));+ if (std::holds_alternative<float>(val))+ return py::cast(std::get<float>(val));+ if (std::holds_alternative<bool>(val))+ return py::cast(std::get<bool>(val));+ if (std::holds_alternative<std::string>(val)) {+ return py::cast(std::get<std::string>(val));+ }+ if (std::holds_alternative<std::vector<int64_t>>(val))+ return py::cast(std::get<std::vector<int64_t>>(val));+ if (std::holds_alternative<std::vector<float>>(val))+ return py::cast(std::get<std::vector<float>>(val));+ if (std::holds_alternative<std::vector<std::string>>(val))+ return py::cast(std::get<std::vector<std::string>>(val));++ return py::none();+}diff --git a/src/pybind11_interface.cpp b/src/pybind11_interface.cppindex b5ce96f39..55c55e2d5 100644--- a/src/pybind11_interface.cpp+++ b/src/pybind11_interface.cpp@@ -3,10 +3,14 @@#include <pybind11/pybind11.h>#include <pybind11/stl.h>#include <pybind11/complex.h>+#include <chrono>+#include <iostream>#include "index/index_engine.h"#include "store/persist_store.h"#include "store/volatile_store.h"#include "common/log_utils.h"+#include "store/bytes_row.h"+#include "py_accessors.h"namespace py = pybind11;namespace vdb = vectordb;@@ -14,6 +18,140 @@ namespace vdb = vectordb;PYBIND11_MODULE(engine, m) {m.def("init_logging", &vdb::init_logging, "Initialize logging");+ py::enum_<vdb::FieldType>(m, "FieldType")+ .value("int64", vdb::FieldType::INT64)+ .value("uint64", vdb::FieldType::UINT64)+ .value("float32", vdb::FieldType::FLOAT32)+ .value("string", vdb::FieldType::STRING)+ .value("binary", vdb::FieldType::BINARY)+ .value("boolean", vdb::FieldType::BOOLEAN)+ .value("list_int64", vdb::FieldType::LIST_INT64)+ .value("list_string", vdb::FieldType::LIST_STRING)+ .value("list_float32", vdb::FieldType::LIST_FLOAT32);++ py::class_<vdb::Schema, std::shared_ptr<vdb::Schema>>(m, "Schema")+ .def(py::init([](const py::list& fields_py) {+ std::vector<vdb::FieldDef> fields;+ for (const auto& item : fields_py) {+ py::dict d = item.cast<py::dict>();+ vdb::FieldDef fd;+ fd.name = d["name"].cast<std::string>();+ fd.data_type = d["data_type"].cast<vdb::FieldType>();+ fd.id = d["id"].cast<int>();+ if (d.contains("default_value")) {+ try {+ switch (fd.data_type) {+ case vdb::FieldType::INT64:+ fd.default_value = d["default_value"].cast<int64_t>();+ break;+ case vdb::FieldType::UINT64:+ fd.default_value = d["default_value"].cast<uint64_t>();+ break;+ case vdb::FieldType::FLOAT32:+ fd.default_value = d["default_value"].cast<float>();+ break;+ case vdb::FieldType::BOOLEAN:+ fd.default_value = d["default_value"].cast<bool>();+ break;+ case vdb::FieldType::STRING:+ fd.default_value = d["default_value"].cast<std::string>();+ break;+ case vdb::FieldType::BINARY:+ fd.default_value = d["default_value"].cast<std::string>();+ break;+ case vdb::FieldType::LIST_INT64:+ fd.default_value =+ d["default_value"].cast<std::vector<int64_t>>();+ break;+ case vdb::FieldType::LIST_FLOAT32:+ fd.default_value =+ d["default_value"].cast<std::vector<float>>();+ break;+ case vdb::FieldType::LIST_STRING:+ fd.default_value =+ d["default_value"].cast<std::vector<std::string>>();+ break;+ }+ } catch (...) {+ fd.default_value = std::monostate{};+ }+ } else {+ fd.default_value = std::monostate{};+ }+ fields.push_back(fd);+ }+ return std::make_shared<vdb::Schema>(fields);+ }))+ .def("get_total_byte_length", &vdb::Schema::get_total_byte_length);++ py::class_<vdb::BytesRow>(m, "BytesRow")+ .def(py::init<std::shared_ptr<vdb::Schema>>())+ .def("serialize",+ [](vdb::BytesRow& self, const py::dict& row_data) {+ PyDictAccessor accessor(self.get_schema());+ std::string serialized =+ self.serialize_template(row_data, accessor);+ return py::bytes(serialized);+ })+ .def("serialize_batch",+ [](vdb::BytesRow& self, const py::list& objects) {+ py::list results;+ const auto& schema = self.get_schema();++ PyDictAccessor dict_accessor(schema);+ PyObjectAccessor obj_accessor(schema);++ for (const auto& obj : objects) {+ std::string serialized;+ if (py::isinstance<py::dict>(obj)) {+ serialized = self.serialize_template(obj.cast<py::dict>(),+ dict_accessor);+ } else {+ serialized = self.serialize_template(obj, obj_accessor);+ }+ results.append(py::bytes(serialized));+ }+ return results;+ })+ .def("deserialize",+ [](vdb::BytesRow& self, const std::string& data) {+ py::dict res_dict;+ const auto& schema = self.get_schema();++ const auto& field_order = schema.get_field_order();+ for (const auto& meta : field_order) {+ vdb::Value val = self.deserialize_field(data, meta.name);++ if (std::holds_alternative<std::monostate>(val))+ continue;++ if (meta.data_type == vdb::FieldType::BINARY) {+ if (std::holds_alternative<std::string>(val)) {+ res_dict[meta.name.c_str()] =+ py::bytes(std::get<std::string>(val));+ continue;+ }+ }+ res_dict[meta.name.c_str()] = value_to_py(val);+ }+ return res_dict;+ })+ .def("deserialize_field",+ [](vdb::BytesRow& self, const std::string& data,+ const std::string& field_name) -> py::object {+ vdb::Value val = self.deserialize_field(data, field_name);+ const auto& schema = self.get_schema();+ const auto* meta = schema.get_field_meta(field_name);++ if (meta && meta->data_type == vdb::FieldType::BINARY) {+ if (std::holds_alternative<std::string>(val)) {+ const auto& s = std::get<std::string>(val);+ return py::bytes(s);+ }+ }+ return value_to_py(val);+ });+py::class_<vdb::AddDataRequest>(m, "AddDataRequest").def(py::init<>()).def_readwrite("label", &vdb::AddDataRequest::label)diff --git a/src/store/bytes_row.cpp b/src/store/bytes_row.cppnew file mode 100644index 000000000..a3c181eef--- /dev/null+++ b/src/store/bytes_row.cpp@@ -0,0 +1,501 @@+#include "bytes_row.h"+#include <cstring>+#include <stdexcept>+#include <algorithm>++namespace vectordb {++// Type size constants+constexpr int INT64_SIZE = 8;+constexpr int UINT64_SIZE = 8;+constexpr int FLOAT32_SIZE = 4;+constexpr int UINT32_SIZE = 4;+constexpr int UINT16_SIZE = 2;+constexpr int BOOL_SIZE = 1;++Schema::Schema(const std::vector<FieldDef>& fields) {+ int current_offset = 1; // Start after 1 byte header++ if (fields.empty()) {+ total_byte_length_ = current_offset;+ return;+ }++ int max_id = -1;+ for (const auto& field : fields) {+ if (field.id < 0) {+ throw std::invalid_argument("Field id must be non-negative");+ }+ if (field.id > max_id)+ max_id = field.id;+ }++ if (max_id != static_cast<int>(fields.size()) - 1) {+ throw std::invalid_argument(+ "Field ids must be contiguous from 0 to N-1");+ }++ std::vector<bool> seen(fields.size(), false);+ for (const auto& field : fields) {+ if (seen[field.id]) {+ throw std::invalid_argument("Duplicate field id found");+ }+ seen[field.id] = true;+ }++ field_orders_.resize(fields.size());++ for (const auto& field : fields) {+ int byte_len = 0;++ // Calculate basic type size+ switch (field.data_type) {+ case FieldType::INT64:+ byte_len = INT64_SIZE;+ break;+ case FieldType::UINT64:+ byte_len = UINT64_SIZE;+ break;+ case FieldType::FLOAT32:+ byte_len = FLOAT32_SIZE;+ break;+ case FieldType::BOOLEAN:+ byte_len = BOOL_SIZE;+ break;+ case FieldType::STRING:+ case FieldType::BINARY:+ case FieldType::LIST_INT64:+ case FieldType::LIST_STRING:+ case FieldType::LIST_FLOAT32:+ byte_len = UINT32_SIZE; // Offset+ break;+ }++ FieldMeta meta;+ meta.name = field.name;+ meta.data_type = field.data_type;+ meta.offset = current_offset;+ meta.id = field.id;+ meta.default_value = field.default_value;++ field_metas_[field.name] = meta;+ field_orders_[field.id] = meta;++ current_offset += byte_len;+ }++ total_byte_length_ = current_offset;+}++const FieldMeta* Schema::get_field_meta(const std::string& name) const {+ auto it = field_metas_.find(name);+ if (it == field_metas_.end()) {+ return nullptr;+ }+ return &it->second;+}++BytesRow::BytesRow(std::shared_ptr<Schema> schema) : schema_(schema) {+}++std::string BytesRow::serialize(const std::vector<Value>& row_data) const {+ const auto& field_order = schema_->get_field_order();++ // Pass 1: Calculate total size+ int total_size = schema_->get_total_byte_length();+ int variable_region_offset = total_size;++ // We'll store intermediate results to avoid recalculating+ struct VarFieldInfo {+ int offset; // Where the data starts in the buffer+ int length; // Length of the data+ };+ std::vector<VarFieldInfo> var_infos(field_order.size());++ for (size_t i = 0; i < field_order.size(); ++i) {+ const auto& meta = field_order[i];+ const Value& val = (i < row_data.size() &&+ !std::holds_alternative<std::monostate>(row_data[i]))+ ? row_data[i]+ : meta.default_value;++ // Skip fixed size fields calculation (already in total_byte_length_)+ // Only calculate variable length parts+ switch (meta.data_type) {+ case FieldType::STRING: {+ if (std::holds_alternative<std::string>(val)) {+ int len = std::get<std::string>(val).length();+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += UINT16_SIZE + len;+ } else {+ var_infos[i] = {variable_region_offset, 0};+ variable_region_offset += UINT16_SIZE;+ }+ break;+ }+ case FieldType::BINARY: {+ if (std::holds_alternative<std::string>(+ val)) { // Binary stored as string+ int len = std::get<std::string>(val).length();+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += UINT32_SIZE + len;+ } else {+ var_infos[i] = {variable_region_offset, 0};+ variable_region_offset += UINT32_SIZE;+ }+ break;+ }+ case FieldType::LIST_INT64: {+ if (std::holds_alternative<std::vector<int64_t>>(val)) {+ const auto& vec = std::get<std::vector<int64_t>>(val);+ int len = vec.size();+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += UINT16_SIZE + len * INT64_SIZE;+ } else {+ var_infos[i] = {variable_region_offset, 0};+ variable_region_offset += UINT16_SIZE;+ }+ break;+ }+ case FieldType::LIST_FLOAT32: {+ if (std::holds_alternative<std::vector<float>>(val)) {+ const auto& vec = std::get<std::vector<float>>(val);+ int len = vec.size();+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += UINT16_SIZE + len * FLOAT32_SIZE;+ } else {+ var_infos[i] = {variable_region_offset, 0};+ variable_region_offset += UINT16_SIZE;+ }+ break;+ }+ case FieldType::LIST_STRING: {+ if (std::holds_alternative<std::vector<std::string>>(val)) {+ const auto& vec = std::get<std::vector<std::string>>(val);+ int len = vec.size();+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += UINT16_SIZE; // List length+ for (const auto& s : vec) {+ variable_region_offset += UINT16_SIZE + s.length();+ }+ } else {+ var_infos[i] = {variable_region_offset, 0};+ variable_region_offset += UINT16_SIZE;+ }+ break;+ }+ default:+ break;+ }+ }++ // Allocate buffer+ std::string buffer;+ buffer.resize(variable_region_offset);+ char* ptr = &buffer[0];++ // Write header (field count)+ // Be careful with alignment if we were doing raw casting, but we use memcpy+ // so it's fine.++ uint8_t field_count = static_cast<uint8_t>(field_order.size());+ ptr[0] = field_count;++ // Pass 2: Write data+ for (size_t i = 0; i < field_order.size(); ++i) {+ const auto& meta = field_order[i];+ const Value& val = (i < row_data.size() &&+ !std::holds_alternative<std::monostate>(row_data[i]))+ ? row_data[i]+ : meta.default_value;++ char* field_ptr = ptr + meta.offset;++ switch (meta.data_type) {+ case FieldType::INT64: {+ int64_t v = 0;+ if (std::holds_alternative<int64_t>(val)) {+ v = std::get<int64_t>(val);+ } else if (std::holds_alternative<uint64_t>(val)) {+ // Implicit cast if needed+ v = static_cast<int64_t>(std::get<uint64_t>(val));+ }+ std::memcpy(field_ptr, &v, sizeof(v));+ break;+ }+ case FieldType::UINT64: {+ uint64_t v = 0;+ if (std::holds_alternative<uint64_t>(val)) {+ v = std::get<uint64_t>(val);+ } else if (std::holds_alternative<int64_t>(val)) {+ v = static_cast<uint64_t>(std::get<int64_t>(val));+ }+ std::memcpy(field_ptr, &v, sizeof(v));+ break;+ }+ case FieldType::FLOAT32: {+ float v =+ std::holds_alternative<float>(val) ? std::get<float>(val) : 0.0f;+ std::memcpy(field_ptr, &v, sizeof(v));+ break;+ }+ case FieldType::BOOLEAN: {+ bool v =+ std::holds_alternative<bool>(val) ? std::get<bool>(val) : false;+ uint8_t b = v ? 1 : 0;+ std::memcpy(field_ptr, &b, sizeof(b));+ break;+ }+ // Variable length fields: write offset to fixed region, then write data+ // to variable region+ case FieldType::STRING:+ case FieldType::BINARY:+ case FieldType::LIST_INT64:+ case FieldType::LIST_FLOAT32:+ case FieldType::LIST_STRING: {+ uint32_t offset = static_cast<uint32_t>(var_infos[i].offset);+ std::memcpy(field_ptr, &offset, sizeof(offset));++ char* var_ptr = ptr + offset;++ if (meta.data_type == FieldType::STRING) {+ const std::string& s = std::holds_alternative<std::string>(val)+ ? std::get<std::string>(val)+ : "";+ uint16_t len = static_cast<uint16_t>(s.length());+ std::memcpy(var_ptr, &len, sizeof(len));+ if (len > 0)+ std::memcpy(var_ptr + sizeof(len), s.data(), len);+ } else if (meta.data_type == FieldType::BINARY) {+ const std::string& s = std::holds_alternative<std::string>(val)+ ? std::get<std::string>(val)+ : "";+ uint32_t len = static_cast<uint32_t>(s.length());+ std::memcpy(var_ptr, &len, sizeof(len));+ if (len > 0)+ std::memcpy(var_ptr + sizeof(len), s.data(), len);+ } else if (meta.data_type == FieldType::LIST_INT64) {+ const auto& vec = std::holds_alternative<std::vector<int64_t>>(val)+ ? std::get<std::vector<int64_t>>(val)+ : std::vector<int64_t>{};+ uint16_t len = static_cast<uint16_t>(vec.size());+ std::memcpy(var_ptr, &len, sizeof(len));+ if (len > 0)+ std::memcpy(var_ptr + sizeof(len), vec.data(),+ len * sizeof(int64_t));+ } else if (meta.data_type == FieldType::LIST_FLOAT32) {+ const auto& vec = std::holds_alternative<std::vector<float>>(val)+ ? std::get<std::vector<float>>(val)+ : std::vector<float>{};+ uint16_t len = static_cast<uint16_t>(vec.size());+ std::memcpy(var_ptr, &len, sizeof(len));+ if (len > 0)+ std::memcpy(var_ptr + sizeof(len), vec.data(), len * sizeof(float));+ } else if (meta.data_type == FieldType::LIST_STRING) {+ const auto& vec =+ std::holds_alternative<std::vector<std::string>>(val)+ ? std::get<std::vector<std::string>>(val)+ : std::vector<std::string>{};+ uint16_t len = static_cast<uint16_t>(vec.size());+ std::memcpy(var_ptr, &len, sizeof(len));+ var_ptr += sizeof(len);+ for (const auto& s : vec) {+ uint16_t s_len = static_cast<uint16_t>(s.length());+ std::memcpy(var_ptr, &s_len, sizeof(s_len));+ var_ptr += sizeof(s_len);+ if (s_len > 0)+ std::memcpy(var_ptr, s.data(), s_len);+ var_ptr += s_len;+ }+ }+ break;+ }+ }+ }++ return buffer;+}++Value BytesRow::deserialize_field(const std::string& serialized_data,+ const std::string& field_name) const {+ const FieldMeta* meta_ptr = schema_->get_field_meta(field_name);+ if (!meta_ptr)+ return std::monostate{};++ const FieldMeta& meta = *meta_ptr;+ const char* ptr = serialized_data.data();++ // Check if data is large enough for this field's offset+ if (serialized_data.size() <= static_cast<size_t>(meta.offset)) {+ return meta.default_value;+ }++ uint8_t field_count = static_cast<uint8_t>(ptr[0]);+ if (meta.id >= field_count) {+ return meta.default_value;+ }++ const char* field_ptr = ptr + meta.offset;++ switch (meta.data_type) {+ case FieldType::INT64: {+ int64_t v;+ std::memcpy(&v, field_ptr, sizeof(v));+ return v;+ }+ case FieldType::UINT64: {+ uint64_t v;+ std::memcpy(&v, field_ptr, sizeof(v));+ return v;+ }+ case FieldType::FLOAT32: {+ float v;+ std::memcpy(&v, field_ptr, sizeof(v));+ return v;+ }+ case FieldType::BOOLEAN: {+ uint8_t b;+ std::memcpy(&b, field_ptr, sizeof(b));+ return (bool)b;+ }+ case FieldType::STRING: {+ uint32_t offset;+ if (sizeof(offset) >+ serialized_data.size() -+ static_cast<size_t>(field_ptr - serialized_data.data()))+ return std::string("");+ std::memcpy(&offset, field_ptr, sizeof(offset));+ if (offset >= serialized_data.size())+ return std::string("");++ uint16_t len;+ if (offset + sizeof(len) > serialized_data.size())+ return std::string("");+ std::memcpy(&len, ptr + offset, sizeof(len));++ if (static_cast<size_t>(offset) + sizeof(len) + len >+ serialized_data.size())+ return std::string("");+ return std::string(ptr + offset + sizeof(len), len);+ }+ case FieldType::BINARY: {+ uint32_t offset;+ if (sizeof(offset) >+ serialized_data.size() -+ static_cast<size_t>(field_ptr - serialized_data.data()))+ return std::string("");+ std::memcpy(&offset, field_ptr, sizeof(offset));+ if (offset >= serialized_data.size())+ return std::string("");++ uint32_t len;+ if (offset + sizeof(len) > serialized_data.size())+ return std::string("");+ std::memcpy(&len, ptr + offset, sizeof(len));++ if (static_cast<size_t>(offset) + sizeof(len) + len >+ serialized_data.size())+ return std::string("");+ return std::string(ptr + offset + sizeof(len), len);+ }+ case FieldType::LIST_INT64: {+ uint32_t offset;+ if (sizeof(offset) >+ serialized_data.size() -+ static_cast<size_t>(field_ptr - serialized_data.data()))+ return std::vector<int64_t>{};+ std::memcpy(&offset, field_ptr, sizeof(offset));+ if (offset >= serialized_data.size())+ return std::vector<int64_t>{};++ uint16_t len;+ if (offset + sizeof(len) > serialized_data.size())+ return std::vector<int64_t>{};+ std::memcpy(&len, ptr + offset, sizeof(len));++ std::vector<int64_t> vec(len);+ if (len > 0) {+ if (static_cast<size_t>(offset) + sizeof(len) + len * sizeof(int64_t) >+ serialized_data.size())+ return std::vector<int64_t>{};+ std::memcpy(vec.data(), ptr + offset + sizeof(len),+ len * sizeof(int64_t));+ }+ return vec;+ }+ case FieldType::LIST_FLOAT32: {+ uint32_t offset;+ if (sizeof(offset) >+ serialized_data.size() -+ static_cast<size_t>(field_ptr - serialized_data.data()))+ return std::vector<float>{};+ std::memcpy(&offset, field_ptr, sizeof(offset));+ if (offset >= serialized_data.size())+ return std::vector<float>{};++ uint16_t len;+ if (offset + sizeof(len) > serialized_data.size())+ return std::vector<float>{};+ std::memcpy(&len, ptr + offset, sizeof(len));++ std::vector<float> vec(len);+ if (len > 0) {+ if (static_cast<size_t>(offset) + sizeof(len) + len * sizeof(float) >+ serialized_data.size())+ return std::vector<float>{};+ std::memcpy(vec.data(), ptr + offset + sizeof(len),+ len * sizeof(float));+ }+ return vec;+ }+ case FieldType::LIST_STRING: {+ uint32_t offset;+ if (sizeof(offset) >+ serialized_data.size() -+ static_cast<size_t>(field_ptr - serialized_data.data()))+ return std::vector<std::string>{};+ std::memcpy(&offset, field_ptr, sizeof(offset));+ if (offset >= serialized_data.size())+ return std::vector<std::string>{};++ const char* var_ptr = ptr + offset;+ uint16_t list_len;++ if (static_cast<size_t>(offset) + sizeof(list_len) >+ serialized_data.size())+ return std::vector<std::string>{};+ std::memcpy(&list_len, var_ptr, sizeof(list_len));+ var_ptr += sizeof(list_len);++ std::vector<std::string> vec;+ vec.reserve(list_len);+ for (int i = 0; i < list_len; ++i) {+ uint16_t s_len;+ if (static_cast<size_t>(var_ptr - ptr) + sizeof(s_len) >+ serialized_data.size())+ break;+ std::memcpy(&s_len, var_ptr, sizeof(s_len));+ var_ptr += sizeof(s_len);++ if (static_cast<size_t>(var_ptr - ptr) + s_len > serialized_data.size())+ break;+ vec.emplace_back(var_ptr, s_len);+ var_ptr += s_len;+ }+ return vec;+ }+ }+ return std::monostate{};+}++std::map<std::string, Value> BytesRow::deserialize(+ const std::string& serialized_data) const {+ std::map<std::string, Value> result;+ const auto& order = schema_->get_field_order();+ for (const auto& meta : order) {+ result[meta.name] = deserialize_field(serialized_data, meta.name);+ }+ return result;+}++} // namespace vectordbdiff --git a/src/store/bytes_row.h b/src/store/bytes_row.hnew file mode 100644index 000000000..db1ab0e7b--- /dev/null+++ b/src/store/bytes_row.h@@ -0,0 +1,374 @@+#pragma once++#include <string>+#include <vector>+#include <map>+#include <variant>+#include <memory>+#include <cstdint>+#include <cstring>++namespace vectordb {++enum class FieldType {+ INT64 = 0,+ UINT64 = 1,+ FLOAT32 = 2,+ STRING = 3,+ BINARY = 4,+ BOOLEAN = 5,+ LIST_INT64 = 6,+ LIST_STRING = 7,+ LIST_FLOAT32 = 8+};++// Use std::monostate for None/Null+using Value = std::variant<std::monostate, int64_t, uint64_t, float, std::string, bool, std::vector<int64_t>, std::vector<std::string>, std::vector<float>>;++struct FieldMeta {+ std::string name;+ FieldType data_type;+ int offset;+ int id;+ Value default_value;+};++struct FieldDef {+ std::string name;+ FieldType data_type;+ int id;+ Value default_value;+};++class Schema {+public:+ Schema(const std::vector<FieldDef>& fields);++ const std::vector<FieldMeta>& get_field_order() const { return field_orders_; }+ int get_total_byte_length() const { return total_byte_length_; }+ const FieldMeta* get_field_meta(const std::string& name) const;++private:+ std::vector<FieldMeta> field_orders_;+ std::map<std::string, FieldMeta> field_metas_;+ int total_byte_length_;+};++class BytesRow {+public:+ explicit BytesRow(std::shared_ptr<Schema> schema);++ // Core serialization logic: takes values in order of schema fields+ std::string serialize(const std::vector<Value>& row_data) const;++ // Generic serialization template+ // Accessor must implement methods like:+ // int64_t get_int64(const RowT& row, int field_idx)+ // ... and so on for all types, plus is_null/has_value check+ template <typename RowT, typename AccessorT>+ std::string serialize_template(const RowT& row, const AccessorT& accessor) const {+ const auto& field_order = schema_->get_field_order();++ auto get_default_string = [](const Value& val) -> const std::string* {+ if (std::holds_alternative<std::string>(val)) {+ return &std::get<std::string>(val);+ }+ return nullptr;+ };++ auto get_default_list_int64 = [](const Value& val) -> const std::vector<int64_t>* {+ if (std::holds_alternative<std::vector<int64_t>>(val)) {+ return &std::get<std::vector<int64_t>>(val);+ }+ return nullptr;+ };++ auto get_default_list_float32 = [](const Value& val) -> const std::vector<float>* {+ if (std::holds_alternative<std::vector<float>>(val)) {+ return &std::get<std::vector<float>>(val);+ }+ return nullptr;+ };++ auto get_default_list_string = [](const Value& val) -> const std::vector<std::string>* {+ if (std::holds_alternative<std::vector<std::string>>(val)) {+ return &std::get<std::vector<std::string>>(val);+ }+ return nullptr;+ };++ auto get_list_string_content_len = [](const std::vector<std::string>& vec) -> int {+ int total = 0;+ for (const auto& s : vec) {+ total += static_cast<int>(s.length());+ }+ return total;+ };++ auto write_string_value = [](const std::string& s, char* dest) {+ uint16_t len = static_cast<uint16_t>(s.length());+ std::memcpy(dest, &len, 2);+ if (len > 0) {+ std::memcpy(dest + 2, s.data(), len);+ }+ };++ auto write_binary_value = [](const std::string& s, char* dest) {+ uint32_t len = static_cast<uint32_t>(s.length());+ std::memcpy(dest, &len, 4);+ if (len > 0) {+ std::memcpy(dest + 4, s.data(), len);+ }+ };++ auto write_list_int64_value = [](const std::vector<int64_t>& vec, char* dest) {+ uint16_t len = static_cast<uint16_t>(vec.size());+ std::memcpy(dest, &len, 2);+ if (len > 0) {+ std::memcpy(dest + 2, vec.data(), len * sizeof(int64_t));+ }+ };++ auto write_list_float32_value = [](const std::vector<float>& vec, char* dest) {+ uint16_t len = static_cast<uint16_t>(vec.size());+ std::memcpy(dest, &len, 2);+ if (len > 0) {+ std::memcpy(dest + 2, vec.data(), len * sizeof(float));+ }+ };++ auto write_list_string_value = [](const std::vector<std::string>& vec, char* dest) {+ uint16_t len = static_cast<uint16_t>(vec.size());+ std::memcpy(dest, &len, 2);+ char* cur = dest + 2;+ for (const auto& s : vec) {+ uint16_t slen = static_cast<uint16_t>(s.length());+ std::memcpy(cur, &slen, 2);+ cur += 2;+ if (slen > 0) {+ std::memcpy(cur, s.data(), slen);+ }+ cur += slen;+ }+ };++ // Pass 1: Calculate total size+ int total_size = schema_->get_total_byte_length();+ int variable_region_offset = total_size;++ struct VarFieldInfo {+ int offset;+ int length;+ };+ // Use a small buffer on stack if possible, or vector+ std::vector<VarFieldInfo> var_infos(field_order.size());++ for (size_t i = 0; i < field_order.size(); ++i) {+ const auto& meta = field_order[i];++ // For variable fields, we need to check length+ switch (meta.data_type) {+ case FieldType::STRING: {+ int len = 0;+ if (accessor.has_value(row, i)) {+ len = accessor.get_string_len(row, i);+ } else if (const auto* def = get_default_string(meta.default_value)) {+ len = static_cast<int>(def->length());+ }+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += 2 + len; // UINT16_SIZE+ break;+ }+ case FieldType::BINARY: {+ int len = 0;+ if (accessor.has_value(row, i)) {+ len = accessor.get_binary_len(row, i);+ } else if (const auto* def = get_default_string(meta.default_value)) {+ len = static_cast<int>(def->length());+ }+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += 4 + len; // UINT32_SIZE+ break;+ }+ case FieldType::LIST_INT64: {+ int len = 0;+ if (accessor.has_value(row, i)) {+ len = accessor.get_list_len(row, i);+ } else if (const auto* def = get_default_list_int64(meta.default_value)) {+ len = static_cast<int>(def->size());+ }+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += 2 + len * 8; // UINT16 + INT64_SIZE+ break;+ }+ case FieldType::LIST_FLOAT32: {+ int len = 0;+ if (accessor.has_value(row, i)) {+ len = accessor.get_list_len(row, i);+ } else if (const auto* def = get_default_list_float32(meta.default_value)) {+ len = static_cast<int>(def->size());+ }+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += 2 + len * 4; // UINT16 + FLOAT32_SIZE+ break;+ }+ case FieldType::LIST_STRING: {+ int list_len = 0;+ int content_len = 0;+ if (accessor.has_value(row, i)) {+ list_len = accessor.get_list_len(row, i);+ content_len = accessor.get_list_string_content_len(row, i);+ } else if (const auto* def = get_default_list_string(meta.default_value)) {+ list_len = static_cast<int>(def->size());+ content_len = get_list_string_content_len(*def);+ }+ var_infos[i] = {variable_region_offset, list_len};+ // list_len(2) + (elem_len(2) + content) * N+ // Actually content_len should include the 2 bytes for each string length if we compute it that way+ // Or we compute it here: 2 + (2 * list_len) + total_string_bytes+ variable_region_offset += 2 + (2 * list_len) + content_len;+ break;+ }+ default: break;+ }+ }++ std::string buffer;+ buffer.resize(variable_region_offset);+ char* ptr = &buffer[0];++ // Header+ ptr[0] = static_cast<uint8_t>(field_order.size());++ // Pass 2: Write data+ for (size_t i = 0; i < field_order.size(); ++i) {+ const auto& meta = field_order[i];+ char* field_ptr = ptr + meta.offset;+ bool has_val = accessor.has_value(row, i);++ switch (meta.data_type) {+ case FieldType::INT64: {+ int64_t v = 0;+ if (has_val) {+ v = accessor.get_int64(row, i);+ } else if (std::holds_alternative<int64_t>(meta.default_value)) {+ v = std::get<int64_t>(meta.default_value);+ } else if (std::holds_alternative<uint64_t>(meta.default_value)) {+ v = static_cast<int64_t>(std::get<uint64_t>(meta.default_value));+ }+ std::memcpy(field_ptr, &v, sizeof(v));+ break;+ }+ case FieldType::UINT64: {+ uint64_t v = 0;+ if (has_val) {+ v = accessor.get_uint64(row, i);+ } else if (std::holds_alternative<uint64_t>(meta.default_value)) {+ v = std::get<uint64_t>(meta.default_value);+ } else if (std::holds_alternative<int64_t>(meta.default_value)) {+ v = static_cast<uint64_t>(std::get<int64_t>(meta.default_value));+ }+ std::memcpy(field_ptr, &v, sizeof(v));+ break;+ }+ case FieldType::FLOAT32: {+ float v = 0.0f;+ if (has_val) {+ v = accessor.get_float(row, i);+ } else if (std::holds_alternative<float>(meta.default_value)) {+ v = std::get<float>(meta.default_value);+ }+ std::memcpy(field_ptr, &v, sizeof(v));+ break;+ }+ case FieldType::BOOLEAN: {+ bool v = false;+ if (has_val) {+ v = accessor.get_bool(row, i);+ } else if (std::holds_alternative<bool>(meta.default_value)) {+ v = std::get<bool>(meta.default_value);+ }+ uint8_t b = v ? 1 : 0;+ std::memcpy(field_ptr, &b, sizeof(b));+ break;+ }+ case FieldType::STRING:+ case FieldType::BINARY:+ case FieldType::LIST_INT64:+ case FieldType::LIST_FLOAT32:+ case FieldType::LIST_STRING: {+ uint32_t offset = static_cast<uint32_t>(var_infos[i].offset);+ std::memcpy(field_ptr, &offset, sizeof(offset));++ char* var_ptr = ptr + offset;++ if (meta.data_type == FieldType::STRING) {+ // Logic handled by accessor to write directly? Or return string_view/string?+ // Accessor returning string creates copy.+ // Better: accessor.write_string(row, i, var_ptr)+ if (has_val) {+ accessor.write_string(row, i, var_ptr);+ } else if (const auto* def = get_default_string(meta.default_value)) {+ write_string_value(*def, var_ptr);+ } else {+ uint16_t len = 0;+ std::memcpy(var_ptr, &len, 2);+ }+ } else if (meta.data_type == FieldType::BINARY) {+ if (has_val) {+ accessor.write_binary(row, i, var_ptr);+ } else if (const auto* def = get_default_string(meta.default_value)) {+ write_binary_value(*def, var_ptr);+ } else {+ uint32_t len = 0;+ std::memcpy(var_ptr, &len, 4);+ }+ } else if (meta.data_type == FieldType::LIST_INT64) {+ if (has_val) {+ accessor.write_list_int64(row, i, var_ptr);+ } else if (const auto* def = get_default_list_int64(meta.default_value)) {+ write_list_int64_value(*def, var_ptr);+ } else {+ uint16_t len = 0;+ std::memcpy(var_ptr, &len, 2);+ }+ } else if (meta.data_type == FieldType::LIST_FLOAT32) {+ if (has_val) {+ accessor.write_list_float32(row, i, var_ptr);+ } else if (const auto* def = get_default_list_float32(meta.default_value)) {+ write_list_float32_value(*def, var_ptr);+ } else {+ uint16_t len = 0;+ std::memcpy(var_ptr, &len, 2);+ }+ } else if (meta.data_type == FieldType::LIST_STRING) {+ if (has_val) {+ accessor.write_list_string(row, i, var_ptr);+ } else if (const auto* def = get_default_list_string(meta.default_value)) {+ write_list_string_value(*def, var_ptr);+ } else {+ uint16_t len = 0;+ std::memcpy(var_ptr, &len, 2);+ }+ }+ break;+ }+ }+ }+ return buffer;+ }++ // Deserialize to a map+ std::map<std::string, Value> deserialize(const std::string& serialized_data) const;++ // Deserialize a single field+ Value deserialize_field(const std::string& serialized_data, const std::string& field_name) const;++ // Get schema+ const Schema& get_schema() const { return *schema_; }++private:+ std::shared_ptr<Schema> schema_;+};++} // namespace vectordbdiff --git a/src/store/persist_store.cpp b/src/store/persist_store.cppindex b0fd4c2c6..8d0b45d86 100644--- a/src/store/persist_store.cpp+++ b/src/store/persist_store.cpp@@ -55,7 +55,9 @@ int PersistStore::put_data(const std::vector<std::string>& keys,for (size_t i = 0; i < keys.size(); ++i) {batch.Put(keys[i], values[i]);}- auto status = db_->Write(leveldb::WriteOptions(), &batch);+ leveldb::WriteOptions write_options;+ write_options.sync = true;+ auto status = db_->Write(write_options, &batch);if (!status.ok()) {SPDLOG_WARN("Failed to put data db: {}", status.ToString());return -1;@@ -68,7 +70,9 @@ int PersistStore::delete_data(const std::vector<std::string>& keys) {for (const auto& key : keys) {batch.Delete(key);}- auto status = db_->Write(leveldb::WriteOptions(), &batch);+ leveldb::WriteOptions write_options;+ write_options.sync = true;+ auto status = db_->Write(write_options, &batch);if (!status.ok()) {SPDLOG_WARN("Failed to delete data db: {}", status.ToString());return -1;@@ -82,7 +86,9 @@ int PersistStore::clear_data() {for (it->SeekToFirst(); it->Valid(); it->Next()) {batch.Delete(it->key().ToString());}- auto status = db_->Write(leveldb::WriteOptions(), &batch);+ leveldb::WriteOptions write_options;+ write_options.sync = true;+ auto status = db_->Write(write_options, &batch);if (!status.ok()) {SPDLOG_WARN("Failed to clear data db: {}", status.ToString());return -1;@@ -118,7 +124,9 @@ int PersistStore::exec_op(const std::vector<StorageOp>& ops) {continue;}}- auto status = db_->Write(leveldb::WriteOptions(), &batch);+ leveldb::WriteOptions write_options;+ write_options.sync = true;+ auto status = db_->Write(write_options, &batch);if (!status.ok()) {SPDLOG_WARN("Failed to exec op data db: {}", status.ToString());return -1;diff --git a/tests/vectordb/test_bytes_row.py b/tests/vectordb/test_bytes_row.pyindex 8b1fdbea6..906aa0b45 100644--- a/tests/vectordb/test_bytes_row.py+++ b/tests/vectordb/test_bytes_row.py@@ -1,13 +1,35 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: Apache-2.0+import json+import random+import stringimport unittestfrom dataclasses import dataclass, fieldfrom typing import List-from openviking.storage.vectordb.store.bytes_row import FieldType+from openviking.storage.vectordb import engine+from openviking.storage.vectordb.store.bytes_row import (+ FieldType,+ _PyBytesRow,+ _PyFieldType,+ _PySchema,+)from openviking.storage.vectordb.store.serializable import serializable+# Define a complex data structure for testing consistency+@serializable+@dataclass+class ComplexData:+ label: int = 0+ vector: List[float] = field(default_factory=list)+ sparse_raw_terms: List[str] = field(default_factory=list)+ sparse_values: List[float] = field(default_factory=list)+ fields: str = ""+ expire_ns_ts: int = 0+ is_deleted: bool = False++class TestBytesRow(unittest.TestCase):def test_basic_serialization(self):@serializable@@ -98,6 +120,169 @@ class TestBytesRow(unittest.TestCase):val = BinaryData.bytes_row.deserialize_field(serialized, "raw")self.assertEqual(val, blob)+ def test_schema_id_validation(self):+ with self.assertRaises(ValueError):+ engine.Schema(+ [+ {"name": "id", "data_type": engine.FieldType.int64, "id": 0},+ {"name": "name", "data_type": engine.FieldType.string, "id": 2},+ ]+ )++ with self.assertRaises(ValueError):+ engine.Schema(+ [+ {"name": "id", "data_type": engine.FieldType.int64, "id": 0},+ {"name": "dup", "data_type": engine.FieldType.string, "id": 0},+ ]+ )++ def test_missing_fields_use_defaults(self):+ schema = engine.Schema(+ [+ {+ "name": "id",+ "data_type": engine.FieldType.int64,+ "id": 0,+ "default_value": 7,+ },+ {+ "name": "name",+ "data_type": engine.FieldType.string,+ "id": 1,+ "default_value": "fallback",+ },+ {+ "name": "tags",+ "data_type": engine.FieldType.list_string,+ "id": 2,+ "default_value": ["a", "b"],+ },+ {+ "name": "score",+ "data_type": engine.FieldType.float32,+ "id": 3,+ },+ ]+ )+ row = engine.BytesRow(schema)++ serialized = row.serialize({"id": 5})++ self.assertEqual(row.deserialize_field(serialized, "id"), 5)+ self.assertEqual(row.deserialize_field(serialized, "name"), "fallback")+ self.assertEqual(row.deserialize_field(serialized, "tags"), ["a", "b"])+ self.assertAlmostEqual(row.deserialize_field(serialized, "score"), 0.0, places=5)+++class TestBytesRowConsistency(unittest.TestCase):+ def setUp(self):+ # Create C++ Schema equivalent to ComplexData+ # Note: IDs must match the order in ComplexData (0-indexed)+ self.cpp_fields = [+ {"name": "label", "data_type": engine.FieldType.int64, "id": 0},+ {"name": "vector", "data_type": engine.FieldType.list_float32, "id": 1},+ {"name": "sparse_raw_terms", "data_type": engine.FieldType.list_string, "id": 2},+ {"name": "sparse_values", "data_type": engine.FieldType.list_float32, "id": 3},+ {"name": "fields", "data_type": engine.FieldType.string, "id": 4},+ {"name": "expire_ns_ts", "data_type": engine.FieldType.int64, "id": 5},+ {"name": "is_deleted", "data_type": engine.FieldType.boolean, "id": 6},+ ]+ self.cpp_schema = engine.Schema(self.cpp_fields)+ self.cpp_row = engine.BytesRow(self.cpp_schema)++ # Create Python Schema equivalent to ComplexData+ self.py_fields = [+ {"name": "label", "data_type": _PyFieldType.int64, "id": 0},+ {"name": "vector", "data_type": _PyFieldType.list_float32, "id": 1},+ {"name": "sparse_raw_terms", "data_type": _PyFieldType.list_string, "id": 2},+ {"name": "sparse_values", "data_type": _PyFieldType.list_float32, "id": 3},+ {"name": "fields", "data_type": _PyFieldType.string, "id": 4},+ {"name": "expire_ns_ts", "data_type": _PyFieldType.int64, "id": 5},+ {"name": "is_deleted", "data_type": _PyFieldType.boolean, "id": 6},+ ]+ self.py_schema = _PySchema(self.py_fields)+ self.py_row = _PyBytesRow(self.py_schema)++ def generate_random_data(self):+ dim = 128+ sparse_dim = 10++ return {+ "label": random.randint(0, 1000000),+ "vector": [random.random() for _ in range(dim)],+ "sparse_raw_terms": [+ "".join(random.choices(string.ascii_letters, k=5)) for _ in range(sparse_dim)+ ],+ "sparse_values": [random.random() for _ in range(sparse_dim)],+ "fields": json.dumps(+ {"key": "value", "data": "".join(random.choices(string.ascii_letters, k=50))}+ ),+ "expire_ns_ts": 1234567890,+ "is_deleted": random.choice([True, False]),+ }++ def test_py_write_cpp_read(self):+ """Test Python serialization -> C++ deserialization"""+ data_dict = self.generate_random_data()++ # Python Serialize (using pure Python impl)+ py_bytes = self.py_row.serialize(data_dict)++ # C++ Deserialize (using ComplexData via serializable or direct engine usage)+ # Here we use direct engine usage to be explicit+ cpp_res = self.cpp_row.deserialize(py_bytes)++ # Verify+ self.assertEqual(cpp_res["label"], data_dict["label"])+ self.assertEqual(len(cpp_res["vector"]), len(data_dict["vector"]))+ for a, b in zip(cpp_res["vector"], data_dict["vector"]):+ self.assertAlmostEqual(a, b, places=5)++ self.assertEqual(cpp_res["sparse_raw_terms"], data_dict["sparse_raw_terms"])++ for a, b in zip(cpp_res["sparse_values"], data_dict["sparse_values"]):+ self.assertAlmostEqual(a, b, places=5)++ self.assertEqual(cpp_res["fields"], data_dict["fields"])+ self.assertEqual(cpp_res["expire_ns_ts"], data_dict["expire_ns_ts"])+ self.assertEqual(cpp_res["is_deleted"], data_dict["is_deleted"])++ def test_cpp_write_py_read(self):+ """Test C++ serialization -> Python deserialization"""+ data_dict = self.generate_random_data()++ # C++ Serialize+ cpp_bytes = self.cpp_row.serialize(data_dict)++ # Python Deserialize+ py_res = self.py_row.deserialize(cpp_bytes)++ # Verify+ self.assertEqual(py_res["label"], data_dict["label"])+ # Check vector with almost equal+ for a, b in zip(py_res["vector"], data_dict["vector"]):+ self.assertAlmostEqual(a, b, places=5)++ self.assertEqual(py_res["sparse_raw_terms"], data_dict["sparse_raw_terms"])++ for a, b in zip(py_res["sparse_values"], data_dict["sparse_values"]):+ self.assertAlmostEqual(a, b, places=5)++ self.assertEqual(py_res["fields"], data_dict["fields"])+ self.assertEqual(py_res["expire_ns_ts"], data_dict["expire_ns_ts"])+ self.assertEqual(py_res["is_deleted"], data_dict["is_deleted"])++ def test_binary_consistency(self):+ """Test that C++ and Python produce identical binary output"""+ data_dict = self.generate_random_data()++ py_bytes = self.py_row.serialize(data_dict)+ cpp_bytes = self.cpp_row.serialize(data_dict)++ self.assertEqual(len(py_bytes), len(cpp_bytes), "Binary length mismatch")+ self.assertEqual(py_bytes, cpp_bytes, "Binary content mismatch")+if __name__ == "__main__":unittest.main()diff --git a/tests/vectordb/test_recall.py b/tests/vectordb/test_recall.pyindex b32300694..07fb01f29 100644--- a/tests/vectordb/test_recall.py+++ b/tests/vectordb/test_recall.py@@ -419,6 +419,182 @@ class TestRecall(unittest.TestCase):)print("✓ Hybrid dense+sparse mix verified")+ def test_complex_schema_missing_fields(self):+ """Test adding data with missing optional fields using complex schema"""+ print("\n=== Test: Complex Schema Missing Fields ===")+ dim = 1024+ name = "test_complex_missing_fields"+ meta_data = {+ "CollectionName": name,+ "Description": "Unified context collection",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {+ "FieldName": "uri",+ "FieldType": "string",+ }, # Changed path to string for simplicity as 'path' might not be standard FieldType+ {"FieldName": "type", "FieldType": "string"},+ {"FieldName": "context_type", "FieldType": "string"},+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},+ {"FieldName": "sparse_vector", "FieldType": "sparse_vector"},+ {+ "FieldName": "created_at",+ "FieldType": "string",+ }, # Simulating date_time as string+ {"FieldName": "updated_at", "FieldType": "string"},+ {"FieldName": "active_count", "FieldType": "int64"},+ {"FieldName": "parent_uri", "FieldType": "string"},+ {"FieldName": "is_leaf", "FieldType": "bool"},+ {"FieldName": "name", "FieldType": "string"},+ {"FieldName": "description", "FieldType": "string"},+ {"FieldName": "tags", "FieldType": "string"},+ {"FieldName": "abstract", "FieldType": "string"},+ ],+ }++ collection = self.register_collection(+ get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)+ )++ # 1. Full record+ full_record = {+ "id": "1",+ "uri": "/path/to/1",+ "type": "doc",+ "context_type": "text",+ "vector": [0.1] * dim,+ "sparse_vector": {"t1": 1.0},+ "created_at": "2023-01-01",+ "updated_at": "2023-01-02",+ "active_count": 10,+ "parent_uri": "/path/to/0",+ "is_leaf": True,+ "name": "Doc 1",+ "description": "A description",+ "tags": "tag1,tag2",+ "abstract": "An abstract",+ }++ # 2. Minimal record (Only ID and Vector are strictly required by engine for indexing usually, but let's see schema validation)+ # Assuming only PK and Vector are strictly mandatory for vector search index, others should be optional/default.+ minimal_record = {+ "id": "2",+ "vector": [0.2] * dim,+ }++ # 3. Partial record+ partial_record = {+ "id": "3",+ "vector": [0.3] * dim,+ "name": "Doc 3",+ "active_count": 5,+ }++ collection.upsert_data([full_record, minimal_record, partial_record])++ # Verify data via Fetch+ res_full = collection.fetch_data(["1"])+ self.assertEqual(len(res_full.items), 1)+ self.assertEqual(res_full.items[0].id, "1")+ # Check fields exist in extra_json or attributes depending on implementation+ # The result object structure depends on how LocalCollection returns data.+ # Typically it returns an object where fields are accessible or in 'fields' dict.+ # Let's assume standard behavior where defined fields are attributes or in a dictionary.+ # For LocalCollection, non-vector fields are often serialized into a 'fields' JSON string or accessible directly if mapped.+ # We need to check if the data came back.++ # NOTE: FetchDataResult structure: result_num, labels, scores, extra_json?+ # Actually fetch_data returns a list of results.++ print(f"Full Record Fetch: {res_full.items[0]}")++ res_min = collection.fetch_data(["2"])+ self.assertEqual(len(res_min.items), 1)+ self.assertEqual(res_min.items[0].id, "2")+ print(f"Minimal Record Fetch: {res_min.items[0]}")++ res_part = collection.fetch_data(["3"])+ self.assertEqual(len(res_part.items), 1)+ self.assertEqual(res_part.items[0].id, "3")+ print(f"Partial Record Fetch: {res_part.items[0]}")++ print("✓ Missing fields handled correctly")++ def test_persistence_crud(self):+ """Test CRUD operations persist after collection close and reopen"""+ print("\n=== Test: Persistence CRUD ===")+ dim = 1024+ name = "test_persistence"+ meta_data = {+ "CollectionName": name,+ "Description": "Persistence test",+ "Fields": [+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},+ {"FieldName": "name", "FieldType": "string"},+ ],+ }++ # 1. Open and Add Data+ collection = get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)+ self.register_collection(collection)++ data = [+ {"id": "1", "vector": [0.1] * dim, "name": "Item 1"},+ {"id": "2", "vector": [0.2] * dim, "name": "Item 2"},+ ]+ collection.upsert_data(data)++ # Verify Add+ res = collection.fetch_data(["1", "2"])+ self.assertEqual(len(res.items), 2)++ # 2. Close Collection (Simulate restart)+ # Note: LocalCollection might not have an explicit close() that unloads everything from memory+ # if it's purely object based, but we can delete the object and re-instantiate.+ # The important part is that data is on disk (RocksDB/LevelDB).+ collection.close()+ del collection++ # 3. Reopen+ collection_new = get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)+ self.register_collection(collection_new)++ # Verify Data Exists+ res_reopen = collection_new.fetch_data(["1", "2"])+ self.assertEqual(len(res_reopen.items), 2)+ # Order is not guaranteed, so check by ID or sort+ ids = sorted([item.id for item in res_reopen.items])+ self.assertEqual(ids, ["1", "2"])++ # 4. Update Data+ update_data = [{"id": "1", "vector": [0.9] * dim, "name": "Item 1 Updated"}]+ collection_new.upsert_data(update_data)++ res_update = collection_new.fetch_data(["1"])+ self.assertEqual(len(res_update.items), 1)+ self.assertEqual(res_update.items[0].fields["name"], "Item 1 Updated")++ # 5. Delete Data+ collection_new.delete_data(["2"])++ res_del = collection_new.fetch_data(["2"])+ self.assertEqual(len(res_del.items), 0, "Deleted item should not be found")+ self.assertEqual(len(res_del.ids_not_exist), 1)++ # 6. Search on persisted data+ collection_new.create_index(+ "idx_persist",+ {"IndexName": "idx_persist", "VectorIndex": {"IndexType": "flat", "Distance": "l2"}},+ )+ search_res = collection_new.search_by_vector(+ "idx_persist", dense_vector=[0.9] * dim, limit=1+ )+ self.assertEqual(len(search_res.data), 1)+ self.assertEqual(search_res.data[0].id, "1")++ print("✓ Persistence verified")+if __name__ == "__main__":unittest.main()