Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

refactor: cpp bytes rows (#105)

kkkwjxc763ab125a4d
diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.py
index 61eb178df..5455621c1 100644
--- a/openviking/storage/vectordb/store/bytes_row.py
+++ b/openviking/storage/vectordb/store/bytes_row.py
@@ -19,13 +19,13 @@ class FieldMeta:
"""Field metadata: records field encoding rules and position."""
name: str
- data_type: "FieldType"
+ data_type: "_PyFieldType"
offset: int # Start offset (calculated from the beginning of row data)
id: int
default_value: Any = None
-class FieldType(Enum):
+class _PyFieldType(Enum):
int64 = 0
uint64 = 1
float32 = 2
@@ -37,7 +37,7 @@ class FieldType(Enum):
list_float32 = 8
-class Schema:
+class _PySchema:
"""Row data schema: manages metadata for all fields and calculates offsets."""
def __init__(self, fields):
@@ -45,10 +45,10 @@ class Schema:
Initialize schema.
fields example:
[
- {"name": "id", "data_type": FieldType.int64, "id": 0},
- {"name": "score", "data_type": FieldType.float32, "id": 1},
- {"name": "name", "data_type": FieldType.string, "id": 2},
- {"name": "is_pass", "data_type": FieldType.boolean, "id": 3}
+ {"name": "id", "data_type": _PyFieldType.int64, "id": 0},
+ {"name": "score", "data_type": _PyFieldType.float32, "id": 1},
+ {"name": "name", "data_type": _PyFieldType.string, "id": 2},
+ {"name": "is_pass", "data_type": _PyFieldType.boolean, "id": 3}
]
"""
self.field_metas: Dict[str, FieldMeta] = {}
@@ -57,15 +57,15 @@ class Schema:
# Type to size and default value mapping
TYPE_INFO = {
- FieldType.int64: (INT64_SIZE, 0),
- FieldType.uint64: (UINT64_SIZE, 0),
- FieldType.float32: (FLOAT32_SIZE, 0.0),
- FieldType.string: (UINT32_SIZE, "default"),
- FieldType.binary: (UINT32_SIZE, b""),
- FieldType.boolean: (BOOL_SIZE, False),
- FieldType.list_int64: (UINT32_SIZE, [0]),
- FieldType.list_string: (UINT32_SIZE, ["default"]),
- FieldType.list_float32: (UINT32_SIZE, [0.0]),
+ _PyFieldType.int64: (INT64_SIZE, 0),
+ _PyFieldType.uint64: (UINT64_SIZE, 0),
+ _PyFieldType.float32: (FLOAT32_SIZE, 0.0),
+ _PyFieldType.string: (UINT32_SIZE, "default"),
+ _PyFieldType.binary: (UINT32_SIZE, b""),
+ _PyFieldType.boolean: (BOOL_SIZE, False),
+ _PyFieldType.list_int64: (UINT32_SIZE, [0]),
+ _PyFieldType.list_string: (UINT32_SIZE, ["default"]),
+ _PyFieldType.list_float32: (UINT32_SIZE, [0.0]),
}
for field in fields:
@@ -107,8 +107,8 @@ class Schema:
return self.field_orders
-class BytesRow:
- def __init__(self, schema: Schema):
+class _PyBytesRow:
+ def __init__(self, schema: _PySchema):
self.schema = schema
self.field_order = schema.get_field_order()
@@ -124,23 +124,23 @@ class BytesRow:
field_name = field_meta.name
value = row_data[field_name] if field_name in row_data else field_meta.default_value
- if field_meta.data_type == FieldType.int64:
+ if field_meta.data_type == _PyFieldType.int64:
fix_fmt_list.append("q")
fix_val_list.append(value)
fix_region_offset += INT64_SIZE
- elif field_meta.data_type == FieldType.uint64:
+ elif field_meta.data_type == _PyFieldType.uint64:
fix_fmt_list.append("Q")
fix_val_list.append(value)
fix_region_offset += UINT64_SIZE
- elif field_meta.data_type == FieldType.float32:
+ elif field_meta.data_type == _PyFieldType.float32:
fix_fmt_list.append("f")
fix_val_list.append(value)
fix_region_offset += FLOAT32_SIZE
- elif field_meta.data_type == FieldType.boolean:
+ elif field_meta.data_type == _PyFieldType.boolean:
fix_fmt_list.append("B")
fix_val_list.append(int(value))
fix_region_offset += BOOL_SIZE
- elif field_meta.data_type == FieldType.string:
+ elif field_meta.data_type == _PyFieldType.string:
fix_fmt_list.append("I")
fix_val_list.append(variable_region_offset)
fix_region_offset += UINT32_SIZE
@@ -152,7 +152,7 @@ class BytesRow:
var_fmt_list.append(f"{bytes_item_len}s")
var_val_list.append(bytes_item)
variable_region_offset += bytes_item_len
- elif field_meta.data_type == FieldType.binary:
+ elif field_meta.data_type == _PyFieldType.binary:
fix_fmt_list.append("I")
fix_val_list.append(variable_region_offset)
fix_region_offset += UINT32_SIZE
@@ -162,7 +162,7 @@ class BytesRow:
var_fmt_list.append(f"{len(value)}s")
var_val_list.append(value)
variable_region_offset += len(value)
- elif field_meta.data_type == FieldType.list_int64:
+ elif field_meta.data_type == _PyFieldType.list_int64:
fix_fmt_list.append("I")
fix_val_list.append(variable_region_offset)
fix_region_offset += UINT32_SIZE
@@ -172,7 +172,7 @@ class BytesRow:
var_fmt_list.append(f"{value_len}q")
var_val_list.extend(value)
variable_region_offset += UINT16_SIZE + len(value) * INT64_SIZE
- elif field_meta.data_type == FieldType.list_float32:
+ elif field_meta.data_type == _PyFieldType.list_float32:
fix_fmt_list.append("I")
fix_val_list.append(variable_region_offset)
fix_region_offset += UINT32_SIZE
@@ -183,7 +183,7 @@ class BytesRow:
var_val_list.extend(value)
variable_region_offset += UINT16_SIZE + len(value) * FLOAT32_SIZE
- elif field_meta.data_type == FieldType.list_string:
+ elif field_meta.data_type == _PyFieldType.list_string:
fix_fmt_list.append("I")
fix_val_list.append(variable_region_offset)
fix_region_offset += UINT32_SIZE
@@ -207,32 +207,35 @@ class BytesRow:
struct.pack_into(fmt, buffer, 1, *(fix_val_list + var_val_list))
return bytes(buffer)
+ def serialize_batch(self, rows_data) -> List[bytes]:
+ return [self.serialize(row_data) for row_data in rows_data]
+
def deserialize_field(self, serialized_data, field_name):
field_meta = self.schema.get_field_meta(field_name)
if field_meta.id >= serialized_data[0]:
return field_meta.default_value
# Use '<' for little-endian in all unpack operations
- if field_meta.data_type == FieldType.int64:
+ if field_meta.data_type == _PyFieldType.int64:
return struct.unpack_from("<q", serialized_data, field_meta.offset)[0]
- elif field_meta.data_type == FieldType.uint64:
+ elif field_meta.data_type == _PyFieldType.uint64:
return struct.unpack_from("<Q", serialized_data, field_meta.offset)[0]
- elif field_meta.data_type == FieldType.float32:
+ elif field_meta.data_type == _PyFieldType.float32:
return struct.unpack_from("<f", serialized_data, field_meta.offset)[0]
- elif field_meta.data_type == FieldType.boolean:
+ elif field_meta.data_type == _PyFieldType.boolean:
# B is 1 byte, endianness doesn't matter, but consistent style
return bool(serialized_data[field_meta.offset])
- elif field_meta.data_type == FieldType.string:
+ elif field_meta.data_type == _PyFieldType.string:
str_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
str_len = struct.unpack_from("<H", serialized_data, str_offset)[0]
str_offset += UINT16_SIZE
return serialized_data[str_offset : str_offset + str_len].decode("utf-8")
- elif field_meta.data_type == FieldType.binary:
+ elif field_meta.data_type == _PyFieldType.binary:
binary_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
binary_len = struct.unpack_from("<I", serialized_data, binary_offset)[0]
binary_offset += UINT32_SIZE
return serialized_data[binary_offset : binary_offset + binary_len]
- elif field_meta.data_type == FieldType.list_string:
+ elif field_meta.data_type == _PyFieldType.list_string:
list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]
list_offset += UINT16_SIZE
@@ -243,13 +246,13 @@ class BytesRow:
str_list[i] = serialized_data[list_offset : list_offset + str_len].decode("utf-8")
list_offset += str_len
return str_list
- elif field_meta.data_type == FieldType.list_int64:
+ elif field_meta.data_type == _PyFieldType.list_int64:
list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]
list_offset += UINT16_SIZE
return list(struct.unpack_from(f"<{list_len}q", serialized_data, list_offset))
- elif field_meta.data_type == FieldType.list_float32:
+ elif field_meta.data_type == _PyFieldType.list_float32:
list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]
list_offset += UINT16_SIZE
@@ -266,3 +269,17 @@ class BytesRow:
data_dict[field_name] = value
return data_dict
+
+
+try:
+ import openviking.storage.vectordb.engine as engine
+
+ # Use C++ implementation if available
+ BytesRow = engine.BytesRow
+ Schema = engine.Schema
+ FieldType = engine.FieldType
+except ImportError:
+ # Fallback to Python implementation
+ BytesRow = _PyBytesRow
+ Schema = _PySchema
+ FieldType = _PyFieldType
diff --git a/openviking/storage/vectordb/store/serializable.py b/openviking/storage/vectordb/store/serializable.py
index 2e4b7376b..1a8a1ae08 100644
--- a/openviking/storage/vectordb/store/serializable.py
+++ b/openviking/storage/vectordb/store/serializable.py
@@ -81,10 +81,12 @@ def serializable(cls):
# Get custom type from metadata, otherwise auto-infer
if f.metadata and "field_type" in f.metadata:
field_type = f.metadata["field_type"]
- if not isinstance(field_type, FieldType):
- raise ValueError(
- f"Field '{field_name}': field_type must be a FieldType, got {type(field_type).__name__}"
- )
+ # Check if it's the correct C++ enum type
+ if hasattr(field_type, "value") and isinstance(field_type.value, int):
+ # Try to find corresponding FieldType
+ # Assuming enum names match
+ if hasattr(FieldType, field_type.name):
+ field_type = getattr(FieldType, field_type.name)
else:
field_type = _python_type_to_field_type(f.type, field_name)
@@ -101,6 +103,7 @@ def serializable(cls):
field_list.append(field_def)
# Create schema and bytes_row
+ # Pass field_list (list of dicts) to C++ Schema constructor
cls.schema = Schema(field_list)
cls.bytes_row = BytesRow(cls.schema)
@@ -112,7 +115,9 @@ def serializable(cls):
def deserialize(self, data: bytes):
data_dict = self.__class__.bytes_row.deserialize(data)
for key, value in data_dict.items():
- setattr(self, key, value)
+ # Handle potential None values if C++ returns std::monostate
+ if value is not None:
+ setattr(self, key, value)
# Automatically generate from_bytes class method
@classmethod
@@ -123,9 +128,18 @@ def serializable(cls):
inst.deserialize(data)
return inst
+ # Automatically generate serialize_list class method
+ @classmethod
+ def serialize_list(cls_method, objects: list) -> list[bytes]:
+ """Batch serialization for a list of objects"""
+ if not objects:
+ return []
+ return cls_method.bytes_row.serialize_batch(objects)
+
# Inject methods into class
cls.serialize = serialize
cls.deserialize = deserialize
cls.from_bytes = from_bytes
+ cls.serialize_list = serialize_list
return cls
diff --git a/openviking/storage/vectordb/store/store_manager.py b/openviking/storage/vectordb/store/store_manager.py
index 6fa5c0923..7a31b6ac8 100644
--- a/openviking/storage/vectordb/store/store_manager.py
+++ b/openviking/storage/vectordb/store/store_manager.py
@@ -95,7 +95,7 @@ class StoreManager:
StoreManager.DeltaTable,
[OpType.PUT] * len(delta_list),
[str(base_ts + i) for i in range(len(delta_list))],
- [data.serialize() for data in delta_list],
+ DeltaRecord.serialize_list(delta_list),
)
)
@@ -109,7 +109,7 @@ class StoreManager:
StoreManager.CandsTable,
[OpType.PUT] * len(cands_list),
[str(data.label) for data in cands_list],
- [data.serialize() for data in cands_list],
+ CandidateData.serialize_list(cands_list),
)
)
@@ -164,7 +164,7 @@ class StoreManager:
StoreManager.DeltaTable,
[OpType.PUT] * len(delta_list),
[str(base_ts + i) for i in range(len(delta_list))],
- [data.serialize() for data in delta_list],
+ DeltaRecord.serialize_list(delta_list),
)
)
@@ -307,7 +307,7 @@ class StoreManager:
StoreManager.DeltaTable,
[OpType.PUT] * len(delta_list),
[str(base_ts + i) for i in range(len(delta_list))],
- [data.serialize() for data in delta_list],
+ DeltaRecord.serialize_list(delta_list),
)
)
diff --git a/src/py_accessors.h b/src/py_accessors.h
new file mode 100644
index 000000000..7712263a9
--- /dev/null
+++ b/src/py_accessors.h
@@ -0,0 +1,276 @@
+// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <cstring>
+#include "store/bytes_row.h"
+
+namespace py = pybind11;
+namespace vdb = vectordb;
+
+// Accessor for Python Dict
+class PyDictAccessor {
+ public:
+ PyDictAccessor(const vdb::Schema& schema)
+ : field_order_(schema.get_field_order()) {
+ }
+
+ bool has_value(const py::dict& row, int field_idx) const {
+ const auto& name = field_order_[field_idx].name;
+ if (!row.contains(name.c_str()))
+ return false;
+ return !row[name.c_str()].is_none();
+ }
+
+ int64_t get_int64(const py::dict& row, int field_idx) const {
+ return row[field_order_[field_idx].name.c_str()].cast<int64_t>();
+ }
+
+ uint64_t get_uint64(const py::dict& row, int field_idx) const {
+ return row[field_order_[field_idx].name.c_str()].cast<uint64_t>();
+ }
+
+ float get_float(const py::dict& row, int field_idx) const {
+ return row[field_order_[field_idx].name.c_str()].cast<float>();
+ }
+
+ bool get_bool(const py::dict& row, int field_idx) const {
+ return row[field_order_[field_idx].name.c_str()].cast<bool>();
+ }
+
+ int get_string_len(const py::dict& row, int field_idx) const {
+ py::object val = row[field_order_[field_idx].name.c_str()];
+ // Assume it's string or bytes
+ if (py::isinstance<py::bytes>(val)) {
+ return PyBytes_Size(val.ptr());
+ }
+ return val.cast<std::string>().length();
+ }
+
+ int get_binary_len(const py::dict& row, int field_idx) const {
+ // Same as string for length
+ return get_string_len(row, field_idx);
+ }
+
+ int get_list_len(const py::dict& row, int field_idx) const {
+ py::list l = row[field_order_[field_idx].name.c_str()].cast<py::list>();
+ return static_cast<int>(l.size());
+ }
+
+ int get_list_string_content_len(const py::dict& row, int field_idx) const {
+ py::list l = row[field_order_[field_idx].name.c_str()].cast<py::list>();
+ int total = 0;
+ for (auto item : l) {
+ total += item.cast<std::string>().length();
+ }
+ return total;
+ }
+
+ // Writers
+ void write_string(const py::dict& row, int field_idx, char* dest) const {
+ std::string s =
+ row[field_order_[field_idx].name.c_str()].cast<std::string>();
+ uint16_t len = static_cast<uint16_t>(s.length());
+ std::memcpy(dest, &len, 2);
+ if (len > 0)
+ std::memcpy(dest + 2, s.data(), len);
+ }
+
+ void write_binary(const py::dict& row, int field_idx, char* dest) const {
+ std::string s =
+ row[field_order_[field_idx].name.c_str()].cast<std::string>();
+ uint32_t len = static_cast<uint32_t>(s.length());
+ std::memcpy(dest, &len, 4);
+ if (len > 0)
+ std::memcpy(dest + 4, s.data(), len);
+ }
+
+ void write_list_int64(const py::dict& row, int field_idx, char* dest) const {
+ py::list l = row[field_order_[field_idx].name.c_str()].cast<py::list>();
+ uint16_t len = static_cast<uint16_t>(l.size());
+ std::memcpy(dest, &len, 2);
+ int64_t* data_ptr = reinterpret_cast<int64_t*>(dest + 2);
+ for (size_t i = 0; i < len; ++i) {
+ data_ptr[i] = l[i].cast<int64_t>();
+ }
+ }
+
+ void write_list_float32(const py::dict& row, int field_idx,
+ char* dest) const {
+ py::list l = row[field_order_[field_idx].name.c_str()].cast<py::list>();
+ uint16_t len = static_cast<uint16_t>(l.size());
+ std::memcpy(dest, &len, 2);
+ float* data_ptr = reinterpret_cast<float*>(dest + 2);
+ for (size_t i = 0; i < len; ++i) {
+ data_ptr[i] = l[i].cast<float>();
+ }
+ }
+
+ void write_list_string(const py::dict& row, int field_idx, char* dest) const {
+ py::list l = row[field_order_[field_idx].name.c_str()].cast<py::list>();
+ uint16_t len = static_cast<uint16_t>(l.size());
+ std::memcpy(dest, &len, 2);
+ char* cur = dest + 2;
+ for (size_t i = 0; i < len; ++i) {
+ std::string s = l[i].cast<std::string>();
+ uint16_t slen = static_cast<uint16_t>(s.length());
+ std::memcpy(cur, &slen, 2);
+ cur += 2;
+ if (slen > 0)
+ std::memcpy(cur, s.data(), slen);
+ cur += slen;
+ }
+ }
+
+ private:
+ const std::vector<vdb::FieldMeta>& field_order_;
+};
+
+// Accessor for Python Object
+class PyObjectAccessor {
+ public:
+ PyObjectAccessor(const vdb::Schema& schema)
+ : field_order_(schema.get_field_order()) {
+ }
+
+ bool has_value(const py::handle& row, int field_idx) const {
+ const char* name = field_order_[field_idx].name.c_str();
+ if (!py::hasattr(row, name))
+ return false;
+ return !row.attr(name).is_none();
+ }
+
+ int64_t get_int64(const py::handle& row, int field_idx) const {
+ return row.attr(field_order_[field_idx].name.c_str()).cast<int64_t>();
+ }
+
+ uint64_t get_uint64(const py::handle& row, int field_idx) const {
+ return row.attr(field_order_[field_idx].name.c_str()).cast<uint64_t>();
+ }
+
+ float get_float(const py::handle& row, int field_idx) const {
+ return row.attr(field_order_[field_idx].name.c_str()).cast<float>();
+ }
+
+ bool get_bool(const py::handle& row, int field_idx) const {
+ return row.attr(field_order_[field_idx].name.c_str()).cast<bool>();
+ }
+
+ int get_string_len(const py::handle& row, int field_idx) const {
+ // See comments in PyDictAccessor about encoding efficiency
+ return row.attr(field_order_[field_idx].name.c_str())
+ .cast<std::string>()
+ .length();
+ }
+
+ int get_binary_len(const py::handle& row, int field_idx) const {
+ return get_string_len(row, field_idx);
+ }
+
+ int get_list_len(const py::handle& row, int field_idx) const {
+ py::list l =
+ row.attr(field_order_[field_idx].name.c_str()).cast<py::list>();
+ return static_cast<int>(l.size());
+ }
+
+ int get_list_string_content_len(const py::handle& row, int field_idx) const {
+ py::list l =
+ row.attr(field_order_[field_idx].name.c_str()).cast<py::list>();
+ int total = 0;
+ for (auto item : l) {
+ total += item.cast<std::string>().length();
+ }
+ return total;
+ }
+
+ void write_string(const py::handle& row, int field_idx, char* dest) const {
+ std::string s =
+ row.attr(field_order_[field_idx].name.c_str()).cast<std::string>();
+ uint16_t len = static_cast<uint16_t>(s.length());
+ std::memcpy(dest, &len, 2);
+ if (len > 0)
+ std::memcpy(dest + 2, s.data(), len);
+ }
+
+ void write_binary(const py::handle& row, int field_idx, char* dest) const {
+ std::string s =
+ row.attr(field_order_[field_idx].name.c_str()).cast<std::string>();
+ uint32_t len = static_cast<uint32_t>(s.length());
+ std::memcpy(dest, &len, 4);
+ if (len > 0)
+ std::memcpy(dest + 4, s.data(), len);
+ }
+
+ void write_list_int64(const py::handle& row, int field_idx,
+ char* dest) const {
+ py::list l =
+ row.attr(field_order_[field_idx].name.c_str()).cast<py::list>();
+ uint16_t len = static_cast<uint16_t>(l.size());
+ std::memcpy(dest, &len, 2);
+ int64_t* data_ptr = reinterpret_cast<int64_t*>(dest + 2);
+ for (size_t i = 0; i < len; ++i) {
+ data_ptr[i] = l[i].cast<int64_t>();
+ }
+ }
+
+ void write_list_float32(const py::handle& row, int field_idx,
+ char* dest) const {
+ py::list l =
+ row.attr(field_order_[field_idx].name.c_str()).cast<py::list>();
+ uint16_t len = static_cast<uint16_t>(l.size());
+ std::memcpy(dest, &len, 2);
+ float* data_ptr = reinterpret_cast<float*>(dest + 2);
+ for (size_t i = 0; i < len; ++i) {
+ data_ptr[i] = l[i].cast<float>();
+ }
+ }
+
+ void write_list_string(const py::handle& row, int field_idx,
+ char* dest) const {
+ py::list l =
+ row.attr(field_order_[field_idx].name.c_str()).cast<py::list>();
+ uint16_t len = static_cast<uint16_t>(l.size());
+ std::memcpy(dest, &len, 2);
+ char* cur = dest + 2;
+ for (size_t i = 0; i < len; ++i) {
+ std::string s = l[i].cast<std::string>();
+ uint16_t slen = static_cast<uint16_t>(s.length());
+ std::memcpy(cur, &slen, 2);
+ cur += 2;
+ if (slen > 0)
+ std::memcpy(cur, s.data(), slen);
+ cur += slen;
+ }
+ }
+
+ private:
+ const std::vector<vdb::FieldMeta>& field_order_;
+};
+
+// Helper to convert C++ Value to Python object
+inline py::object value_to_py(const vdb::Value& val) {
+ if (std::holds_alternative<std::monostate>(val))
+ return py::none();
+ if (std::holds_alternative<int64_t>(val))
+ return py::cast(std::get<int64_t>(val));
+ if (std::holds_alternative<uint64_t>(val))
+ return py::cast(std::get<uint64_t>(val));
+ if (std::holds_alternative<float>(val))
+ return py::cast(std::get<float>(val));
+ if (std::holds_alternative<bool>(val))
+ return py::cast(std::get<bool>(val));
+ if (std::holds_alternative<std::string>(val)) {
+ return py::cast(std::get<std::string>(val));
+ }
+ if (std::holds_alternative<std::vector<int64_t>>(val))
+ return py::cast(std::get<std::vector<int64_t>>(val));
+ if (std::holds_alternative<std::vector<float>>(val))
+ return py::cast(std::get<std::vector<float>>(val));
+ if (std::holds_alternative<std::vector<std::string>>(val))
+ return py::cast(std::get<std::vector<std::string>>(val));
+
+ return py::none();
+}
diff --git a/src/pybind11_interface.cpp b/src/pybind11_interface.cpp
index b5ce96f39..55c55e2d5 100644
--- a/src/pybind11_interface.cpp
+++ b/src/pybind11_interface.cpp
@@ -3,10 +3,14 @@
#include <pybind11/pybind11.h>
#include <pybind11/stl.h>
#include <pybind11/complex.h>
+#include <chrono>
+#include <iostream>
#include "index/index_engine.h"
#include "store/persist_store.h"
#include "store/volatile_store.h"
#include "common/log_utils.h"
+#include "store/bytes_row.h"
+#include "py_accessors.h"
namespace py = pybind11;
namespace vdb = vectordb;
@@ -14,6 +18,140 @@ namespace vdb = vectordb;
PYBIND11_MODULE(engine, m) {
m.def("init_logging", &vdb::init_logging, "Initialize logging");
+ py::enum_<vdb::FieldType>(m, "FieldType")
+ .value("int64", vdb::FieldType::INT64)
+ .value("uint64", vdb::FieldType::UINT64)
+ .value("float32", vdb::FieldType::FLOAT32)
+ .value("string", vdb::FieldType::STRING)
+ .value("binary", vdb::FieldType::BINARY)
+ .value("boolean", vdb::FieldType::BOOLEAN)
+ .value("list_int64", vdb::FieldType::LIST_INT64)
+ .value("list_string", vdb::FieldType::LIST_STRING)
+ .value("list_float32", vdb::FieldType::LIST_FLOAT32);
+
+ py::class_<vdb::Schema, std::shared_ptr<vdb::Schema>>(m, "Schema")
+ .def(py::init([](const py::list& fields_py) {
+ std::vector<vdb::FieldDef> fields;
+ for (const auto& item : fields_py) {
+ py::dict d = item.cast<py::dict>();
+ vdb::FieldDef fd;
+ fd.name = d["name"].cast<std::string>();
+ fd.data_type = d["data_type"].cast<vdb::FieldType>();
+ fd.id = d["id"].cast<int>();
+ if (d.contains("default_value")) {
+ try {
+ switch (fd.data_type) {
+ case vdb::FieldType::INT64:
+ fd.default_value = d["default_value"].cast<int64_t>();
+ break;
+ case vdb::FieldType::UINT64:
+ fd.default_value = d["default_value"].cast<uint64_t>();
+ break;
+ case vdb::FieldType::FLOAT32:
+ fd.default_value = d["default_value"].cast<float>();
+ break;
+ case vdb::FieldType::BOOLEAN:
+ fd.default_value = d["default_value"].cast<bool>();
+ break;
+ case vdb::FieldType::STRING:
+ fd.default_value = d["default_value"].cast<std::string>();
+ break;
+ case vdb::FieldType::BINARY:
+ fd.default_value = d["default_value"].cast<std::string>();
+ break;
+ case vdb::FieldType::LIST_INT64:
+ fd.default_value =
+ d["default_value"].cast<std::vector<int64_t>>();
+ break;
+ case vdb::FieldType::LIST_FLOAT32:
+ fd.default_value =
+ d["default_value"].cast<std::vector<float>>();
+ break;
+ case vdb::FieldType::LIST_STRING:
+ fd.default_value =
+ d["default_value"].cast<std::vector<std::string>>();
+ break;
+ }
+ } catch (...) {
+ fd.default_value = std::monostate{};
+ }
+ } else {
+ fd.default_value = std::monostate{};
+ }
+ fields.push_back(fd);
+ }
+ return std::make_shared<vdb::Schema>(fields);
+ }))
+ .def("get_total_byte_length", &vdb::Schema::get_total_byte_length);
+
+ py::class_<vdb::BytesRow>(m, "BytesRow")
+ .def(py::init<std::shared_ptr<vdb::Schema>>())
+ .def("serialize",
+ [](vdb::BytesRow& self, const py::dict& row_data) {
+ PyDictAccessor accessor(self.get_schema());
+ std::string serialized =
+ self.serialize_template(row_data, accessor);
+ return py::bytes(serialized);
+ })
+ .def("serialize_batch",
+ [](vdb::BytesRow& self, const py::list& objects) {
+ py::list results;
+ const auto& schema = self.get_schema();
+
+ PyDictAccessor dict_accessor(schema);
+ PyObjectAccessor obj_accessor(schema);
+
+ for (const auto& obj : objects) {
+ std::string serialized;
+ if (py::isinstance<py::dict>(obj)) {
+ serialized = self.serialize_template(obj.cast<py::dict>(),
+ dict_accessor);
+ } else {
+ serialized = self.serialize_template(obj, obj_accessor);
+ }
+ results.append(py::bytes(serialized));
+ }
+ return results;
+ })
+ .def("deserialize",
+ [](vdb::BytesRow& self, const std::string& data) {
+ py::dict res_dict;
+ const auto& schema = self.get_schema();
+
+ const auto& field_order = schema.get_field_order();
+ for (const auto& meta : field_order) {
+ vdb::Value val = self.deserialize_field(data, meta.name);
+
+ if (std::holds_alternative<std::monostate>(val))
+ continue;
+
+ if (meta.data_type == vdb::FieldType::BINARY) {
+ if (std::holds_alternative<std::string>(val)) {
+ res_dict[meta.name.c_str()] =
+ py::bytes(std::get<std::string>(val));
+ continue;
+ }
+ }
+ res_dict[meta.name.c_str()] = value_to_py(val);
+ }
+ return res_dict;
+ })
+ .def("deserialize_field",
+ [](vdb::BytesRow& self, const std::string& data,
+ const std::string& field_name) -> py::object {
+ vdb::Value val = self.deserialize_field(data, field_name);
+ const auto& schema = self.get_schema();
+ const auto* meta = schema.get_field_meta(field_name);
+
+ if (meta && meta->data_type == vdb::FieldType::BINARY) {
+ if (std::holds_alternative<std::string>(val)) {
+ const auto& s = std::get<std::string>(val);
+ return py::bytes(s);
+ }
+ }
+ return value_to_py(val);
+ });
+
py::class_<vdb::AddDataRequest>(m, "AddDataRequest")
.def(py::init<>())
.def_readwrite("label", &vdb::AddDataRequest::label)
diff --git a/src/store/bytes_row.cpp b/src/store/bytes_row.cpp
new file mode 100644
index 000000000..a3c181eef
--- /dev/null
+++ b/src/store/bytes_row.cpp
@@ -0,0 +1,501 @@
+#include "bytes_row.h"
+#include <cstring>
+#include <stdexcept>
+#include <algorithm>
+
+namespace vectordb {
+
+// Type size constants
+constexpr int INT64_SIZE = 8;
+constexpr int UINT64_SIZE = 8;
+constexpr int FLOAT32_SIZE = 4;
+constexpr int UINT32_SIZE = 4;
+constexpr int UINT16_SIZE = 2;
+constexpr int BOOL_SIZE = 1;
+
+Schema::Schema(const std::vector<FieldDef>& fields) {
+ int current_offset = 1; // Start after 1 byte header
+
+ if (fields.empty()) {
+ total_byte_length_ = current_offset;
+ return;
+ }
+
+ int max_id = -1;
+ for (const auto& field : fields) {
+ if (field.id < 0) {
+ throw std::invalid_argument("Field id must be non-negative");
+ }
+ if (field.id > max_id)
+ max_id = field.id;
+ }
+
+ if (max_id != static_cast<int>(fields.size()) - 1) {
+ throw std::invalid_argument(
+ "Field ids must be contiguous from 0 to N-1");
+ }
+
+ std::vector<bool> seen(fields.size(), false);
+ for (const auto& field : fields) {
+ if (seen[field.id]) {
+ throw std::invalid_argument("Duplicate field id found");
+ }
+ seen[field.id] = true;
+ }
+
+ field_orders_.resize(fields.size());
+
+ for (const auto& field : fields) {
+ int byte_len = 0;
+
+ // Calculate basic type size
+ switch (field.data_type) {
+ case FieldType::INT64:
+ byte_len = INT64_SIZE;
+ break;
+ case FieldType::UINT64:
+ byte_len = UINT64_SIZE;
+ break;
+ case FieldType::FLOAT32:
+ byte_len = FLOAT32_SIZE;
+ break;
+ case FieldType::BOOLEAN:
+ byte_len = BOOL_SIZE;
+ break;
+ case FieldType::STRING:
+ case FieldType::BINARY:
+ case FieldType::LIST_INT64:
+ case FieldType::LIST_STRING:
+ case FieldType::LIST_FLOAT32:
+ byte_len = UINT32_SIZE; // Offset
+ break;
+ }
+
+ FieldMeta meta;
+ meta.name = field.name;
+ meta.data_type = field.data_type;
+ meta.offset = current_offset;
+ meta.id = field.id;
+ meta.default_value = field.default_value;
+
+ field_metas_[field.name] = meta;
+ field_orders_[field.id] = meta;
+
+ current_offset += byte_len;
+ }
+
+ total_byte_length_ = current_offset;
+}
+
+const FieldMeta* Schema::get_field_meta(const std::string& name) const {
+ auto it = field_metas_.find(name);
+ if (it == field_metas_.end()) {
+ return nullptr;
+ }
+ return &it->second;
+}
+
+BytesRow::BytesRow(std::shared_ptr<Schema> schema) : schema_(schema) {
+}
+
+std::string BytesRow::serialize(const std::vector<Value>& row_data) const {
+ const auto& field_order = schema_->get_field_order();
+
+ // Pass 1: Calculate total size
+ int total_size = schema_->get_total_byte_length();
+ int variable_region_offset = total_size;
+
+ // We'll store intermediate results to avoid recalculating
+ struct VarFieldInfo {
+ int offset; // Where the data starts in the buffer
+ int length; // Length of the data
+ };
+ std::vector<VarFieldInfo> var_infos(field_order.size());
+
+ for (size_t i = 0; i < field_order.size(); ++i) {
+ const auto& meta = field_order[i];
+ const Value& val = (i < row_data.size() &&
+ !std::holds_alternative<std::monostate>(row_data[i]))
+ ? row_data[i]
+ : meta.default_value;
+
+ // Skip fixed size fields calculation (already in total_byte_length_)
+ // Only calculate variable length parts
+ switch (meta.data_type) {
+ case FieldType::STRING: {
+ if (std::holds_alternative<std::string>(val)) {
+ int len = std::get<std::string>(val).length();
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += UINT16_SIZE + len;
+ } else {
+ var_infos[i] = {variable_region_offset, 0};
+ variable_region_offset += UINT16_SIZE;
+ }
+ break;
+ }
+ case FieldType::BINARY: {
+ if (std::holds_alternative<std::string>(
+ val)) { // Binary stored as string
+ int len = std::get<std::string>(val).length();
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += UINT32_SIZE + len;
+ } else {
+ var_infos[i] = {variable_region_offset, 0};
+ variable_region_offset += UINT32_SIZE;
+ }
+ break;
+ }
+ case FieldType::LIST_INT64: {
+ if (std::holds_alternative<std::vector<int64_t>>(val)) {
+ const auto& vec = std::get<std::vector<int64_t>>(val);
+ int len = vec.size();
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += UINT16_SIZE + len * INT64_SIZE;
+ } else {
+ var_infos[i] = {variable_region_offset, 0};
+ variable_region_offset += UINT16_SIZE;
+ }
+ break;
+ }
+ case FieldType::LIST_FLOAT32: {
+ if (std::holds_alternative<std::vector<float>>(val)) {
+ const auto& vec = std::get<std::vector<float>>(val);
+ int len = vec.size();
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += UINT16_SIZE + len * FLOAT32_SIZE;
+ } else {
+ var_infos[i] = {variable_region_offset, 0};
+ variable_region_offset += UINT16_SIZE;
+ }
+ break;
+ }
+ case FieldType::LIST_STRING: {
+ if (std::holds_alternative<std::vector<std::string>>(val)) {
+ const auto& vec = std::get<std::vector<std::string>>(val);
+ int len = vec.size();
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += UINT16_SIZE; // List length
+ for (const auto& s : vec) {
+ variable_region_offset += UINT16_SIZE + s.length();
+ }
+ } else {
+ var_infos[i] = {variable_region_offset, 0};
+ variable_region_offset += UINT16_SIZE;
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ }
+
+ // Allocate buffer
+ std::string buffer;
+ buffer.resize(variable_region_offset);
+ char* ptr = &buffer[0];
+
+ // Write header (field count)
+ // Be careful with alignment if we were doing raw casting, but we use memcpy
+ // so it's fine.
+
+ uint8_t field_count = static_cast<uint8_t>(field_order.size());
+ ptr[0] = field_count;
+
+ // Pass 2: Write data
+ for (size_t i = 0; i < field_order.size(); ++i) {
+ const auto& meta = field_order[i];
+ const Value& val = (i < row_data.size() &&
+ !std::holds_alternative<std::monostate>(row_data[i]))
+ ? row_data[i]
+ : meta.default_value;
+
+ char* field_ptr = ptr + meta.offset;
+
+ switch (meta.data_type) {
+ case FieldType::INT64: {
+ int64_t v = 0;
+ if (std::holds_alternative<int64_t>(val)) {
+ v = std::get<int64_t>(val);
+ } else if (std::holds_alternative<uint64_t>(val)) {
+ // Implicit cast if needed
+ v = static_cast<int64_t>(std::get<uint64_t>(val));
+ }
+ std::memcpy(field_ptr, &v, sizeof(v));
+ break;
+ }
+ case FieldType::UINT64: {
+ uint64_t v = 0;
+ if (std::holds_alternative<uint64_t>(val)) {
+ v = std::get<uint64_t>(val);
+ } else if (std::holds_alternative<int64_t>(val)) {
+ v = static_cast<uint64_t>(std::get<int64_t>(val));
+ }
+ std::memcpy(field_ptr, &v, sizeof(v));
+ break;
+ }
+ case FieldType::FLOAT32: {
+ float v =
+ std::holds_alternative<float>(val) ? std::get<float>(val) : 0.0f;
+ std::memcpy(field_ptr, &v, sizeof(v));
+ break;
+ }
+ case FieldType::BOOLEAN: {
+ bool v =
+ std::holds_alternative<bool>(val) ? std::get<bool>(val) : false;
+ uint8_t b = v ? 1 : 0;
+ std::memcpy(field_ptr, &b, sizeof(b));
+ break;
+ }
+ // Variable length fields: write offset to fixed region, then write data
+ // to variable region
+ case FieldType::STRING:
+ case FieldType::BINARY:
+ case FieldType::LIST_INT64:
+ case FieldType::LIST_FLOAT32:
+ case FieldType::LIST_STRING: {
+ uint32_t offset = static_cast<uint32_t>(var_infos[i].offset);
+ std::memcpy(field_ptr, &offset, sizeof(offset));
+
+ char* var_ptr = ptr + offset;
+
+ if (meta.data_type == FieldType::STRING) {
+ const std::string& s = std::holds_alternative<std::string>(val)
+ ? std::get<std::string>(val)
+ : "";
+ uint16_t len = static_cast<uint16_t>(s.length());
+ std::memcpy(var_ptr, &len, sizeof(len));
+ if (len > 0)
+ std::memcpy(var_ptr + sizeof(len), s.data(), len);
+ } else if (meta.data_type == FieldType::BINARY) {
+ const std::string& s = std::holds_alternative<std::string>(val)
+ ? std::get<std::string>(val)
+ : "";
+ uint32_t len = static_cast<uint32_t>(s.length());
+ std::memcpy(var_ptr, &len, sizeof(len));
+ if (len > 0)
+ std::memcpy(var_ptr + sizeof(len), s.data(), len);
+ } else if (meta.data_type == FieldType::LIST_INT64) {
+ const auto& vec = std::holds_alternative<std::vector<int64_t>>(val)
+ ? std::get<std::vector<int64_t>>(val)
+ : std::vector<int64_t>{};
+ uint16_t len = static_cast<uint16_t>(vec.size());
+ std::memcpy(var_ptr, &len, sizeof(len));
+ if (len > 0)
+ std::memcpy(var_ptr + sizeof(len), vec.data(),
+ len * sizeof(int64_t));
+ } else if (meta.data_type == FieldType::LIST_FLOAT32) {
+ const auto& vec = std::holds_alternative<std::vector<float>>(val)
+ ? std::get<std::vector<float>>(val)
+ : std::vector<float>{};
+ uint16_t len = static_cast<uint16_t>(vec.size());
+ std::memcpy(var_ptr, &len, sizeof(len));
+ if (len > 0)
+ std::memcpy(var_ptr + sizeof(len), vec.data(), len * sizeof(float));
+ } else if (meta.data_type == FieldType::LIST_STRING) {
+ const auto& vec =
+ std::holds_alternative<std::vector<std::string>>(val)
+ ? std::get<std::vector<std::string>>(val)
+ : std::vector<std::string>{};
+ uint16_t len = static_cast<uint16_t>(vec.size());
+ std::memcpy(var_ptr, &len, sizeof(len));
+ var_ptr += sizeof(len);
+ for (const auto& s : vec) {
+ uint16_t s_len = static_cast<uint16_t>(s.length());
+ std::memcpy(var_ptr, &s_len, sizeof(s_len));
+ var_ptr += sizeof(s_len);
+ if (s_len > 0)
+ std::memcpy(var_ptr, s.data(), s_len);
+ var_ptr += s_len;
+ }
+ }
+ break;
+ }
+ }
+ }
+
+ return buffer;
+}
+
+Value BytesRow::deserialize_field(const std::string& serialized_data,
+ const std::string& field_name) const {
+ const FieldMeta* meta_ptr = schema_->get_field_meta(field_name);
+ if (!meta_ptr)
+ return std::monostate{};
+
+ const FieldMeta& meta = *meta_ptr;
+ const char* ptr = serialized_data.data();
+
+ // Check if data is large enough for this field's offset
+ if (serialized_data.size() <= static_cast<size_t>(meta.offset)) {
+ return meta.default_value;
+ }
+
+ uint8_t field_count = static_cast<uint8_t>(ptr[0]);
+ if (meta.id >= field_count) {
+ return meta.default_value;
+ }
+
+ const char* field_ptr = ptr + meta.offset;
+
+ switch (meta.data_type) {
+ case FieldType::INT64: {
+ int64_t v;
+ std::memcpy(&v, field_ptr, sizeof(v));
+ return v;
+ }
+ case FieldType::UINT64: {
+ uint64_t v;
+ std::memcpy(&v, field_ptr, sizeof(v));
+ return v;
+ }
+ case FieldType::FLOAT32: {
+ float v;
+ std::memcpy(&v, field_ptr, sizeof(v));
+ return v;
+ }
+ case FieldType::BOOLEAN: {
+ uint8_t b;
+ std::memcpy(&b, field_ptr, sizeof(b));
+ return (bool)b;
+ }
+ case FieldType::STRING: {
+ uint32_t offset;
+ if (sizeof(offset) >
+ serialized_data.size() -
+ static_cast<size_t>(field_ptr - serialized_data.data()))
+ return std::string("");
+ std::memcpy(&offset, field_ptr, sizeof(offset));
+ if (offset >= serialized_data.size())
+ return std::string("");
+
+ uint16_t len;
+ if (offset + sizeof(len) > serialized_data.size())
+ return std::string("");
+ std::memcpy(&len, ptr + offset, sizeof(len));
+
+ if (static_cast<size_t>(offset) + sizeof(len) + len >
+ serialized_data.size())
+ return std::string("");
+ return std::string(ptr + offset + sizeof(len), len);
+ }
+ case FieldType::BINARY: {
+ uint32_t offset;
+ if (sizeof(offset) >
+ serialized_data.size() -
+ static_cast<size_t>(field_ptr - serialized_data.data()))
+ return std::string("");
+ std::memcpy(&offset, field_ptr, sizeof(offset));
+ if (offset >= serialized_data.size())
+ return std::string("");
+
+ uint32_t len;
+ if (offset + sizeof(len) > serialized_data.size())
+ return std::string("");
+ std::memcpy(&len, ptr + offset, sizeof(len));
+
+ if (static_cast<size_t>(offset) + sizeof(len) + len >
+ serialized_data.size())
+ return std::string("");
+ return std::string(ptr + offset + sizeof(len), len);
+ }
+ case FieldType::LIST_INT64: {
+ uint32_t offset;
+ if (sizeof(offset) >
+ serialized_data.size() -
+ static_cast<size_t>(field_ptr - serialized_data.data()))
+ return std::vector<int64_t>{};
+ std::memcpy(&offset, field_ptr, sizeof(offset));
+ if (offset >= serialized_data.size())
+ return std::vector<int64_t>{};
+
+ uint16_t len;
+ if (offset + sizeof(len) > serialized_data.size())
+ return std::vector<int64_t>{};
+ std::memcpy(&len, ptr + offset, sizeof(len));
+
+ std::vector<int64_t> vec(len);
+ if (len > 0) {
+ if (static_cast<size_t>(offset) + sizeof(len) + len * sizeof(int64_t) >
+ serialized_data.size())
+ return std::vector<int64_t>{};
+ std::memcpy(vec.data(), ptr + offset + sizeof(len),
+ len * sizeof(int64_t));
+ }
+ return vec;
+ }
+ case FieldType::LIST_FLOAT32: {
+ uint32_t offset;
+ if (sizeof(offset) >
+ serialized_data.size() -
+ static_cast<size_t>(field_ptr - serialized_data.data()))
+ return std::vector<float>{};
+ std::memcpy(&offset, field_ptr, sizeof(offset));
+ if (offset >= serialized_data.size())
+ return std::vector<float>{};
+
+ uint16_t len;
+ if (offset + sizeof(len) > serialized_data.size())
+ return std::vector<float>{};
+ std::memcpy(&len, ptr + offset, sizeof(len));
+
+ std::vector<float> vec(len);
+ if (len > 0) {
+ if (static_cast<size_t>(offset) + sizeof(len) + len * sizeof(float) >
+ serialized_data.size())
+ return std::vector<float>{};
+ std::memcpy(vec.data(), ptr + offset + sizeof(len),
+ len * sizeof(float));
+ }
+ return vec;
+ }
+ case FieldType::LIST_STRING: {
+ uint32_t offset;
+ if (sizeof(offset) >
+ serialized_data.size() -
+ static_cast<size_t>(field_ptr - serialized_data.data()))
+ return std::vector<std::string>{};
+ std::memcpy(&offset, field_ptr, sizeof(offset));
+ if (offset >= serialized_data.size())
+ return std::vector<std::string>{};
+
+ const char* var_ptr = ptr + offset;
+ uint16_t list_len;
+
+ if (static_cast<size_t>(offset) + sizeof(list_len) >
+ serialized_data.size())
+ return std::vector<std::string>{};
+ std::memcpy(&list_len, var_ptr, sizeof(list_len));
+ var_ptr += sizeof(list_len);
+
+ std::vector<std::string> vec;
+ vec.reserve(list_len);
+ for (int i = 0; i < list_len; ++i) {
+ uint16_t s_len;
+ if (static_cast<size_t>(var_ptr - ptr) + sizeof(s_len) >
+ serialized_data.size())
+ break;
+ std::memcpy(&s_len, var_ptr, sizeof(s_len));
+ var_ptr += sizeof(s_len);
+
+ if (static_cast<size_t>(var_ptr - ptr) + s_len > serialized_data.size())
+ break;
+ vec.emplace_back(var_ptr, s_len);
+ var_ptr += s_len;
+ }
+ return vec;
+ }
+ }
+ return std::monostate{};
+}
+
+std::map<std::string, Value> BytesRow::deserialize(
+ const std::string& serialized_data) const {
+ std::map<std::string, Value> result;
+ const auto& order = schema_->get_field_order();
+ for (const auto& meta : order) {
+ result[meta.name] = deserialize_field(serialized_data, meta.name);
+ }
+ return result;
+}
+
+} // namespace vectordb
diff --git a/src/store/bytes_row.h b/src/store/bytes_row.h
new file mode 100644
index 000000000..db1ab0e7b
--- /dev/null
+++ b/src/store/bytes_row.h
@@ -0,0 +1,374 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <map>
+#include <variant>
+#include <memory>
+#include <cstdint>
+#include <cstring>
+
+namespace vectordb {
+
+enum class FieldType {
+ INT64 = 0,
+ UINT64 = 1,
+ FLOAT32 = 2,
+ STRING = 3,
+ BINARY = 4,
+ BOOLEAN = 5,
+ LIST_INT64 = 6,
+ LIST_STRING = 7,
+ LIST_FLOAT32 = 8
+};
+
+// Use std::monostate for None/Null
+using Value = std::variant<std::monostate, int64_t, uint64_t, float, std::string, bool, std::vector<int64_t>, std::vector<std::string>, std::vector<float>>;
+
+struct FieldMeta {
+ std::string name;
+ FieldType data_type;
+ int offset;
+ int id;
+ Value default_value;
+};
+
+struct FieldDef {
+ std::string name;
+ FieldType data_type;
+ int id;
+ Value default_value;
+};
+
+class Schema {
+public:
+ Schema(const std::vector<FieldDef>& fields);
+
+ const std::vector<FieldMeta>& get_field_order() const { return field_orders_; }
+ int get_total_byte_length() const { return total_byte_length_; }
+ const FieldMeta* get_field_meta(const std::string& name) const;
+
+private:
+ std::vector<FieldMeta> field_orders_;
+ std::map<std::string, FieldMeta> field_metas_;
+ int total_byte_length_;
+};
+
+class BytesRow {
+public:
+ explicit BytesRow(std::shared_ptr<Schema> schema);
+
+ // Core serialization logic: takes values in order of schema fields
+ std::string serialize(const std::vector<Value>& row_data) const;
+
+ // Generic serialization template
+ // Accessor must implement methods like:
+ // int64_t get_int64(const RowT& row, int field_idx)
+ // ... and so on for all types, plus is_null/has_value check
+ template <typename RowT, typename AccessorT>
+ std::string serialize_template(const RowT& row, const AccessorT& accessor) const {
+ const auto& field_order = schema_->get_field_order();
+
+ auto get_default_string = [](const Value& val) -> const std::string* {
+ if (std::holds_alternative<std::string>(val)) {
+ return &std::get<std::string>(val);
+ }
+ return nullptr;
+ };
+
+ auto get_default_list_int64 = [](const Value& val) -> const std::vector<int64_t>* {
+ if (std::holds_alternative<std::vector<int64_t>>(val)) {
+ return &std::get<std::vector<int64_t>>(val);
+ }
+ return nullptr;
+ };
+
+ auto get_default_list_float32 = [](const Value& val) -> const std::vector<float>* {
+ if (std::holds_alternative<std::vector<float>>(val)) {
+ return &std::get<std::vector<float>>(val);
+ }
+ return nullptr;
+ };
+
+ auto get_default_list_string = [](const Value& val) -> const std::vector<std::string>* {
+ if (std::holds_alternative<std::vector<std::string>>(val)) {
+ return &std::get<std::vector<std::string>>(val);
+ }
+ return nullptr;
+ };
+
+ auto get_list_string_content_len = [](const std::vector<std::string>& vec) -> int {
+ int total = 0;
+ for (const auto& s : vec) {
+ total += static_cast<int>(s.length());
+ }
+ return total;
+ };
+
+ auto write_string_value = [](const std::string& s, char* dest) {
+ uint16_t len = static_cast<uint16_t>(s.length());
+ std::memcpy(dest, &len, 2);
+ if (len > 0) {
+ std::memcpy(dest + 2, s.data(), len);
+ }
+ };
+
+ auto write_binary_value = [](const std::string& s, char* dest) {
+ uint32_t len = static_cast<uint32_t>(s.length());
+ std::memcpy(dest, &len, 4);
+ if (len > 0) {
+ std::memcpy(dest + 4, s.data(), len);
+ }
+ };
+
+ auto write_list_int64_value = [](const std::vector<int64_t>& vec, char* dest) {
+ uint16_t len = static_cast<uint16_t>(vec.size());
+ std::memcpy(dest, &len, 2);
+ if (len > 0) {
+ std::memcpy(dest + 2, vec.data(), len * sizeof(int64_t));
+ }
+ };
+
+ auto write_list_float32_value = [](const std::vector<float>& vec, char* dest) {
+ uint16_t len = static_cast<uint16_t>(vec.size());
+ std::memcpy(dest, &len, 2);
+ if (len > 0) {
+ std::memcpy(dest + 2, vec.data(), len * sizeof(float));
+ }
+ };
+
+ auto write_list_string_value = [](const std::vector<std::string>& vec, char* dest) {
+ uint16_t len = static_cast<uint16_t>(vec.size());
+ std::memcpy(dest, &len, 2);
+ char* cur = dest + 2;
+ for (const auto& s : vec) {
+ uint16_t slen = static_cast<uint16_t>(s.length());
+ std::memcpy(cur, &slen, 2);
+ cur += 2;
+ if (slen > 0) {
+ std::memcpy(cur, s.data(), slen);
+ }
+ cur += slen;
+ }
+ };
+
+ // Pass 1: Calculate total size
+ int total_size = schema_->get_total_byte_length();
+ int variable_region_offset = total_size;
+
+ struct VarFieldInfo {
+ int offset;
+ int length;
+ };
+ // Use a small buffer on stack if possible, or vector
+ std::vector<VarFieldInfo> var_infos(field_order.size());
+
+ for (size_t i = 0; i < field_order.size(); ++i) {
+ const auto& meta = field_order[i];
+
+ // For variable fields, we need to check length
+ switch (meta.data_type) {
+ case FieldType::STRING: {
+ int len = 0;
+ if (accessor.has_value(row, i)) {
+ len = accessor.get_string_len(row, i);
+ } else if (const auto* def = get_default_string(meta.default_value)) {
+ len = static_cast<int>(def->length());
+ }
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += 2 + len; // UINT16_SIZE
+ break;
+ }
+ case FieldType::BINARY: {
+ int len = 0;
+ if (accessor.has_value(row, i)) {
+ len = accessor.get_binary_len(row, i);
+ } else if (const auto* def = get_default_string(meta.default_value)) {
+ len = static_cast<int>(def->length());
+ }
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += 4 + len; // UINT32_SIZE
+ break;
+ }
+ case FieldType::LIST_INT64: {
+ int len = 0;
+ if (accessor.has_value(row, i)) {
+ len = accessor.get_list_len(row, i);
+ } else if (const auto* def = get_default_list_int64(meta.default_value)) {
+ len = static_cast<int>(def->size());
+ }
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += 2 + len * 8; // UINT16 + INT64_SIZE
+ break;
+ }
+ case FieldType::LIST_FLOAT32: {
+ int len = 0;
+ if (accessor.has_value(row, i)) {
+ len = accessor.get_list_len(row, i);
+ } else if (const auto* def = get_default_list_float32(meta.default_value)) {
+ len = static_cast<int>(def->size());
+ }
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += 2 + len * 4; // UINT16 + FLOAT32_SIZE
+ break;
+ }
+ case FieldType::LIST_STRING: {
+ int list_len = 0;
+ int content_len = 0;
+ if (accessor.has_value(row, i)) {
+ list_len = accessor.get_list_len(row, i);
+ content_len = accessor.get_list_string_content_len(row, i);
+ } else if (const auto* def = get_default_list_string(meta.default_value)) {
+ list_len = static_cast<int>(def->size());
+ content_len = get_list_string_content_len(*def);
+ }
+ var_infos[i] = {variable_region_offset, list_len};
+ // list_len(2) + (elem_len(2) + content) * N
+ // Actually content_len should include the 2 bytes for each string length if we compute it that way
+ // Or we compute it here: 2 + (2 * list_len) + total_string_bytes
+ variable_region_offset += 2 + (2 * list_len) + content_len;
+ break;
+ }
+ default: break;
+ }
+ }
+
+ std::string buffer;
+ buffer.resize(variable_region_offset);
+ char* ptr = &buffer[0];
+
+ // Header
+ ptr[0] = static_cast<uint8_t>(field_order.size());
+
+ // Pass 2: Write data
+ for (size_t i = 0; i < field_order.size(); ++i) {
+ const auto& meta = field_order[i];
+ char* field_ptr = ptr + meta.offset;
+ bool has_val = accessor.has_value(row, i);
+
+ switch (meta.data_type) {
+ case FieldType::INT64: {
+ int64_t v = 0;
+ if (has_val) {
+ v = accessor.get_int64(row, i);
+ } else if (std::holds_alternative<int64_t>(meta.default_value)) {
+ v = std::get<int64_t>(meta.default_value);
+ } else if (std::holds_alternative<uint64_t>(meta.default_value)) {
+ v = static_cast<int64_t>(std::get<uint64_t>(meta.default_value));
+ }
+ std::memcpy(field_ptr, &v, sizeof(v));
+ break;
+ }
+ case FieldType::UINT64: {
+ uint64_t v = 0;
+ if (has_val) {
+ v = accessor.get_uint64(row, i);
+ } else if (std::holds_alternative<uint64_t>(meta.default_value)) {
+ v = std::get<uint64_t>(meta.default_value);
+ } else if (std::holds_alternative<int64_t>(meta.default_value)) {
+ v = static_cast<uint64_t>(std::get<int64_t>(meta.default_value));
+ }
+ std::memcpy(field_ptr, &v, sizeof(v));
+ break;
+ }
+ case FieldType::FLOAT32: {
+ float v = 0.0f;
+ if (has_val) {
+ v = accessor.get_float(row, i);
+ } else if (std::holds_alternative<float>(meta.default_value)) {
+ v = std::get<float>(meta.default_value);
+ }
+ std::memcpy(field_ptr, &v, sizeof(v));
+ break;
+ }
+ case FieldType::BOOLEAN: {
+ bool v = false;
+ if (has_val) {
+ v = accessor.get_bool(row, i);
+ } else if (std::holds_alternative<bool>(meta.default_value)) {
+ v = std::get<bool>(meta.default_value);
+ }
+ uint8_t b = v ? 1 : 0;
+ std::memcpy(field_ptr, &b, sizeof(b));
+ break;
+ }
+ case FieldType::STRING:
+ case FieldType::BINARY:
+ case FieldType::LIST_INT64:
+ case FieldType::LIST_FLOAT32:
+ case FieldType::LIST_STRING: {
+ uint32_t offset = static_cast<uint32_t>(var_infos[i].offset);
+ std::memcpy(field_ptr, &offset, sizeof(offset));
+
+ char* var_ptr = ptr + offset;
+
+ if (meta.data_type == FieldType::STRING) {
+ // Logic handled by accessor to write directly? Or return string_view/string?
+ // Accessor returning string creates copy.
+ // Better: accessor.write_string(row, i, var_ptr)
+ if (has_val) {
+ accessor.write_string(row, i, var_ptr);
+ } else if (const auto* def = get_default_string(meta.default_value)) {
+ write_string_value(*def, var_ptr);
+ } else {
+ uint16_t len = 0;
+ std::memcpy(var_ptr, &len, 2);
+ }
+ } else if (meta.data_type == FieldType::BINARY) {
+ if (has_val) {
+ accessor.write_binary(row, i, var_ptr);
+ } else if (const auto* def = get_default_string(meta.default_value)) {
+ write_binary_value(*def, var_ptr);
+ } else {
+ uint32_t len = 0;
+ std::memcpy(var_ptr, &len, 4);
+ }
+ } else if (meta.data_type == FieldType::LIST_INT64) {
+ if (has_val) {
+ accessor.write_list_int64(row, i, var_ptr);
+ } else if (const auto* def = get_default_list_int64(meta.default_value)) {
+ write_list_int64_value(*def, var_ptr);
+ } else {
+ uint16_t len = 0;
+ std::memcpy(var_ptr, &len, 2);
+ }
+ } else if (meta.data_type == FieldType::LIST_FLOAT32) {
+ if (has_val) {
+ accessor.write_list_float32(row, i, var_ptr);
+ } else if (const auto* def = get_default_list_float32(meta.default_value)) {
+ write_list_float32_value(*def, var_ptr);
+ } else {
+ uint16_t len = 0;
+ std::memcpy(var_ptr, &len, 2);
+ }
+ } else if (meta.data_type == FieldType::LIST_STRING) {
+ if (has_val) {
+ accessor.write_list_string(row, i, var_ptr);
+ } else if (const auto* def = get_default_list_string(meta.default_value)) {
+ write_list_string_value(*def, var_ptr);
+ } else {
+ uint16_t len = 0;
+ std::memcpy(var_ptr, &len, 2);
+ }
+ }
+ break;
+ }
+ }
+ }
+ return buffer;
+ }
+
+ // Deserialize to a map
+ std::map<std::string, Value> deserialize(const std::string& serialized_data) const;
+
+ // Deserialize a single field
+ Value deserialize_field(const std::string& serialized_data, const std::string& field_name) const;
+
+ // Get schema
+ const Schema& get_schema() const { return *schema_; }
+
+private:
+ std::shared_ptr<Schema> schema_;
+};
+
+} // namespace vectordb
diff --git a/src/store/persist_store.cpp b/src/store/persist_store.cpp
index b0fd4c2c6..8d0b45d86 100644
--- a/src/store/persist_store.cpp
+++ b/src/store/persist_store.cpp
@@ -55,7 +55,9 @@ int PersistStore::put_data(const std::vector<std::string>& keys,
for (size_t i = 0; i < keys.size(); ++i) {
batch.Put(keys[i], values[i]);
}
- auto status = db_->Write(leveldb::WriteOptions(), &batch);
+ leveldb::WriteOptions write_options;
+ write_options.sync = true;
+ auto status = db_->Write(write_options, &batch);
if (!status.ok()) {
SPDLOG_WARN("Failed to put data db: {}", status.ToString());
return -1;
@@ -68,7 +70,9 @@ int PersistStore::delete_data(const std::vector<std::string>& keys) {
for (const auto& key : keys) {
batch.Delete(key);
}
- auto status = db_->Write(leveldb::WriteOptions(), &batch);
+ leveldb::WriteOptions write_options;
+ write_options.sync = true;
+ auto status = db_->Write(write_options, &batch);
if (!status.ok()) {
SPDLOG_WARN("Failed to delete data db: {}", status.ToString());
return -1;
@@ -82,7 +86,9 @@ int PersistStore::clear_data() {
for (it->SeekToFirst(); it->Valid(); it->Next()) {
batch.Delete(it->key().ToString());
}
- auto status = db_->Write(leveldb::WriteOptions(), &batch);
+ leveldb::WriteOptions write_options;
+ write_options.sync = true;
+ auto status = db_->Write(write_options, &batch);
if (!status.ok()) {
SPDLOG_WARN("Failed to clear data db: {}", status.ToString());
return -1;
@@ -118,7 +124,9 @@ int PersistStore::exec_op(const std::vector<StorageOp>& ops) {
continue;
}
}
- auto status = db_->Write(leveldb::WriteOptions(), &batch);
+ leveldb::WriteOptions write_options;
+ write_options.sync = true;
+ auto status = db_->Write(write_options, &batch);
if (!status.ok()) {
SPDLOG_WARN("Failed to exec op data db: {}", status.ToString());
return -1;
diff --git a/tests/vectordb/test_bytes_row.py b/tests/vectordb/test_bytes_row.py
index 8b1fdbea6..906aa0b45 100644
--- a/tests/vectordb/test_bytes_row.py
+++ b/tests/vectordb/test_bytes_row.py
@@ -1,13 +1,35 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: Apache-2.0
+import json
+import random
+import string
import unittest
from dataclasses import dataclass, field
from typing import List
-from openviking.storage.vectordb.store.bytes_row import FieldType
+from openviking.storage.vectordb import engine
+from openviking.storage.vectordb.store.bytes_row import (
+ FieldType,
+ _PyBytesRow,
+ _PyFieldType,
+ _PySchema,
+)
from openviking.storage.vectordb.store.serializable import serializable
+# Define a complex data structure for testing consistency
+@serializable
+@dataclass
+class ComplexData:
+ label: int = 0
+ vector: List[float] = field(default_factory=list)
+ sparse_raw_terms: List[str] = field(default_factory=list)
+ sparse_values: List[float] = field(default_factory=list)
+ fields: str = ""
+ expire_ns_ts: int = 0
+ is_deleted: bool = False
+
+
class TestBytesRow(unittest.TestCase):
def test_basic_serialization(self):
@serializable
@@ -98,6 +120,169 @@ class TestBytesRow(unittest.TestCase):
val = BinaryData.bytes_row.deserialize_field(serialized, "raw")
self.assertEqual(val, blob)
+ def test_schema_id_validation(self):
+ with self.assertRaises(ValueError):
+ engine.Schema(
+ [
+ {"name": "id", "data_type": engine.FieldType.int64, "id": 0},
+ {"name": "name", "data_type": engine.FieldType.string, "id": 2},
+ ]
+ )
+
+ with self.assertRaises(ValueError):
+ engine.Schema(
+ [
+ {"name": "id", "data_type": engine.FieldType.int64, "id": 0},
+ {"name": "dup", "data_type": engine.FieldType.string, "id": 0},
+ ]
+ )
+
+ def test_missing_fields_use_defaults(self):
+ schema = engine.Schema(
+ [
+ {
+ "name": "id",
+ "data_type": engine.FieldType.int64,
+ "id": 0,
+ "default_value": 7,
+ },
+ {
+ "name": "name",
+ "data_type": engine.FieldType.string,
+ "id": 1,
+ "default_value": "fallback",
+ },
+ {
+ "name": "tags",
+ "data_type": engine.FieldType.list_string,
+ "id": 2,
+ "default_value": ["a", "b"],
+ },
+ {
+ "name": "score",
+ "data_type": engine.FieldType.float32,
+ "id": 3,
+ },
+ ]
+ )
+ row = engine.BytesRow(schema)
+
+ serialized = row.serialize({"id": 5})
+
+ self.assertEqual(row.deserialize_field(serialized, "id"), 5)
+ self.assertEqual(row.deserialize_field(serialized, "name"), "fallback")
+ self.assertEqual(row.deserialize_field(serialized, "tags"), ["a", "b"])
+ self.assertAlmostEqual(row.deserialize_field(serialized, "score"), 0.0, places=5)
+
+
+class TestBytesRowConsistency(unittest.TestCase):
+ def setUp(self):
+ # Create C++ Schema equivalent to ComplexData
+ # Note: IDs must match the order in ComplexData (0-indexed)
+ self.cpp_fields = [
+ {"name": "label", "data_type": engine.FieldType.int64, "id": 0},
+ {"name": "vector", "data_type": engine.FieldType.list_float32, "id": 1},
+ {"name": "sparse_raw_terms", "data_type": engine.FieldType.list_string, "id": 2},
+ {"name": "sparse_values", "data_type": engine.FieldType.list_float32, "id": 3},
+ {"name": "fields", "data_type": engine.FieldType.string, "id": 4},
+ {"name": "expire_ns_ts", "data_type": engine.FieldType.int64, "id": 5},
+ {"name": "is_deleted", "data_type": engine.FieldType.boolean, "id": 6},
+ ]
+ self.cpp_schema = engine.Schema(self.cpp_fields)
+ self.cpp_row = engine.BytesRow(self.cpp_schema)
+
+ # Create Python Schema equivalent to ComplexData
+ self.py_fields = [
+ {"name": "label", "data_type": _PyFieldType.int64, "id": 0},
+ {"name": "vector", "data_type": _PyFieldType.list_float32, "id": 1},
+ {"name": "sparse_raw_terms", "data_type": _PyFieldType.list_string, "id": 2},
+ {"name": "sparse_values", "data_type": _PyFieldType.list_float32, "id": 3},
+ {"name": "fields", "data_type": _PyFieldType.string, "id": 4},
+ {"name": "expire_ns_ts", "data_type": _PyFieldType.int64, "id": 5},
+ {"name": "is_deleted", "data_type": _PyFieldType.boolean, "id": 6},
+ ]
+ self.py_schema = _PySchema(self.py_fields)
+ self.py_row = _PyBytesRow(self.py_schema)
+
+ def generate_random_data(self):
+ dim = 128
+ sparse_dim = 10
+
+ return {
+ "label": random.randint(0, 1000000),
+ "vector": [random.random() for _ in range(dim)],
+ "sparse_raw_terms": [
+ "".join(random.choices(string.ascii_letters, k=5)) for _ in range(sparse_dim)
+ ],
+ "sparse_values": [random.random() for _ in range(sparse_dim)],
+ "fields": json.dumps(
+ {"key": "value", "data": "".join(random.choices(string.ascii_letters, k=50))}
+ ),
+ "expire_ns_ts": 1234567890,
+ "is_deleted": random.choice([True, False]),
+ }
+
+ def test_py_write_cpp_read(self):
+ """Test Python serialization -> C++ deserialization"""
+ data_dict = self.generate_random_data()
+
+ # Python Serialize (using pure Python impl)
+ py_bytes = self.py_row.serialize(data_dict)
+
+ # C++ Deserialize (using ComplexData via serializable or direct engine usage)
+ # Here we use direct engine usage to be explicit
+ cpp_res = self.cpp_row.deserialize(py_bytes)
+
+ # Verify
+ self.assertEqual(cpp_res["label"], data_dict["label"])
+ self.assertEqual(len(cpp_res["vector"]), len(data_dict["vector"]))
+ for a, b in zip(cpp_res["vector"], data_dict["vector"]):
+ self.assertAlmostEqual(a, b, places=5)
+
+ self.assertEqual(cpp_res["sparse_raw_terms"], data_dict["sparse_raw_terms"])
+
+ for a, b in zip(cpp_res["sparse_values"], data_dict["sparse_values"]):
+ self.assertAlmostEqual(a, b, places=5)
+
+ self.assertEqual(cpp_res["fields"], data_dict["fields"])
+ self.assertEqual(cpp_res["expire_ns_ts"], data_dict["expire_ns_ts"])
+ self.assertEqual(cpp_res["is_deleted"], data_dict["is_deleted"])
+
+ def test_cpp_write_py_read(self):
+ """Test C++ serialization -> Python deserialization"""
+ data_dict = self.generate_random_data()
+
+ # C++ Serialize
+ cpp_bytes = self.cpp_row.serialize(data_dict)
+
+ # Python Deserialize
+ py_res = self.py_row.deserialize(cpp_bytes)
+
+ # Verify
+ self.assertEqual(py_res["label"], data_dict["label"])
+ # Check vector with almost equal
+ for a, b in zip(py_res["vector"], data_dict["vector"]):
+ self.assertAlmostEqual(a, b, places=5)
+
+ self.assertEqual(py_res["sparse_raw_terms"], data_dict["sparse_raw_terms"])
+
+ for a, b in zip(py_res["sparse_values"], data_dict["sparse_values"]):
+ self.assertAlmostEqual(a, b, places=5)
+
+ self.assertEqual(py_res["fields"], data_dict["fields"])
+ self.assertEqual(py_res["expire_ns_ts"], data_dict["expire_ns_ts"])
+ self.assertEqual(py_res["is_deleted"], data_dict["is_deleted"])
+
+ def test_binary_consistency(self):
+ """Test that C++ and Python produce identical binary output"""
+ data_dict = self.generate_random_data()
+
+ py_bytes = self.py_row.serialize(data_dict)
+ cpp_bytes = self.cpp_row.serialize(data_dict)
+
+ self.assertEqual(len(py_bytes), len(cpp_bytes), "Binary length mismatch")
+ self.assertEqual(py_bytes, cpp_bytes, "Binary content mismatch")
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/vectordb/test_recall.py b/tests/vectordb/test_recall.py
index b32300694..07fb01f29 100644
--- a/tests/vectordb/test_recall.py
+++ b/tests/vectordb/test_recall.py
@@ -419,6 +419,182 @@ class TestRecall(unittest.TestCase):
)
print("✓ Hybrid dense+sparse mix verified")
+ def test_complex_schema_missing_fields(self):
+ """Test adding data with missing optional fields using complex schema"""
+ print("\n=== Test: Complex Schema Missing Fields ===")
+ dim = 1024
+ name = "test_complex_missing_fields"
+ meta_data = {
+ "CollectionName": name,
+ "Description": "Unified context collection",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},
+ {
+ "FieldName": "uri",
+ "FieldType": "string",
+ }, # Changed path to string for simplicity as 'path' might not be standard FieldType
+ {"FieldName": "type", "FieldType": "string"},
+ {"FieldName": "context_type", "FieldType": "string"},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},
+ {"FieldName": "sparse_vector", "FieldType": "sparse_vector"},
+ {
+ "FieldName": "created_at",
+ "FieldType": "string",
+ }, # Simulating date_time as string
+ {"FieldName": "updated_at", "FieldType": "string"},
+ {"FieldName": "active_count", "FieldType": "int64"},
+ {"FieldName": "parent_uri", "FieldType": "string"},
+ {"FieldName": "is_leaf", "FieldType": "bool"},
+ {"FieldName": "name", "FieldType": "string"},
+ {"FieldName": "description", "FieldType": "string"},
+ {"FieldName": "tags", "FieldType": "string"},
+ {"FieldName": "abstract", "FieldType": "string"},
+ ],
+ }
+
+ collection = self.register_collection(
+ get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)
+ )
+
+ # 1. Full record
+ full_record = {
+ "id": "1",
+ "uri": "/path/to/1",
+ "type": "doc",
+ "context_type": "text",
+ "vector": [0.1] * dim,
+ "sparse_vector": {"t1": 1.0},
+ "created_at": "2023-01-01",
+ "updated_at": "2023-01-02",
+ "active_count": 10,
+ "parent_uri": "/path/to/0",
+ "is_leaf": True,
+ "name": "Doc 1",
+ "description": "A description",
+ "tags": "tag1,tag2",
+ "abstract": "An abstract",
+ }
+
+ # 2. Minimal record (Only ID and Vector are strictly required by engine for indexing usually, but let's see schema validation)
+ # Assuming only PK and Vector are strictly mandatory for vector search index, others should be optional/default.
+ minimal_record = {
+ "id": "2",
+ "vector": [0.2] * dim,
+ }
+
+ # 3. Partial record
+ partial_record = {
+ "id": "3",
+ "vector": [0.3] * dim,
+ "name": "Doc 3",
+ "active_count": 5,
+ }
+
+ collection.upsert_data([full_record, minimal_record, partial_record])
+
+ # Verify data via Fetch
+ res_full = collection.fetch_data(["1"])
+ self.assertEqual(len(res_full.items), 1)
+ self.assertEqual(res_full.items[0].id, "1")
+ # Check fields exist in extra_json or attributes depending on implementation
+ # The result object structure depends on how LocalCollection returns data.
+ # Typically it returns an object where fields are accessible or in 'fields' dict.
+ # Let's assume standard behavior where defined fields are attributes or in a dictionary.
+ # For LocalCollection, non-vector fields are often serialized into a 'fields' JSON string or accessible directly if mapped.
+ # We need to check if the data came back.
+
+ # NOTE: FetchDataResult structure: result_num, labels, scores, extra_json?
+ # Actually fetch_data returns a list of results.
+
+ print(f"Full Record Fetch: {res_full.items[0]}")
+
+ res_min = collection.fetch_data(["2"])
+ self.assertEqual(len(res_min.items), 1)
+ self.assertEqual(res_min.items[0].id, "2")
+ print(f"Minimal Record Fetch: {res_min.items[0]}")
+
+ res_part = collection.fetch_data(["3"])
+ self.assertEqual(len(res_part.items), 1)
+ self.assertEqual(res_part.items[0].id, "3")
+ print(f"Partial Record Fetch: {res_part.items[0]}")
+
+ print("✓ Missing fields handled correctly")
+
+ def test_persistence_crud(self):
+ """Test CRUD operations persist after collection close and reopen"""
+ print("\n=== Test: Persistence CRUD ===")
+ dim = 1024
+ name = "test_persistence"
+ meta_data = {
+ "CollectionName": name,
+ "Description": "Persistence test",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},
+ {"FieldName": "name", "FieldType": "string"},
+ ],
+ }
+
+ # 1. Open and Add Data
+ collection = get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)
+ self.register_collection(collection)
+
+ data = [
+ {"id": "1", "vector": [0.1] * dim, "name": "Item 1"},
+ {"id": "2", "vector": [0.2] * dim, "name": "Item 2"},
+ ]
+ collection.upsert_data(data)
+
+ # Verify Add
+ res = collection.fetch_data(["1", "2"])
+ self.assertEqual(len(res.items), 2)
+
+ # 2. Close Collection (Simulate restart)
+ # Note: LocalCollection might not have an explicit close() that unloads everything from memory
+ # if it's purely object based, but we can delete the object and re-instantiate.
+ # The important part is that data is on disk (RocksDB/LevelDB).
+ collection.close()
+ del collection
+
+ # 3. Reopen
+ collection_new = get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)
+ self.register_collection(collection_new)
+
+ # Verify Data Exists
+ res_reopen = collection_new.fetch_data(["1", "2"])
+ self.assertEqual(len(res_reopen.items), 2)
+ # Order is not guaranteed, so check by ID or sort
+ ids = sorted([item.id for item in res_reopen.items])
+ self.assertEqual(ids, ["1", "2"])
+
+ # 4. Update Data
+ update_data = [{"id": "1", "vector": [0.9] * dim, "name": "Item 1 Updated"}]
+ collection_new.upsert_data(update_data)
+
+ res_update = collection_new.fetch_data(["1"])
+ self.assertEqual(len(res_update.items), 1)
+ self.assertEqual(res_update.items[0].fields["name"], "Item 1 Updated")
+
+ # 5. Delete Data
+ collection_new.delete_data(["2"])
+
+ res_del = collection_new.fetch_data(["2"])
+ self.assertEqual(len(res_del.items), 0, "Deleted item should not be found")
+ self.assertEqual(len(res_del.ids_not_exist), 1)
+
+ # 6. Search on persisted data
+ collection_new.create_index(
+ "idx_persist",
+ {"IndexName": "idx_persist", "VectorIndex": {"IndexType": "flat", "Distance": "l2"}},
+ )
+ search_res = collection_new.search_by_vector(
+ "idx_persist", dense_vector=[0.9] * dim, limit=1
+ )
+ self.assertEqual(len(search_res.data), 1)
+ self.assertEqual(search_res.data[0].id, "1")
+
+ print("✓ Persistence verified")
+
if __name__ == "__main__":
unittest.main()