Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(vectordb): add text field type for large strings (#3725)

Jiahui Zhouf857349d6bc0
diff --git a/openviking/storage/vectordb/engine/_python_api.py b/openviking/storage/vectordb/engine/_python_api.py
index 9ae9f1d3e..d7c62dfc1 100644
--- a/openviking/storage/vectordb/engine/_python_api.py
+++ b/openviking/storage/vectordb/engine/_python_api.py
@@ -25,6 +25,7 @@ class FieldType(IntEnum):
list_int64 = 6
list_string = 7
list_float32 = 8
+ text = 9
class StorageOpType(IntEnum):
@@ -52,6 +53,7 @@ def _default_value_for_type(data_type: FieldType) -> Any:
FieldType.list_int64: [],
FieldType.list_string: [],
FieldType.list_float32: [],
+ FieldType.text: "",
}[data_type]
@@ -66,6 +68,7 @@ def _field_type_size(data_type: FieldType) -> int:
FieldType.list_int64: UINT32_SIZE,
FieldType.list_string: UINT32_SIZE,
FieldType.list_float32: UINT32_SIZE,
+ FieldType.text: UINT32_SIZE,
}[data_type]
@@ -243,6 +246,11 @@ class BytesRow:
blob_len = struct.unpack_from("<I", serialized_data, blob_offset)[0]
blob_offset += UINT32_SIZE
return serialized_data[blob_offset : blob_offset + blob_len]
+ if field_meta.data_type == FieldType.text:
+ text_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
+ text_len = struct.unpack_from("<I", serialized_data, text_offset)[0]
+ text_offset += UINT32_SIZE
+ return serialized_data[text_offset : text_offset + text_len].decode("utf-8")
if field_meta.data_type == FieldType.list_int64:
list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]
diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.py
index fc14f4e23..cd9f2ad00 100644
--- a/openviking/storage/vectordb/store/bytes_row.py
+++ b/openviking/storage/vectordb/store/bytes_row.py
@@ -36,6 +36,7 @@ class _PyFieldType(Enum):
list_int64 = 6
list_string = 7
list_float32 = 8
+ text = 9
class _PySchema:
@@ -67,6 +68,7 @@ class _PySchema:
_PyFieldType.list_int64: (UINT32_SIZE, [0]),
_PyFieldType.list_string: (UINT32_SIZE, ["default"]),
_PyFieldType.list_float32: (UINT32_SIZE, [0.0]),
+ _PyFieldType.text: (UINT32_SIZE, ""),
}
for field in fields:
@@ -167,6 +169,18 @@ class _PyBytesRow:
var_fmt_list.append(f"{len(value)}s")
var_val_list.append(value)
variable_region_offset += len(value)
+ elif field_meta.data_type == _PyFieldType.text:
+ fix_fmt_list.append("I")
+ fix_val_list.append(variable_region_offset)
+ fix_region_offset += UINT32_SIZE
+ bytes_item = value.encode("utf-8")
+ bytes_item_len = len(bytes_item)
+ var_fmt_list.append("I")
+ var_val_list.append(bytes_item_len)
+ variable_region_offset += UINT32_SIZE
+ var_fmt_list.append(f"{bytes_item_len}s")
+ var_val_list.append(bytes_item)
+ variable_region_offset += bytes_item_len
elif field_meta.data_type == _PyFieldType.list_int64:
fix_fmt_list.append("I")
fix_val_list.append(variable_region_offset)
@@ -240,6 +254,11 @@ class _PyBytesRow:
binary_len = struct.unpack_from("<I", serialized_data, binary_offset)[0]
binary_offset += UINT32_SIZE
return serialized_data[binary_offset : binary_offset + binary_len]
+ elif field_meta.data_type == _PyFieldType.text:
+ text_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
+ text_len = struct.unpack_from("<I", serialized_data, text_offset)[0]
+ text_offset += UINT32_SIZE
+ return serialized_data[text_offset : text_offset + text_len].decode("utf-8")
elif field_meta.data_type == _PyFieldType.list_string:
list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]
diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cpp
index d2a893442..f88f09d09 100644
--- a/src/abi3_engine_backend.cpp
+++ b/src/abi3_engine_backend.cpp
@@ -365,6 +365,13 @@ PyObject* value_to_py(const vdb::Value& value, vdb::FieldType field_type) {
text.data(), static_cast<Py_ssize_t>(text.size()));
}
break;
+ case vdb::FieldType::TEXT:
+ if (std::holds_alternative<std::string>(value)) {
+ const auto& text = std::get<std::string>(value);
+ return PyUnicode_FromStringAndSize(
+ text.data(), static_cast<Py_ssize_t>(text.size()));
+ }
+ break;
case vdb::FieldType::BINARY:
if (std::holds_alternative<std::string>(value)) {
const auto& blob = std::get<std::string>(value);
@@ -433,7 +440,8 @@ bool py_to_field_value(PyObject* obj, vdb::FieldType data_type,
return true;
}
case vdb::FieldType::STRING:
- case vdb::FieldType::BINARY: {
+ case vdb::FieldType::BINARY:
+ case vdb::FieldType::TEXT: {
std::string value;
if (!py_to_string(obj, &value, true)) {
return false;
@@ -546,6 +554,9 @@ bool py_to_field_type(PyObject* obj, vdb::FieldType* out) {
case 8:
*out = vdb::FieldType::LIST_FLOAT32;
return true;
+ case 9:
+ *out = vdb::FieldType::TEXT;
+ return true;
default:
raise_value_error("Unsupported field type");
return false;
diff --git a/src/store/bytes_row.cpp b/src/store/bytes_row.cpp
index cd65e5c46..7f6014e41 100644
--- a/src/store/bytes_row.cpp
+++ b/src/store/bytes_row.cpp
@@ -63,6 +63,7 @@ Schema::Schema(const std::vector<FieldDef>& fields) {
break;
case FieldType::STRING:
case FieldType::BINARY:
+ case FieldType::TEXT:
case FieldType::LIST_INT64:
case FieldType::LIST_STRING:
case FieldType::LIST_FLOAT32:
@@ -148,6 +149,17 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {
}
break;
}
+ case FieldType::TEXT: {
+ if (std::holds_alternative<std::string>(val)) {
+ int len = std::get<std::string>(val).length();
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += UINT32_SIZE + len;
+ } else {
+ var_infos[i] = {variable_region_offset, 0};
+ variable_region_offset += UINT32_SIZE;
+ }
+ break;
+ }
case FieldType::LIST_INT64: {
if (std::holds_alternative<std::vector<int64_t>>(val)) {
const auto& vec = std::get<std::vector<int64_t>>(val);
@@ -253,6 +265,7 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {
// to variable region
case FieldType::STRING:
case FieldType::BINARY:
+ case FieldType::TEXT:
case FieldType::LIST_INT64:
case FieldType::LIST_FLOAT32:
case FieldType::LIST_STRING: {
@@ -269,7 +282,8 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {
std::memcpy(var_ptr, &len, sizeof(len));
if (len > 0)
std::memcpy(var_ptr + sizeof(len), s.data(), len);
- } else if (meta.data_type == FieldType::BINARY) {
+ } else if (meta.data_type == FieldType::BINARY ||
+ meta.data_type == FieldType::TEXT) {
const std::string& s = std::holds_alternative<std::string>(val)
? std::get<std::string>(val)
: "";
@@ -401,6 +415,26 @@ Value BytesRow::deserialize_field(const std::string& serialized_data,
return std::string("");
return std::string(ptr + offset + sizeof(len), len);
}
+ case FieldType::TEXT: {
+ uint32_t offset;
+ if (sizeof(offset) >
+ serialized_data.size() -
+ static_cast<size_t>(field_ptr - serialized_data.data()))
+ return std::string("");
+ std::memcpy(&offset, field_ptr, sizeof(offset));
+ if (offset >= serialized_data.size())
+ return std::string("");
+
+ uint32_t len;
+ if (offset + sizeof(len) > serialized_data.size())
+ return std::string("");
+ std::memcpy(&len, ptr + offset, sizeof(len));
+
+ if (static_cast<size_t>(offset) + sizeof(len) + len >
+ serialized_data.size())
+ return std::string("");
+ return std::string(ptr + offset + sizeof(len), len);
+ }
case FieldType::LIST_INT64: {
uint32_t offset;
if (sizeof(offset) >
diff --git a/src/store/bytes_row.h b/src/store/bytes_row.h
index 881e85573..575870da0 100644
--- a/src/store/bytes_row.h
+++ b/src/store/bytes_row.h
@@ -22,7 +22,8 @@ enum class FieldType {
BOOLEAN = 5,
LIST_INT64 = 6,
LIST_STRING = 7,
- LIST_FLOAT32 = 8
+ LIST_FLOAT32 = 8,
+ TEXT = 9
};
// Use std::monostate for None/Null
@@ -212,6 +213,17 @@ class BytesRow {
variable_region_offset += 4 + len; // UINT32_SIZE
break;
}
+ case FieldType::TEXT: {
+ int len = 0;
+ if (accessor.has_value(row, i)) {
+ len = accessor.get_binary_len(row, i);
+ } else if (const auto* def = get_default_string(meta.default_value)) {
+ len = static_cast<int>(def->length());
+ }
+ var_infos[i] = {variable_region_offset, len};
+ variable_region_offset += 4 + len; // UINT32_SIZE
+ break;
+ }
case FieldType::LIST_INT64: {
int len = 0;
if (accessor.has_value(row, i)) {
@@ -321,6 +333,7 @@ class BytesRow {
}
case FieldType::STRING:
case FieldType::BINARY:
+ case FieldType::TEXT:
case FieldType::LIST_INT64:
case FieldType::LIST_FLOAT32:
case FieldType::LIST_STRING: {
@@ -342,7 +355,8 @@ class BytesRow {
uint16_t len = 0;
std::memcpy(var_ptr, &len, 2);
}
- } else if (meta.data_type == FieldType::BINARY) {
+ } else if (meta.data_type == FieldType::BINARY ||
+ meta.data_type == FieldType::TEXT) {
if (has_val) {
accessor.write_binary(row, i, var_ptr);
} else if (const auto* def =
diff --git a/tests/vectordb/test_bytes_row.py b/tests/vectordb/test_bytes_row.py
index 264623a2c..394dcccae 100644
--- a/tests/vectordb/test_bytes_row.py
+++ b/tests/vectordb/test_bytes_row.py
@@ -301,5 +301,69 @@ class TestBytesRowConsistency(unittest.TestCase):
self.assertEqual(py_bytes, cpp_bytes, "Binary content mismatch")
+class TestTextFieldType(unittest.TestCase):
+ """The `text` field type mirrors STRING but uses a uint32 length prefix,
+ so it supports values larger than the 65535-byte STRING limit."""
+
+ def setUp(self):
+ self.cpp_fields = [
+ {"name": "label", "data_type": engine.FieldType.int64, "id": 0},
+ {"name": "body", "data_type": engine.FieldType.text, "id": 1},
+ ]
+ self.cpp_schema = engine.Schema(self.cpp_fields)
+ self.cpp_row = engine.BytesRow(self.cpp_schema)
+
+ self.py_fields = [
+ {"name": "label", "data_type": _PyFieldType.int64, "id": 0},
+ {"name": "body", "data_type": _PyFieldType.text, "id": 1},
+ ]
+ self.py_schema = _PySchema(self.py_fields)
+ self.py_row = _PyBytesRow(self.py_schema)
+
+ def _make_data(self):
+ # Well beyond the 65535-byte STRING limit, plus multibyte content.
+ return {"label": 42, "body": ("你好x" * 40000)}
+
+ def test_large_text_roundtrip_cpp(self):
+ data = self._make_data()
+ serialized = self.cpp_row.serialize(data)
+ self.assertEqual(self.cpp_row.deserialize_field(serialized, "body"), data["body"])
+ self.assertEqual(self.cpp_row.deserialize_field(serialized, "label"), 42)
+
+ def test_large_text_roundtrip_py(self):
+ data = self._make_data()
+ serialized = self.py_row.serialize(data)
+ self.assertEqual(self.py_row.deserialize_field(serialized, "body"), data["body"])
+ self.assertEqual(self.py_row.deserialize_field(serialized, "label"), 42)
+
+ def test_py_write_cpp_read(self):
+ data = self._make_data()
+ py_bytes = self.py_row.serialize(data)
+ self.assertEqual(self.cpp_row.deserialize_field(py_bytes, "body"), data["body"])
+
+ def test_cpp_write_py_read(self):
+ data = self._make_data()
+ cpp_bytes = self.cpp_row.serialize(data)
+ self.assertEqual(self.py_row.deserialize_field(cpp_bytes, "body"), data["body"])
+
+ def test_binary_consistency(self):
+ data = self._make_data()
+ py_bytes = self.py_row.serialize(data)
+ cpp_bytes = self.cpp_row.serialize(data)
+ self.assertEqual(len(py_bytes), len(cpp_bytes), "Binary length mismatch")
+ self.assertEqual(py_bytes, cpp_bytes, "Binary content mismatch")
+
+ def test_text_declared_via_metadata(self):
+ @serializable
+ @dataclass
+ class TextData:
+ body: str = field(default="", metadata={"field_type": FieldType.text})
+
+ text = "y" * 70000
+ data = TextData(body=text)
+ serialized = data.serialize()
+ self.assertEqual(TextData.from_bytes(serialized).body, text)
+
+
if __name__ == "__main__":
unittest.main()