Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(vectordb): add text field type for large strings (#3725)
Jiahui Zhouf857349d6bc0
diff --git a/openviking/storage/vectordb/engine/_python_api.py b/openviking/storage/vectordb/engine/_python_api.pyindex 9ae9f1d3e..d7c62dfc1 100644--- a/openviking/storage/vectordb/engine/_python_api.py+++ b/openviking/storage/vectordb/engine/_python_api.py@@ -25,6 +25,7 @@ class FieldType(IntEnum):list_int64 = 6list_string = 7list_float32 = 8+ text = 9class StorageOpType(IntEnum):@@ -52,6 +53,7 @@ def _default_value_for_type(data_type: FieldType) -> Any:FieldType.list_int64: [],FieldType.list_string: [],FieldType.list_float32: [],+ FieldType.text: "",}[data_type]@@ -66,6 +68,7 @@ def _field_type_size(data_type: FieldType) -> int:FieldType.list_int64: UINT32_SIZE,FieldType.list_string: UINT32_SIZE,FieldType.list_float32: UINT32_SIZE,+ FieldType.text: UINT32_SIZE,}[data_type]@@ -243,6 +246,11 @@ class BytesRow:blob_len = struct.unpack_from("<I", serialized_data, blob_offset)[0]blob_offset += UINT32_SIZEreturn serialized_data[blob_offset : blob_offset + blob_len]+ if field_meta.data_type == FieldType.text:+ text_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]+ text_len = struct.unpack_from("<I", serialized_data, text_offset)[0]+ text_offset += UINT32_SIZE+ return serialized_data[text_offset : text_offset + text_len].decode("utf-8")if field_meta.data_type == FieldType.list_int64:list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.pyindex fc14f4e23..cd9f2ad00 100644--- a/openviking/storage/vectordb/store/bytes_row.py+++ b/openviking/storage/vectordb/store/bytes_row.py@@ -36,6 +36,7 @@ class _PyFieldType(Enum):list_int64 = 6list_string = 7list_float32 = 8+ text = 9class _PySchema:@@ -67,6 +68,7 @@ class _PySchema:_PyFieldType.list_int64: (UINT32_SIZE, [0]),_PyFieldType.list_string: (UINT32_SIZE, ["default"]),_PyFieldType.list_float32: (UINT32_SIZE, [0.0]),+ _PyFieldType.text: (UINT32_SIZE, ""),}for field in fields:@@ -167,6 +169,18 @@ class _PyBytesRow:var_fmt_list.append(f"{len(value)}s")var_val_list.append(value)variable_region_offset += len(value)+ elif field_meta.data_type == _PyFieldType.text:+ fix_fmt_list.append("I")+ fix_val_list.append(variable_region_offset)+ fix_region_offset += UINT32_SIZE+ bytes_item = value.encode("utf-8")+ bytes_item_len = len(bytes_item)+ var_fmt_list.append("I")+ var_val_list.append(bytes_item_len)+ variable_region_offset += UINT32_SIZE+ var_fmt_list.append(f"{bytes_item_len}s")+ var_val_list.append(bytes_item)+ variable_region_offset += bytes_item_lenelif field_meta.data_type == _PyFieldType.list_int64:fix_fmt_list.append("I")fix_val_list.append(variable_region_offset)@@ -240,6 +254,11 @@ class _PyBytesRow:binary_len = struct.unpack_from("<I", serialized_data, binary_offset)[0]binary_offset += UINT32_SIZEreturn serialized_data[binary_offset : binary_offset + binary_len]+ elif field_meta.data_type == _PyFieldType.text:+ text_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]+ text_len = struct.unpack_from("<I", serialized_data, text_offset)[0]+ text_offset += UINT32_SIZE+ return serialized_data[text_offset : text_offset + text_len].decode("utf-8")elif field_meta.data_type == _PyFieldType.list_string:list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cppindex d2a893442..f88f09d09 100644--- a/src/abi3_engine_backend.cpp+++ b/src/abi3_engine_backend.cpp@@ -365,6 +365,13 @@ PyObject* value_to_py(const vdb::Value& value, vdb::FieldType field_type) {text.data(), static_cast<Py_ssize_t>(text.size()));}break;+ case vdb::FieldType::TEXT:+ if (std::holds_alternative<std::string>(value)) {+ const auto& text = std::get<std::string>(value);+ return PyUnicode_FromStringAndSize(+ text.data(), static_cast<Py_ssize_t>(text.size()));+ }+ break;case vdb::FieldType::BINARY:if (std::holds_alternative<std::string>(value)) {const auto& blob = std::get<std::string>(value);@@ -433,7 +440,8 @@ bool py_to_field_value(PyObject* obj, vdb::FieldType data_type,return true;}case vdb::FieldType::STRING:- case vdb::FieldType::BINARY: {+ case vdb::FieldType::BINARY:+ case vdb::FieldType::TEXT: {std::string value;if (!py_to_string(obj, &value, true)) {return false;@@ -546,6 +554,9 @@ bool py_to_field_type(PyObject* obj, vdb::FieldType* out) {case 8:*out = vdb::FieldType::LIST_FLOAT32;return true;+ case 9:+ *out = vdb::FieldType::TEXT;+ return true;default:raise_value_error("Unsupported field type");return false;diff --git a/src/store/bytes_row.cpp b/src/store/bytes_row.cppindex cd65e5c46..7f6014e41 100644--- a/src/store/bytes_row.cpp+++ b/src/store/bytes_row.cpp@@ -63,6 +63,7 @@ Schema::Schema(const std::vector<FieldDef>& fields) {break;case FieldType::STRING:case FieldType::BINARY:+ case FieldType::TEXT:case FieldType::LIST_INT64:case FieldType::LIST_STRING:case FieldType::LIST_FLOAT32:@@ -148,6 +149,17 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {}break;}+ case FieldType::TEXT: {+ if (std::holds_alternative<std::string>(val)) {+ int len = std::get<std::string>(val).length();+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += UINT32_SIZE + len;+ } else {+ var_infos[i] = {variable_region_offset, 0};+ variable_region_offset += UINT32_SIZE;+ }+ break;+ }case FieldType::LIST_INT64: {if (std::holds_alternative<std::vector<int64_t>>(val)) {const auto& vec = std::get<std::vector<int64_t>>(val);@@ -253,6 +265,7 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {// to variable regioncase FieldType::STRING:case FieldType::BINARY:+ case FieldType::TEXT:case FieldType::LIST_INT64:case FieldType::LIST_FLOAT32:case FieldType::LIST_STRING: {@@ -269,7 +282,8 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {std::memcpy(var_ptr, &len, sizeof(len));if (len > 0)std::memcpy(var_ptr + sizeof(len), s.data(), len);- } else if (meta.data_type == FieldType::BINARY) {+ } else if (meta.data_type == FieldType::BINARY ||+ meta.data_type == FieldType::TEXT) {const std::string& s = std::holds_alternative<std::string>(val)? std::get<std::string>(val): "";@@ -401,6 +415,26 @@ Value BytesRow::deserialize_field(const std::string& serialized_data,return std::string("");return std::string(ptr + offset + sizeof(len), len);}+ case FieldType::TEXT: {+ uint32_t offset;+ if (sizeof(offset) >+ serialized_data.size() -+ static_cast<size_t>(field_ptr - serialized_data.data()))+ return std::string("");+ std::memcpy(&offset, field_ptr, sizeof(offset));+ if (offset >= serialized_data.size())+ return std::string("");++ uint32_t len;+ if (offset + sizeof(len) > serialized_data.size())+ return std::string("");+ std::memcpy(&len, ptr + offset, sizeof(len));++ if (static_cast<size_t>(offset) + sizeof(len) + len >+ serialized_data.size())+ return std::string("");+ return std::string(ptr + offset + sizeof(len), len);+ }case FieldType::LIST_INT64: {uint32_t offset;if (sizeof(offset) >diff --git a/src/store/bytes_row.h b/src/store/bytes_row.hindex 881e85573..575870da0 100644--- a/src/store/bytes_row.h+++ b/src/store/bytes_row.h@@ -22,7 +22,8 @@ enum class FieldType {BOOLEAN = 5,LIST_INT64 = 6,LIST_STRING = 7,- LIST_FLOAT32 = 8+ LIST_FLOAT32 = 8,+ TEXT = 9};// Use std::monostate for None/Null@@ -212,6 +213,17 @@ class BytesRow {variable_region_offset += 4 + len; // UINT32_SIZEbreak;}+ case FieldType::TEXT: {+ int len = 0;+ if (accessor.has_value(row, i)) {+ len = accessor.get_binary_len(row, i);+ } else if (const auto* def = get_default_string(meta.default_value)) {+ len = static_cast<int>(def->length());+ }+ var_infos[i] = {variable_region_offset, len};+ variable_region_offset += 4 + len; // UINT32_SIZE+ break;+ }case FieldType::LIST_INT64: {int len = 0;if (accessor.has_value(row, i)) {@@ -321,6 +333,7 @@ class BytesRow {}case FieldType::STRING:case FieldType::BINARY:+ case FieldType::TEXT:case FieldType::LIST_INT64:case FieldType::LIST_FLOAT32:case FieldType::LIST_STRING: {@@ -342,7 +355,8 @@ class BytesRow {uint16_t len = 0;std::memcpy(var_ptr, &len, 2);}- } else if (meta.data_type == FieldType::BINARY) {+ } else if (meta.data_type == FieldType::BINARY ||+ meta.data_type == FieldType::TEXT) {if (has_val) {accessor.write_binary(row, i, var_ptr);} else if (const auto* def =diff --git a/tests/vectordb/test_bytes_row.py b/tests/vectordb/test_bytes_row.pyindex 264623a2c..394dcccae 100644--- a/tests/vectordb/test_bytes_row.py+++ b/tests/vectordb/test_bytes_row.py@@ -301,5 +301,69 @@ class TestBytesRowConsistency(unittest.TestCase):self.assertEqual(py_bytes, cpp_bytes, "Binary content mismatch")+class TestTextFieldType(unittest.TestCase):+ """The `text` field type mirrors STRING but uses a uint32 length prefix,+ so it supports values larger than the 65535-byte STRING limit."""++ def setUp(self):+ self.cpp_fields = [+ {"name": "label", "data_type": engine.FieldType.int64, "id": 0},+ {"name": "body", "data_type": engine.FieldType.text, "id": 1},+ ]+ self.cpp_schema = engine.Schema(self.cpp_fields)+ self.cpp_row = engine.BytesRow(self.cpp_schema)++ self.py_fields = [+ {"name": "label", "data_type": _PyFieldType.int64, "id": 0},+ {"name": "body", "data_type": _PyFieldType.text, "id": 1},+ ]+ self.py_schema = _PySchema(self.py_fields)+ self.py_row = _PyBytesRow(self.py_schema)++ def _make_data(self):+ # Well beyond the 65535-byte STRING limit, plus multibyte content.+ return {"label": 42, "body": ("你好x" * 40000)}++ def test_large_text_roundtrip_cpp(self):+ data = self._make_data()+ serialized = self.cpp_row.serialize(data)+ self.assertEqual(self.cpp_row.deserialize_field(serialized, "body"), data["body"])+ self.assertEqual(self.cpp_row.deserialize_field(serialized, "label"), 42)++ def test_large_text_roundtrip_py(self):+ data = self._make_data()+ serialized = self.py_row.serialize(data)+ self.assertEqual(self.py_row.deserialize_field(serialized, "body"), data["body"])+ self.assertEqual(self.py_row.deserialize_field(serialized, "label"), 42)++ def test_py_write_cpp_read(self):+ data = self._make_data()+ py_bytes = self.py_row.serialize(data)+ self.assertEqual(self.cpp_row.deserialize_field(py_bytes, "body"), data["body"])++ def test_cpp_write_py_read(self):+ data = self._make_data()+ cpp_bytes = self.cpp_row.serialize(data)+ self.assertEqual(self.py_row.deserialize_field(cpp_bytes, "body"), data["body"])++ def test_binary_consistency(self):+ data = self._make_data()+ py_bytes = self.py_row.serialize(data)+ cpp_bytes = self.cpp_row.serialize(data)+ self.assertEqual(len(py_bytes), len(cpp_bytes), "Binary length mismatch")+ self.assertEqual(py_bytes, cpp_bytes, "Binary content mismatch")++ def test_text_declared_via_metadata(self):+ @serializable+ @dataclass+ class TextData:+ body: str = field(default="", metadata={"field_type": FieldType.text})++ text = "y" * 70000+ data = TextData(body=text)+ serialized = data.serialize()+ self.assertEqual(TextData.from_bytes(serialized).body, text)++if __name__ == "__main__":unittest.main()