Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(vectordb): reject oversized bytes row strings (#2171)
Qin Haojief1afd40a6661
diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.pyindex 58ae4c18d..fc14f4e23 100644--- a/openviking/storage/vectordb/store/bytes_row.py+++ b/openviking/storage/vectordb/store/bytes_row.py@@ -12,6 +12,7 @@ FLOAT32_SIZE = 4UINT32_SIZE = 4 # Used for string/binary length and offsetUINT16_SIZE = 2 # Used for list length and string/binary length inside listsBOOL_SIZE = 1+STRING_MAX_UINT16_LENGTH = 0xFFFF@dataclass@@ -146,6 +147,10 @@ class _PyBytesRow:fix_region_offset += UINT32_SIZEbytes_item = value.encode("utf-8")bytes_item_len = len(bytes_item)+ if bytes_item_len > STRING_MAX_UINT16_LENGTH:+ raise ValueError(+ f"string field '{field_meta.name}' exceeds 65535 bytes"+ )var_fmt_list.append("H")var_val_list.append(bytes_item_len)variable_region_offset += UINT16_SIZEdiff --git a/src/store/bytes_row.cpp b/src/store/bytes_row.cppindex 8f2b0e048..cd65e5c46 100644--- a/src/store/bytes_row.cpp+++ b/src/store/bytes_row.cpp@@ -123,9 +123,13 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {switch (meta.data_type) {case FieldType::STRING: {if (std::holds_alternative<std::string>(val)) {- int len = std::get<std::string>(val).length();- var_infos[i] = {variable_region_offset, len};- variable_region_offset += UINT16_SIZE + len;+ size_t len = std::get<std::string>(val).length();+ if (len > STRING_MAX_UINT16_LENGTH) {+ throw std::invalid_argument("string field '" + meta.name ++ "' exceeds 65535 bytes");+ }+ var_infos[i] = {variable_region_offset, static_cast<int>(len)};+ variable_region_offset += UINT16_SIZE + static_cast<int>(len);} else {var_infos[i] = {variable_region_offset, 0};variable_region_offset += UINT16_SIZE;diff --git a/src/store/bytes_row.h b/src/store/bytes_row.hindex a03832392..881e85573 100644--- a/src/store/bytes_row.h+++ b/src/store/bytes_row.h@@ -7,9 +7,12 @@#include <memory>#include <cstdint>#include <cstring>+#include <stdexcept>namespace vectordb {+constexpr uint32_t STRING_MAX_UINT16_LENGTH = 0xFFFF;+enum class FieldType {INT64 = 0,UINT64 = 1,@@ -117,10 +120,11 @@ class BytesRow {};auto write_string_value = [](const std::string& s, char* dest) {- uint16_t len = static_cast<uint16_t>(s.length());- std::memcpy(dest, &len, 2);- if (len > 0) {- std::memcpy(dest + 2, s.data(), len);+ uint32_t len = static_cast<uint32_t>(s.length());+ uint16_t short_len = static_cast<uint16_t>(len);+ std::memcpy(dest, &short_len, 2);+ if (short_len > 0) {+ std::memcpy(dest + 2, s.data(), short_len);}};@@ -189,6 +193,10 @@ class BytesRow {} else if (const auto* def = get_default_string(meta.default_value)) {len = static_cast<int>(def->length());}+ if (len > static_cast<int>(STRING_MAX_UINT16_LENGTH)) {+ throw std::invalid_argument("string field '" + meta.name ++ "' exceeds 65535 bytes");+ }var_infos[i] = {variable_region_offset, len};variable_region_offset += 2 + len; // UINT16_SIZEbreak;diff --git a/tests/vectordb/test_bytes_row.py b/tests/vectordb/test_bytes_row.pyindex 650f1e438..264623a2c 100644--- a/tests/vectordb/test_bytes_row.py+++ b/tests/vectordb/test_bytes_row.py@@ -108,6 +108,23 @@ class TestBytesRow(unittest.TestCase):val = UnicodeData.bytes_row.deserialize_field(serialized, "text")self.assertEqual(val, text)+ def test_oversized_string_serialization_raises(self):+ @serializable+ @dataclass+ class LargeStringData:+ text: str = ""++ text = "x" * 65536+ with self.assertRaisesRegex(+ (RuntimeError, ValueError), "text.*exceeds 65535 bytes"+ ):+ LargeStringData(text=text).serialize()++ py_schema = _PySchema([{"name": "text", "data_type": _PyFieldType.string, "id": 0}])+ py_row = _PyBytesRow(py_schema)+ with self.assertRaisesRegex(ValueError, "text.*exceeds 65535 bytes"):+ py_row.serialize({"text": text})+def test_binary_data(self):@serializable@dataclassdiff --git a/tests/vectordb/test_filter_ops.py b/tests/vectordb/test_filter_ops.pyindex f5cf6ae19..f7f9ccec4 100644--- a/tests/vectordb/test_filter_ops.py+++ b/tests/vectordb/test_filter_ops.py@@ -13,6 +13,7 @@ db_path_basic = "./test_data/db_test_filters_basic/"db_path_complex = "./test_data/db_test_filters_complex/"db_path_lifecycle = "./test_data/db_test_filters_lifecycle/"db_path_scale = "./test_data/db_test_filters_scale/"+db_path_large_fields = "./test_data/db_test_filters_large_fields/"def clean_dir(path):@@ -114,6 +115,47 @@ class TestFilterOpsBasic(unittest.TestCase):) # chERry, eldERbERry+class TestFilterOpsLargeFields(unittest.TestCase):+ def setUp(self):+ clean_dir(db_path_large_fields)+ self.path = db_path_large_fields+ self.collection = self._create_collection()++ def tearDown(self):+ if self.collection:+ self.collection.drop()+ clean_dir(self.path)++ def _create_collection(self):+ collection_meta = {+ "CollectionName": "test_filters_large_fields",+ "Fields": [+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},+ {"FieldName": "embedding", "FieldType": "vector", "Dim": 4},+ {"FieldName": "uri", "FieldType": "string"},+ {"FieldName": "abstract", "FieldType": "string"},+ ],+ }+ return get_or_create_local_collection(meta_data=collection_meta, path=self.path)++ def test_upsert_record_with_oversized_json_field_raises(self):+ large_abstract = 'prefix "quoted" \\\\ path\n' + ("x" * 66000)+ uri = "viking://user/memories/large.md"+ with self.assertRaisesRegex(+ (RuntimeError, ValueError), "fields.*exceeds 65535 bytes"+ ):+ self.collection.upsert_data(+ [+ {+ "id": 1,+ "embedding": [1.0, 0, 0, 0],+ "uri": uri,+ "abstract": large_abstract,+ }+ ]+ )++class TestFilterOpsComplex(unittest.TestCase):"""Complex mixed logic tests"""