Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(vectordb): reject oversized bytes row strings (#2171)

Qin Haojief1afd40a6661
diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.py
index 58ae4c18d..fc14f4e23 100644
--- a/openviking/storage/vectordb/store/bytes_row.py
+++ b/openviking/storage/vectordb/store/bytes_row.py
@@ -12,6 +12,7 @@ FLOAT32_SIZE = 4
UINT32_SIZE = 4 # Used for string/binary length and offset
UINT16_SIZE = 2 # Used for list length and string/binary length inside lists
BOOL_SIZE = 1
+STRING_MAX_UINT16_LENGTH = 0xFFFF
@dataclass
@@ -146,6 +147,10 @@ class _PyBytesRow:
fix_region_offset += UINT32_SIZE
bytes_item = value.encode("utf-8")
bytes_item_len = len(bytes_item)
+ if bytes_item_len > STRING_MAX_UINT16_LENGTH:
+ raise ValueError(
+ f"string field '{field_meta.name}' exceeds 65535 bytes"
+ )
var_fmt_list.append("H")
var_val_list.append(bytes_item_len)
variable_region_offset += UINT16_SIZE
diff --git a/src/store/bytes_row.cpp b/src/store/bytes_row.cpp
index 8f2b0e048..cd65e5c46 100644
--- a/src/store/bytes_row.cpp
+++ b/src/store/bytes_row.cpp
@@ -123,9 +123,13 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {
switch (meta.data_type) {
case FieldType::STRING: {
if (std::holds_alternative<std::string>(val)) {
- int len = std::get<std::string>(val).length();
- var_infos[i] = {variable_region_offset, len};
- variable_region_offset += UINT16_SIZE + len;
+ size_t len = std::get<std::string>(val).length();
+ if (len > STRING_MAX_UINT16_LENGTH) {
+ throw std::invalid_argument("string field '" + meta.name +
+ "' exceeds 65535 bytes");
+ }
+ var_infos[i] = {variable_region_offset, static_cast<int>(len)};
+ variable_region_offset += UINT16_SIZE + static_cast<int>(len);
} else {
var_infos[i] = {variable_region_offset, 0};
variable_region_offset += UINT16_SIZE;
diff --git a/src/store/bytes_row.h b/src/store/bytes_row.h
index a03832392..881e85573 100644
--- a/src/store/bytes_row.h
+++ b/src/store/bytes_row.h
@@ -7,9 +7,12 @@
#include <memory>
#include <cstdint>
#include <cstring>
+#include <stdexcept>
namespace vectordb {
+constexpr uint32_t STRING_MAX_UINT16_LENGTH = 0xFFFF;
+
enum class FieldType {
INT64 = 0,
UINT64 = 1,
@@ -117,10 +120,11 @@ class BytesRow {
};
auto write_string_value = [](const std::string& s, char* dest) {
- uint16_t len = static_cast<uint16_t>(s.length());
- std::memcpy(dest, &len, 2);
- if (len > 0) {
- std::memcpy(dest + 2, s.data(), len);
+ uint32_t len = static_cast<uint32_t>(s.length());
+ uint16_t short_len = static_cast<uint16_t>(len);
+ std::memcpy(dest, &short_len, 2);
+ if (short_len > 0) {
+ std::memcpy(dest + 2, s.data(), short_len);
}
};
@@ -189,6 +193,10 @@ class BytesRow {
} else if (const auto* def = get_default_string(meta.default_value)) {
len = static_cast<int>(def->length());
}
+ if (len > static_cast<int>(STRING_MAX_UINT16_LENGTH)) {
+ throw std::invalid_argument("string field '" + meta.name +
+ "' exceeds 65535 bytes");
+ }
var_infos[i] = {variable_region_offset, len};
variable_region_offset += 2 + len; // UINT16_SIZE
break;
diff --git a/tests/vectordb/test_bytes_row.py b/tests/vectordb/test_bytes_row.py
index 650f1e438..264623a2c 100644
--- a/tests/vectordb/test_bytes_row.py
+++ b/tests/vectordb/test_bytes_row.py
@@ -108,6 +108,23 @@ class TestBytesRow(unittest.TestCase):
val = UnicodeData.bytes_row.deserialize_field(serialized, "text")
self.assertEqual(val, text)
+ def test_oversized_string_serialization_raises(self):
+ @serializable
+ @dataclass
+ class LargeStringData:
+ text: str = ""
+
+ text = "x" * 65536
+ with self.assertRaisesRegex(
+ (RuntimeError, ValueError), "text.*exceeds 65535 bytes"
+ ):
+ LargeStringData(text=text).serialize()
+
+ py_schema = _PySchema([{"name": "text", "data_type": _PyFieldType.string, "id": 0}])
+ py_row = _PyBytesRow(py_schema)
+ with self.assertRaisesRegex(ValueError, "text.*exceeds 65535 bytes"):
+ py_row.serialize({"text": text})
+
def test_binary_data(self):
@serializable
@dataclass
diff --git a/tests/vectordb/test_filter_ops.py b/tests/vectordb/test_filter_ops.py
index f5cf6ae19..f7f9ccec4 100644
--- a/tests/vectordb/test_filter_ops.py
+++ b/tests/vectordb/test_filter_ops.py
@@ -13,6 +13,7 @@ db_path_basic = "./test_data/db_test_filters_basic/"
db_path_complex = "./test_data/db_test_filters_complex/"
db_path_lifecycle = "./test_data/db_test_filters_lifecycle/"
db_path_scale = "./test_data/db_test_filters_scale/"
+db_path_large_fields = "./test_data/db_test_filters_large_fields/"
def clean_dir(path):
@@ -114,6 +115,47 @@ class TestFilterOpsBasic(unittest.TestCase):
) # chERry, eldERbERry
+class TestFilterOpsLargeFields(unittest.TestCase):
+ def setUp(self):
+ clean_dir(db_path_large_fields)
+ self.path = db_path_large_fields
+ self.collection = self._create_collection()
+
+ def tearDown(self):
+ if self.collection:
+ self.collection.drop()
+ clean_dir(self.path)
+
+ def _create_collection(self):
+ collection_meta = {
+ "CollectionName": "test_filters_large_fields",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
+ {"FieldName": "embedding", "FieldType": "vector", "Dim": 4},
+ {"FieldName": "uri", "FieldType": "string"},
+ {"FieldName": "abstract", "FieldType": "string"},
+ ],
+ }
+ return get_or_create_local_collection(meta_data=collection_meta, path=self.path)
+
+ def test_upsert_record_with_oversized_json_field_raises(self):
+ large_abstract = 'prefix "quoted" \\\\ path\n' + ("x" * 66000)
+ uri = "viking://user/memories/large.md"
+ with self.assertRaisesRegex(
+ (RuntimeError, ValueError), "fields.*exceeds 65535 bytes"
+ ):
+ self.collection.upsert_data(
+ [
+ {
+ "id": 1,
+ "embedding": [1.0, 0, 0, 0],
+ "uri": uri,
+ "abstract": large_abstract,
+ }
+ ]
+ )
+
+
class TestFilterOpsComplex(unittest.TestCase):
"""Complex mixed logic tests"""