Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(vectordb): 本地字段 JSON 改用 text 并兼容旧数据 (#5091)
Qin Haojiefa822315449c
diff --git a/docs/en/concepts/05-storage.md b/docs/en/concepts/05-storage.mdindex 2f5efd38a..eb372e555 100644--- a/docs/en/concepts/05-storage.md+++ b/docs/en/concepts/05-storage.md@@ -96,6 +96,20 @@ viking://resources/docs/auth/The vector index stores semantic indices, supporting vector search and scalar filtering.+### Local Record Format Compatibility++The local backend packs non-vector fields into a JSON payload. New candidate records and delta+logs store that payload as `text` with a 32-bit byte length, removing the previous 65,535-byte+limit on the combined JSON. The payload is preserved without truncation.++New records carry a format version. Existing unversioned records remain readable, and updates+write the new format; no full database rewrite, index rebuild, or embedding recomputation is+required for this format upgrade. Delta replay also accepts both formats.++This compatibility is one-way: older releases cannot read the new record format. To downgrade,+restore a backup taken before the first write with the new format; changing the executable alone+is insufficient.+### Context Collection Schema| Field | Type | Description |diff --git a/docs/zh/concepts/05-storage.md b/docs/zh/concepts/05-storage.mdindex 1b649f3b5..e61de85f7 100644--- a/docs/zh/concepts/05-storage.md+++ b/docs/zh/concepts/05-storage.md@@ -95,6 +95,17 @@ viking://resources/docs/auth/向量库存储语义索引,支持向量搜索和标量过滤。+### 本地记录格式兼容++本地后端将非向量字段打包成 JSON。新写入的当前记录和 Delta 日志使用带 32 位字节长度的+`text` 保存这份 JSON,解除原先整个 JSON 合计 65,535 字节的限制,不截断内容。++新记录带有格式版本。已有的无版本记录继续可读,更新时写入新格式;本次格式升级不需要+全库重写、重建索引或重新计算 embedding。Delta 恢复也支持新旧格式混存。++兼容方向为新版读取旧数据:旧版程序无法读取新格式。需要降级时,应恢复首次写入新格式+之前的备份,不能只替换回旧版程序。+### Context 集合 Schema| 字段 | 类型 | 说明 |diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.pyindex 73011924e..81840cf0b 100644--- a/openviking/storage/vectordb/store/bytes_row.py+++ b/openviking/storage/vectordb/store/bytes_row.py@@ -13,6 +13,7 @@ UINT32_SIZE = 4 # Used for string/binary length and offsetUINT16_SIZE = 2 # Used for list length and string/binary length inside listsBOOL_SIZE = 1STRING_MAX_UINT16_LENGTH = 0xFFFF+_VERSIONED_ROW_HEADER = b"\x00\x01"@dataclass@@ -24,6 +25,7 @@ class FieldMeta:offset: int # Start offset (calculated from the beginning of row data)id: intdefault_value: Any = None+ legacy_data_type: "_PyFieldType | None" = Noneclass _PyFieldType(Enum):@@ -56,6 +58,7 @@ class _PySchema:self.field_metas: Dict[str, FieldMeta] = {}self.field_orders: List[FieldMeta] = [None] * len(fields) # type: ignorecurrent_offset = 1+ self.has_legacy_fields = False# Type to size and default value mappingTYPE_INFO = {@@ -74,6 +77,11 @@ class _PySchema:for field in fields:name = field["name"]data_type = field["data_type"]+ legacy_type = field.get("legacy_data_type", data_type)+ if legacy_type != data_type:+ if legacy_type != _PyFieldType.string or data_type != _PyFieldType.text:+ raise ValueError("Only STRING to TEXT migration is supported")+ self.has_legacy_fields = Truefield_id = field["id"]if data_type not in TYPE_INFO:@@ -92,6 +100,7 @@ class _PySchema:offset=current_offset,id=field_id,default_value=default_value,+ legacy_data_type=legacy_type,)self.field_orders[field_id] = self.field_metas[name]# Update start offset for the next field@@ -225,46 +234,62 @@ class _PyBytesRow:# Use '<' for little-endianfmt = "<" + "".join(fix_fmt_list) + "".join(var_fmt_list)- buffer = bytearray(1 + struct.calcsize(fmt))- buffer[0] = len(self.field_order) # <= 255- struct.pack_into(fmt, buffer, 1, *(fix_val_list + var_val_list))+ header_size = len(_VERSIONED_ROW_HEADER) if self.schema.has_legacy_fields else 0+ buffer = bytearray(header_size + 1 + struct.calcsize(fmt))+ if header_size:+ buffer[:header_size] = _VERSIONED_ROW_HEADER+ buffer[header_size] = len(self.field_order) # <= 255+ struct.pack_into(fmt, buffer, header_size + 1, *(fix_val_list + var_val_list))return bytes(buffer)def serialize_batch(self, rows_data) -> List[bytes]:return [self.serialize(row_data) for row_data in rows_data]def deserialize_field(self, serialized_data, field_name):+ row_offset = 0+ legacy = self.schema.has_legacy_fields+ if legacy and serialized_data and serialized_data[0] == 0:+ if not serialized_data.startswith(_VERSIONED_ROW_HEADER) or len(serialized_data) <= len(+ _VERSIONED_ROW_HEADER+ ):+ raise ValueError("Invalid or unsupported bytes_row record version")+ row_offset = len(_VERSIONED_ROW_HEADER)+ legacy = False+field_meta = self.schema.get_field_meta(field_name)- if field_meta.id >= serialized_data[0]:+ if field_meta.id >= serialized_data[row_offset]:return field_meta.default_value+ field_offset = row_offset + field_meta.offset+ data_type = field_meta.legacy_data_type if legacy else field_meta.data_type+# Use '<' for little-endian in all unpack operations- if field_meta.data_type == _PyFieldType.int64:- return struct.unpack_from("<q", serialized_data, field_meta.offset)[0]- elif field_meta.data_type == _PyFieldType.uint64:- return struct.unpack_from("<Q", serialized_data, field_meta.offset)[0]- elif field_meta.data_type == _PyFieldType.float32:- return struct.unpack_from("<f", serialized_data, field_meta.offset)[0]- elif field_meta.data_type == _PyFieldType.boolean:+ if data_type == _PyFieldType.int64:+ return struct.unpack_from("<q", serialized_data, field_offset)[0]+ elif data_type == _PyFieldType.uint64:+ return struct.unpack_from("<Q", serialized_data, field_offset)[0]+ elif data_type == _PyFieldType.float32:+ return struct.unpack_from("<f", serialized_data, field_offset)[0]+ elif data_type == _PyFieldType.boolean:# B is 1 byte, endianness doesn't matter, but consistent style- return bool(serialized_data[field_meta.offset])- elif field_meta.data_type == _PyFieldType.string:- str_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]+ return bool(serialized_data[field_offset])+ elif data_type == _PyFieldType.string:+ str_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offsetstr_len = struct.unpack_from("<H", serialized_data, str_offset)[0]str_offset += UINT16_SIZEreturn serialized_data[str_offset : str_offset + str_len].decode("utf-8")- elif field_meta.data_type == _PyFieldType.binary:- binary_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]+ elif data_type == _PyFieldType.binary:+ binary_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offsetbinary_len = struct.unpack_from("<I", serialized_data, binary_offset)[0]binary_offset += UINT32_SIZEreturn serialized_data[binary_offset : binary_offset + binary_len]- elif field_meta.data_type == _PyFieldType.text:- text_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]+ elif data_type == _PyFieldType.text:+ text_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offsettext_len = struct.unpack_from("<I", serialized_data, text_offset)[0]text_offset += UINT32_SIZEreturn serialized_data[text_offset : text_offset + text_len].decode("utf-8")- elif field_meta.data_type == _PyFieldType.list_string:- list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]+ elif data_type == _PyFieldType.list_string:+ list_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offsetlist_len = struct.unpack_from("<H", serialized_data, list_offset)[0]list_offset += UINT16_SIZEstr_list = [None] * list_len@@ -274,14 +299,14 @@ class _PyBytesRow:str_list[i] = serialized_data[list_offset : list_offset + str_len].decode("utf-8")list_offset += str_lenreturn str_list- elif field_meta.data_type == _PyFieldType.list_int64:- list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]+ elif data_type == _PyFieldType.list_int64:+ list_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offsetlist_len = struct.unpack_from("<H", serialized_data, list_offset)[0]list_offset += UINT16_SIZEreturn list(struct.unpack_from(f"<{list_len}q", serialized_data, list_offset))- elif field_meta.data_type == _PyFieldType.list_float32:- list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]+ elif data_type == _PyFieldType.list_float32:+ list_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offsetlist_len = struct.unpack_from("<H", serialized_data, list_offset)[0]list_offset += UINT16_SIZEreturn list(struct.unpack_from(f"<{list_len}f", serialized_data, list_offset))diff --git a/openviking/storage/vectordb/store/data.py b/openviking/storage/vectordb/store/data.pyindex cad0574bc..5e0a9340e 100644--- a/openviking/storage/vectordb/store/data.py+++ b/openviking/storage/vectordb/store/data.py@@ -4,6 +4,7 @@ import jsonfrom dataclasses import dataclass, fieldfrom typing import List+from openviking.storage.vectordb.store.bytes_row import FieldTypefrom openviking.storage.vectordb.store.serializable import serializable@@ -14,7 +15,9 @@ class CandidateData:vector: List[float] = field(default_factory=list)sparse_raw_terms: List[str] = field(default_factory=list)sparse_values: List[float] = field(default_factory=list)- fields: str = ""+ fields: str = field(+ default="", metadata={"field_type": FieldType.text, "legacy_field_type": FieldType.string}+ )expire_ns_ts: int = 0def __str__(self):@@ -44,8 +47,12 @@ class DeltaRecord:vector: List[float] = field(default_factory=list)sparse_raw_terms: List[str] = field(default_factory=list)sparse_values: List[float] = field(default_factory=list)- fields: str = ""- old_fields: str = ""+ fields: str = field(+ default="", metadata={"field_type": FieldType.text, "legacy_field_type": FieldType.string}+ )+ old_fields: str = field(+ default="", metadata={"field_type": FieldType.text, "legacy_field_type": FieldType.string}+ )def __str__(self):data_dict = {diff --git a/openviking/storage/vectordb/store/serializable.py b/openviking/storage/vectordb/store/serializable.pyindex d8bc5ee1d..2c2d5f50f 100644--- a/openviking/storage/vectordb/store/serializable.py+++ b/openviking/storage/vectordb/store/serializable.py@@ -60,6 +60,8 @@ def serializable(cls):Optional field metadata:- field_type: Override the auto-inferred type with FieldType enum (e.g., FieldType.int64 vs FieldType.uint64)- default_value: Override the default value+ - legacy_field_type: Read unversioned STRING records after upgrading a+ field to TEXT. BytesRow writes and reads the versioned format directly.Example:@serializable@@ -100,6 +102,9 @@ def serializable(cls):if f.metadata and "default_value" in f.metadata:field_def["default_value"] = f.metadata["default_value"]+ if "legacy_field_type" in f.metadata:+ field_def["legacy_data_type"] = f.metadata["legacy_field_type"]+field_list.append(field_def)# Create schema and bytes_rowdiff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cppindex 0b366b722..8b25bdd50 100644--- a/src/abi3_engine_backend.cpp+++ b/src/abi3_engine_backend.cpp@@ -602,6 +602,16 @@ bool parse_schema_fields(PyObject* fields_obj,return false;}+ PyObject* legacy_type_obj = PyDict_GetItemString(item, "legacy_data_type");+ if (legacy_type_obj != nullptr) {+ vdb::FieldType legacy_type;+ if (!py_to_field_type(legacy_type_obj, &legacy_type)) {+ Py_DECREF(item);+ return false;+ }+ field.legacy_data_type = legacy_type;+ }+const long field_id = PyLong_AsLong(id_obj);if (PyErr_Occurred() != nullptr) {Py_DECREF(item);@@ -1053,10 +1063,13 @@ PyObject* py_bytes_row_deserialize(PyObject*, PyObject* args) {return nullptr;}- std::string payload;- if (!py_to_string(payload_obj, &payload, true)) {+ char* payload_data = nullptr;+ Py_ssize_t payload_size = 0;+ if (PyBytes_AsStringAndSize(payload_obj, &payload_data, &payload_size) < 0) {return nullptr;}+ // The call arguments keep these immutable bytes alive while the GIL is released.+ const std::string_view payload(payload_data, static_cast<size_t>(payload_size));PyObject* result = PyDict_New();if (result == nullptr) {@@ -1106,10 +1119,12 @@ PyObject* py_bytes_row_deserialize_field(PyObject*, PyObject* args) {return nullptr;}- std::string payload;- if (!py_to_string(payload_obj, &payload, true)) {+ char* payload_data = nullptr;+ Py_ssize_t payload_size = 0;+ if (PyBytes_AsStringAndSize(payload_obj, &payload_data, &payload_size) < 0) {return nullptr;}+ const std::string_view payload(payload_data, static_cast<size_t>(payload_size));const auto* meta = handle->schema->get_field_meta(field_name);if (meta == nullptr) {diff --git a/src/store/bytes_row.cpp b/src/store/bytes_row.cppindex 7f6014e41..b8a57ac33 100644--- a/src/store/bytes_row.cpp+++ b/src/store/bytes_row.cpp@@ -77,6 +77,15 @@ Schema::Schema(const std::vector<FieldDef>& fields) {meta.offset = current_offset;meta.id = field.id;meta.default_value = field.default_value;+ meta.legacy_data_type = field.legacy_data_type.value_or(field.data_type);+ if (meta.legacy_data_type != meta.data_type) {+ // STRING -> TEXT preserves the fixed-region offset layout.+ if (meta.legacy_data_type != FieldType::STRING ||+ meta.data_type != FieldType::TEXT) {+ throw std::invalid_argument("Only STRING to TEXT migration is supported");+ }+ has_legacy_fields_ = true;+ }field_metas_[field.name] = meta;field_orders_[field.id] = meta;@@ -205,9 +214,14 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {}// Allocate buffer- std::string buffer;- buffer.resize(variable_region_offset);- char* ptr = &buffer[0];+ // Include the version in the original allocation; offsets remain relative+ // to the row body so legacy and current schemas share the fixed layout.+ const size_t header_size =+ schema_->has_legacy_fields() ? VERSIONED_ROW_HEADER.size() : 0;+ std::string buffer(variable_region_offset + header_size, '\0');+ if (header_size != 0)+ std::memcpy(buffer.data(), VERSIONED_ROW_HEADER.data(), header_size);+ char* ptr = buffer.data() + header_size;// Write header (field count)// Be careful with alignment if we were doing raw casting, but we use memcpy@@ -333,8 +347,17 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {return buffer;}-Value BytesRow::deserialize_field(const std::string& serialized_data,+Value BytesRow::deserialize_field(std::string_view serialized_data,const std::string& field_name) const {+ bool legacy = schema_->has_legacy_fields();+ if (legacy && !serialized_data.empty() && serialized_data[0] == '\0') {+ if (serialized_data.size() <= VERSIONED_ROW_HEADER.size() ||+ serialized_data.substr(0, VERSIONED_ROW_HEADER.size()) != VERSIONED_ROW_HEADER) {+ throw std::invalid_argument("Invalid or unsupported bytes_row record version");+ }+ serialized_data.remove_prefix(VERSIONED_ROW_HEADER.size());+ legacy = false;+ }const FieldMeta* meta_ptr = schema_->get_field_meta(field_name);if (!meta_ptr)return std::monostate{};@@ -354,7 +377,7 @@ Value BytesRow::deserialize_field(const std::string& serialized_data,const char* field_ptr = ptr + meta.offset;- switch (meta.data_type) {+ switch (legacy ? meta.legacy_data_type : meta.data_type) {case FieldType::INT64: {int64_t v;std::memcpy(&v, field_ptr, sizeof(v));@@ -526,7 +549,7 @@ Value BytesRow::deserialize_field(const std::string& serialized_data,}std::map<std::string, Value> BytesRow::deserialize(- const std::string& serialized_data) const {+ std::string_view serialized_data) const {std::map<std::string, Value> result;const auto& order = schema_->get_field_order();for (const auto& meta : order) {diff --git a/src/store/bytes_row.h b/src/store/bytes_row.hindex 575870da0..319ffec9a 100644--- a/src/store/bytes_row.h+++ b/src/store/bytes_row.h@@ -1,6 +1,8 @@#pragma once#include <string>+#include <string_view>+#include <optional>#include <vector>#include <map>#include <variant>@@ -12,6 +14,8 @@namespace vectordb {constexpr uint32_t STRING_MAX_UINT16_LENGTH = 0xFFFF;+// A zero field count distinguishes versioned rows from legacy records.+constexpr std::string_view VERSIONED_ROW_HEADER{"\0\1", 2};enum class FieldType {INT64 = 0,@@ -37,6 +41,7 @@ struct FieldMeta {int offset;int id;Value default_value;+ FieldType legacy_data_type;};struct FieldDef {@@ -44,6 +49,7 @@ struct FieldDef {FieldType data_type;int id;Value default_value;+ std::optional<FieldType> legacy_data_type;};class Schema {@@ -56,12 +62,16 @@ class Schema {int get_total_byte_length() const {return total_byte_length_;}+ bool has_legacy_fields() const {+ return has_legacy_fields_;+ }const FieldMeta* get_field_meta(const std::string& name) const;private:std::vector<FieldMeta> field_orders_;std::map<std::string, FieldMeta> field_metas_;int total_byte_length_;+ bool has_legacy_fields_ = false;};class BytesRow {@@ -272,9 +282,12 @@ class BytesRow {}}- std::string buffer;- buffer.resize(variable_region_offset);- char* ptr = &buffer[0];+ const size_t header_size =+ schema_->has_legacy_fields() ? VERSIONED_ROW_HEADER.size() : 0;+ std::string buffer(variable_region_offset + header_size, '\0');+ if (header_size != 0)+ std::memcpy(buffer.data(), VERSIONED_ROW_HEADER.data(), header_size);+ char* ptr = buffer.data() + header_size;// Headerptr[0] = static_cast<uint8_t>(field_order.size());@@ -406,10 +419,10 @@ class BytesRow {// Deserialize to a mapstd::map<std::string, Value> deserialize(- const std::string& serialized_data) const;+ std::string_view serialized_data) const;// Deserialize a single field- Value deserialize_field(const std::string& serialized_data,+ Value deserialize_field(std::string_view serialized_data,const std::string& field_name) const;// Get schemadiff --git a/tests/vectordb/test_bytes_row.py b/tests/vectordb/test_bytes_row.pyindex 8b197e2f1..25550c368 100644--- a/tests/vectordb/test_bytes_row.py+++ b/tests/vectordb/test_bytes_row.py@@ -363,11 +363,27 @@ class TestTextFieldType(unittest.TestCase):self.assertEqual(self.py_row.deserialize_field(cpp_bytes, "body"), data["body"])def test_binary_consistency(self):+ legacy_data = {"label": 42, "body": "升级前的记录"}+ readers = []+ for fields, field_type, schema, row in (+ (self.cpp_fields, engine.FieldType, engine.Schema, engine.BytesRow),+ (self.py_fields, _PyFieldType, _PySchema, _PyBytesRow),+ ):+ legacy_schema = schema([fields[0], {**fields[1], "data_type": field_type.string}])+ legacy_bytes = row(legacy_schema).serialize(legacy_data)+ reader = row(schema([fields[0], {**fields[1], "legacy_data_type": field_type.string}]))+ self.assertEqual(reader.deserialize(legacy_bytes), legacy_data)+ readers.append(reader)+data = self._make_data()- py_bytes = self.py_row.serialize(data)- cpp_bytes = self.cpp_row.serialize(data)- self.assertEqual(len(py_bytes), len(cpp_bytes), "Binary length mismatch")- self.assertEqual(py_bytes, cpp_bytes, "Binary content mismatch")+ cpp_bytes, py_bytes = [reader.serialize(data) for reader in readers]+ self.assertEqual(cpp_bytes, py_bytes, "Binary content mismatch")+ for reader in readers:+ self.assertEqual(reader.deserialize(cpp_bytes), data)+ self.assertEqual(reader.deserialize_field(cpp_bytes, "label"), 42)+ for invalid in (b"\x00\x01", b"\x00\x02\x02"):+ with self.assertRaisesRegex((ValueError, RuntimeError), "record version"):+ reader.deserialize(invalid)def test_text_declared_via_metadata(self):@serializablediff --git a/tests/vectordb/test_crash_recovery.py b/tests/vectordb/test_crash_recovery.pyindex d05a2dd7e..463821e7c 100644--- a/tests/vectordb/test_crash_recovery.py+++ b/tests/vectordb/test_crash_recovery.py@@ -6,11 +6,16 @@ import shutilimport sysimport timeimport unittest+from contextlib import ExitStack+from unittest.mock import patch+from openviking.storage.vectordb import enginefrom openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection+from openviking.storage.vectordb.store.data import CandidateData, DeltaRecordDB_PATH_CRASH = "./test_data/test_db_crash_recovery"DB_PATH_ROBUST = "./test_data/test_db_robust_crash"+LARGE_TEXT = "旧数据升级后的长文本" * 4000def worker_write_and_crash(path, start_id, count, event_ready):@@ -52,8 +57,42 @@ def worker_write_and_crash(path, start_id, count, event_ready):})- print(f"[Subprocess] Upserting {count} items...")- col.upsert_data(data)+ # Seed real unversioned candidate and delta rows as an older writer did.+ # Keep one legacy row untouched; update the other after switching writers.+ legacy_fields = [+ ("label", "uint64"),+ ("vector", "list_float32"),+ ("sparse_raw_terms", "list_string"),+ ("sparse_values", "list_float32"),+ ("fields", "string"),+ ]+ with ExitStack() as legacy_writer:+ for record_type, definitions in (+ (CandidateData, [*legacy_fields, ("expire_ns_ts", "uint64")]),+ (DeltaRecord, [("type", "uint64"), *legacy_fields, ("old_fields", "string")]),+ ):+ legacy_schema = engine.Schema(+ [+ {+ "name": name,+ "id": index,+ "data_type": getattr(engine.FieldType, field_type),+ }+ for index, (name, field_type) in enumerate(definitions)+ ]+ )+ legacy_writer.enter_context(+ patch.object(record_type, "bytes_row", engine.BytesRow(legacy_schema))+ )+ col.upsert_data(data[:2])++ print(f"[Subprocess] Upserting {count} mixed-format items...")+ col.upsert_data(data[2:])+ col.update_data([{"id": start_id, "data": LARGE_TEXT}])+ col.update_data([{"id": start_id, "data": LARGE_TEXT + " updated"}])+ # Leave a large deletion in the log too, without changing the final count.+ col.upsert_data([{"id": start_id + count, "vector": [0.1] * 4, "data": LARGE_TEXT}])+ col.delete_data([start_id + count])print("[Subprocess] Upsert done. Not closing.")# Notify main process that write is done@@ -208,6 +247,10 @@ class TestCrashRecovery(unittest.TestCase):self.assertEqual(len(res.items), data_count, f"Should find all {data_count} items in KV Store")+ payloads = {item.id: item.fields["data"] for item in res.items}+ self.assertEqual(payloads[0], LARGE_TEXT + " updated")+ self.assertEqual(payloads[1], "crash_data_1")+ self.assertEqual(col.fetch_data([data_count]).ids_not_exist, [data_count])print("[Main] Data fetch verified.")@@ -226,6 +269,7 @@ class TestCrashRecovery(unittest.TestCase):self.assertEqual(len(found_ids), data_count, "Index should contain all items after recovery")+ self.assertEqual(set(found_ids), set(all_ids))col.close()print("[Main] Simple recovery test passed.")diff --git a/tests/vectordb/test_filter_ops.py b/tests/vectordb/test_filter_ops.pyindex 608d64520..f4aca36ab 100644--- a/tests/vectordb/test_filter_ops.py+++ b/tests/vectordb/test_filter_ops.py@@ -14,7 +14,6 @@ db_path_basic = "./test_data/db_test_filters_basic/"db_path_complex = "./test_data/db_test_filters_complex/"db_path_lifecycle = "./test_data/db_test_filters_lifecycle/"db_path_scale = "./test_data/db_test_filters_scale/"-db_path_large_fields = "./test_data/db_test_filters_large_fields/"def clean_dir(path):@@ -116,45 +115,6 @@ class TestFilterOpsBasic(unittest.TestCase):) # chERry, eldERbERry-class TestFilterOpsLargeFields(unittest.TestCase):- def setUp(self):- clean_dir(db_path_large_fields)- self.path = db_path_large_fields- self.collection = self._create_collection()-- def tearDown(self):- if self.collection:- self.collection.drop()- clean_dir(self.path)-- def _create_collection(self):- collection_meta = {- "CollectionName": "test_filters_large_fields",- "Fields": [- {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},- {"FieldName": "embedding", "FieldType": "vector", "Dim": 4},- {"FieldName": "uri", "FieldType": "string"},- {"FieldName": "abstract", "FieldType": "string"},- ],- }- return get_or_create_local_collection(meta_data=collection_meta, path=self.path)-- def test_upsert_record_with_oversized_json_field_raises(self):- large_abstract = 'prefix "quoted" \\\\ path\n' + ("x" * 66000)- uri = "viking://user/memories/large.md"- with self.assertRaisesRegex((RuntimeError, ValueError), "fields.*exceeds 65535 bytes"):- self.collection.upsert_data(- [- {- "id": 1,- "embedding": [1.0, 0, 0, 0],- "uri": uri,- "abstract": large_abstract,- }- ]- )--class TestFilterOpsComplex(unittest.TestCase):"""Complex mixed logic tests"""