Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(vectordb): 本地字段 JSON 改用 text 并兼容旧数据 (#5091)

Qin Haojiefa822315449c
diff --git a/docs/en/concepts/05-storage.md b/docs/en/concepts/05-storage.md
index 2f5efd38a..eb372e555 100644
--- a/docs/en/concepts/05-storage.md
+++ b/docs/en/concepts/05-storage.md
@@ -96,6 +96,20 @@ viking://resources/docs/auth/
The vector index stores semantic indices, supporting vector search and scalar filtering.
+### Local Record Format Compatibility
+
+The local backend packs non-vector fields into a JSON payload. New candidate records and delta
+logs store that payload as `text` with a 32-bit byte length, removing the previous 65,535-byte
+limit on the combined JSON. The payload is preserved without truncation.
+
+New records carry a format version. Existing unversioned records remain readable, and updates
+write the new format; no full database rewrite, index rebuild, or embedding recomputation is
+required for this format upgrade. Delta replay also accepts both formats.
+
+This compatibility is one-way: older releases cannot read the new record format. To downgrade,
+restore a backup taken before the first write with the new format; changing the executable alone
+is insufficient.
+
### Context Collection Schema
| Field | Type | Description |
diff --git a/docs/zh/concepts/05-storage.md b/docs/zh/concepts/05-storage.md
index 1b649f3b5..e61de85f7 100644
--- a/docs/zh/concepts/05-storage.md
+++ b/docs/zh/concepts/05-storage.md
@@ -95,6 +95,17 @@ viking://resources/docs/auth/
向量库存储语义索引,支持向量搜索和标量过滤。
+### 本地记录格式兼容
+
+本地后端将非向量字段打包成 JSON。新写入的当前记录和 Delta 日志使用带 32 位字节长度的
+`text` 保存这份 JSON,解除原先整个 JSON 合计 65,535 字节的限制,不截断内容。
+
+新记录带有格式版本。已有的无版本记录继续可读,更新时写入新格式;本次格式升级不需要
+全库重写、重建索引或重新计算 embedding。Delta 恢复也支持新旧格式混存。
+
+兼容方向为新版读取旧数据:旧版程序无法读取新格式。需要降级时,应恢复首次写入新格式
+之前的备份,不能只替换回旧版程序。
+
### Context 集合 Schema
| 字段 | 类型 | 说明 |
diff --git a/openviking/storage/vectordb/store/bytes_row.py b/openviking/storage/vectordb/store/bytes_row.py
index 73011924e..81840cf0b 100644
--- a/openviking/storage/vectordb/store/bytes_row.py
+++ b/openviking/storage/vectordb/store/bytes_row.py
@@ -13,6 +13,7 @@ UINT32_SIZE = 4 # Used for string/binary length and offset
UINT16_SIZE = 2 # Used for list length and string/binary length inside lists
BOOL_SIZE = 1
STRING_MAX_UINT16_LENGTH = 0xFFFF
+_VERSIONED_ROW_HEADER = b"\x00\x01"
@dataclass
@@ -24,6 +25,7 @@ class FieldMeta:
offset: int # Start offset (calculated from the beginning of row data)
id: int
default_value: Any = None
+ legacy_data_type: "_PyFieldType | None" = None
class _PyFieldType(Enum):
@@ -56,6 +58,7 @@ class _PySchema:
self.field_metas: Dict[str, FieldMeta] = {}
self.field_orders: List[FieldMeta] = [None] * len(fields) # type: ignore
current_offset = 1
+ self.has_legacy_fields = False
# Type to size and default value mapping
TYPE_INFO = {
@@ -74,6 +77,11 @@ class _PySchema:
for field in fields:
name = field["name"]
data_type = field["data_type"]
+ legacy_type = field.get("legacy_data_type", data_type)
+ if legacy_type != data_type:
+ if legacy_type != _PyFieldType.string or data_type != _PyFieldType.text:
+ raise ValueError("Only STRING to TEXT migration is supported")
+ self.has_legacy_fields = True
field_id = field["id"]
if data_type not in TYPE_INFO:
@@ -92,6 +100,7 @@ class _PySchema:
offset=current_offset,
id=field_id,
default_value=default_value,
+ legacy_data_type=legacy_type,
)
self.field_orders[field_id] = self.field_metas[name]
# Update start offset for the next field
@@ -225,46 +234,62 @@ class _PyBytesRow:
# Use '<' for little-endian
fmt = "<" + "".join(fix_fmt_list) + "".join(var_fmt_list)
- buffer = bytearray(1 + struct.calcsize(fmt))
- buffer[0] = len(self.field_order) # <= 255
- struct.pack_into(fmt, buffer, 1, *(fix_val_list + var_val_list))
+ header_size = len(_VERSIONED_ROW_HEADER) if self.schema.has_legacy_fields else 0
+ buffer = bytearray(header_size + 1 + struct.calcsize(fmt))
+ if header_size:
+ buffer[:header_size] = _VERSIONED_ROW_HEADER
+ buffer[header_size] = len(self.field_order) # <= 255
+ struct.pack_into(fmt, buffer, header_size + 1, *(fix_val_list + var_val_list))
return bytes(buffer)
def serialize_batch(self, rows_data) -> List[bytes]:
return [self.serialize(row_data) for row_data in rows_data]
def deserialize_field(self, serialized_data, field_name):
+ row_offset = 0
+ legacy = self.schema.has_legacy_fields
+ if legacy and serialized_data and serialized_data[0] == 0:
+ if not serialized_data.startswith(_VERSIONED_ROW_HEADER) or len(serialized_data) <= len(
+ _VERSIONED_ROW_HEADER
+ ):
+ raise ValueError("Invalid or unsupported bytes_row record version")
+ row_offset = len(_VERSIONED_ROW_HEADER)
+ legacy = False
+
field_meta = self.schema.get_field_meta(field_name)
- if field_meta.id >= serialized_data[0]:
+ if field_meta.id >= serialized_data[row_offset]:
return field_meta.default_value
+ field_offset = row_offset + field_meta.offset
+ data_type = field_meta.legacy_data_type if legacy else field_meta.data_type
+
# Use '<' for little-endian in all unpack operations
- if field_meta.data_type == _PyFieldType.int64:
- return struct.unpack_from("<q", serialized_data, field_meta.offset)[0]
- elif field_meta.data_type == _PyFieldType.uint64:
- return struct.unpack_from("<Q", serialized_data, field_meta.offset)[0]
- elif field_meta.data_type == _PyFieldType.float32:
- return struct.unpack_from("<f", serialized_data, field_meta.offset)[0]
- elif field_meta.data_type == _PyFieldType.boolean:
+ if data_type == _PyFieldType.int64:
+ return struct.unpack_from("<q", serialized_data, field_offset)[0]
+ elif data_type == _PyFieldType.uint64:
+ return struct.unpack_from("<Q", serialized_data, field_offset)[0]
+ elif data_type == _PyFieldType.float32:
+ return struct.unpack_from("<f", serialized_data, field_offset)[0]
+ elif data_type == _PyFieldType.boolean:
# B is 1 byte, endianness doesn't matter, but consistent style
- return bool(serialized_data[field_meta.offset])
- elif field_meta.data_type == _PyFieldType.string:
- str_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
+ return bool(serialized_data[field_offset])
+ elif data_type == _PyFieldType.string:
+ str_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offset
str_len = struct.unpack_from("<H", serialized_data, str_offset)[0]
str_offset += UINT16_SIZE
return serialized_data[str_offset : str_offset + str_len].decode("utf-8")
- elif field_meta.data_type == _PyFieldType.binary:
- binary_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
+ elif data_type == _PyFieldType.binary:
+ binary_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offset
binary_len = struct.unpack_from("<I", serialized_data, binary_offset)[0]
binary_offset += UINT32_SIZE
return serialized_data[binary_offset : binary_offset + binary_len]
- elif field_meta.data_type == _PyFieldType.text:
- text_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
+ elif data_type == _PyFieldType.text:
+ text_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offset
text_len = struct.unpack_from("<I", serialized_data, text_offset)[0]
text_offset += UINT32_SIZE
return serialized_data[text_offset : text_offset + text_len].decode("utf-8")
- elif field_meta.data_type == _PyFieldType.list_string:
- list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
+ elif data_type == _PyFieldType.list_string:
+ list_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offset
list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]
list_offset += UINT16_SIZE
str_list = [None] * list_len
@@ -274,14 +299,14 @@ class _PyBytesRow:
str_list[i] = serialized_data[list_offset : list_offset + str_len].decode("utf-8")
list_offset += str_len
return str_list
- elif field_meta.data_type == _PyFieldType.list_int64:
- list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
+ elif data_type == _PyFieldType.list_int64:
+ list_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offset
list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]
list_offset += UINT16_SIZE
return list(struct.unpack_from(f"<{list_len}q", serialized_data, list_offset))
- elif field_meta.data_type == _PyFieldType.list_float32:
- list_offset = struct.unpack_from("<I", serialized_data, field_meta.offset)[0]
+ elif data_type == _PyFieldType.list_float32:
+ list_offset = struct.unpack_from("<I", serialized_data, field_offset)[0] + row_offset
list_len = struct.unpack_from("<H", serialized_data, list_offset)[0]
list_offset += UINT16_SIZE
return list(struct.unpack_from(f"<{list_len}f", serialized_data, list_offset))
diff --git a/openviking/storage/vectordb/store/data.py b/openviking/storage/vectordb/store/data.py
index cad0574bc..5e0a9340e 100644
--- a/openviking/storage/vectordb/store/data.py
+++ b/openviking/storage/vectordb/store/data.py
@@ -4,6 +4,7 @@ import json
from dataclasses import dataclass, field
from typing import List
+from openviking.storage.vectordb.store.bytes_row import FieldType
from openviking.storage.vectordb.store.serializable import serializable
@@ -14,7 +15,9 @@ class CandidateData:
vector: List[float] = field(default_factory=list)
sparse_raw_terms: List[str] = field(default_factory=list)
sparse_values: List[float] = field(default_factory=list)
- fields: str = ""
+ fields: str = field(
+ default="", metadata={"field_type": FieldType.text, "legacy_field_type": FieldType.string}
+ )
expire_ns_ts: int = 0
def __str__(self):
@@ -44,8 +47,12 @@ class DeltaRecord:
vector: List[float] = field(default_factory=list)
sparse_raw_terms: List[str] = field(default_factory=list)
sparse_values: List[float] = field(default_factory=list)
- fields: str = ""
- old_fields: str = ""
+ fields: str = field(
+ default="", metadata={"field_type": FieldType.text, "legacy_field_type": FieldType.string}
+ )
+ old_fields: str = field(
+ default="", metadata={"field_type": FieldType.text, "legacy_field_type": FieldType.string}
+ )
def __str__(self):
data_dict = {
diff --git a/openviking/storage/vectordb/store/serializable.py b/openviking/storage/vectordb/store/serializable.py
index d8bc5ee1d..2c2d5f50f 100644
--- a/openviking/storage/vectordb/store/serializable.py
+++ b/openviking/storage/vectordb/store/serializable.py
@@ -60,6 +60,8 @@ def serializable(cls):
Optional field metadata:
- field_type: Override the auto-inferred type with FieldType enum (e.g., FieldType.int64 vs FieldType.uint64)
- default_value: Override the default value
+ - legacy_field_type: Read unversioned STRING records after upgrading a
+ field to TEXT. BytesRow writes and reads the versioned format directly.
Example:
@serializable
@@ -100,6 +102,9 @@ def serializable(cls):
if f.metadata and "default_value" in f.metadata:
field_def["default_value"] = f.metadata["default_value"]
+ if "legacy_field_type" in f.metadata:
+ field_def["legacy_data_type"] = f.metadata["legacy_field_type"]
+
field_list.append(field_def)
# Create schema and bytes_row
diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cpp
index 0b366b722..8b25bdd50 100644
--- a/src/abi3_engine_backend.cpp
+++ b/src/abi3_engine_backend.cpp
@@ -602,6 +602,16 @@ bool parse_schema_fields(PyObject* fields_obj,
return false;
}
+ PyObject* legacy_type_obj = PyDict_GetItemString(item, "legacy_data_type");
+ if (legacy_type_obj != nullptr) {
+ vdb::FieldType legacy_type;
+ if (!py_to_field_type(legacy_type_obj, &legacy_type)) {
+ Py_DECREF(item);
+ return false;
+ }
+ field.legacy_data_type = legacy_type;
+ }
+
const long field_id = PyLong_AsLong(id_obj);
if (PyErr_Occurred() != nullptr) {
Py_DECREF(item);
@@ -1053,10 +1063,13 @@ PyObject* py_bytes_row_deserialize(PyObject*, PyObject* args) {
return nullptr;
}
- std::string payload;
- if (!py_to_string(payload_obj, &payload, true)) {
+ char* payload_data = nullptr;
+ Py_ssize_t payload_size = 0;
+ if (PyBytes_AsStringAndSize(payload_obj, &payload_data, &payload_size) < 0) {
return nullptr;
}
+ // The call arguments keep these immutable bytes alive while the GIL is released.
+ const std::string_view payload(payload_data, static_cast<size_t>(payload_size));
PyObject* result = PyDict_New();
if (result == nullptr) {
@@ -1106,10 +1119,12 @@ PyObject* py_bytes_row_deserialize_field(PyObject*, PyObject* args) {
return nullptr;
}
- std::string payload;
- if (!py_to_string(payload_obj, &payload, true)) {
+ char* payload_data = nullptr;
+ Py_ssize_t payload_size = 0;
+ if (PyBytes_AsStringAndSize(payload_obj, &payload_data, &payload_size) < 0) {
return nullptr;
}
+ const std::string_view payload(payload_data, static_cast<size_t>(payload_size));
const auto* meta = handle->schema->get_field_meta(field_name);
if (meta == nullptr) {
diff --git a/src/store/bytes_row.cpp b/src/store/bytes_row.cpp
index 7f6014e41..b8a57ac33 100644
--- a/src/store/bytes_row.cpp
+++ b/src/store/bytes_row.cpp
@@ -77,6 +77,15 @@ Schema::Schema(const std::vector<FieldDef>& fields) {
meta.offset = current_offset;
meta.id = field.id;
meta.default_value = field.default_value;
+ meta.legacy_data_type = field.legacy_data_type.value_or(field.data_type);
+ if (meta.legacy_data_type != meta.data_type) {
+ // STRING -> TEXT preserves the fixed-region offset layout.
+ if (meta.legacy_data_type != FieldType::STRING ||
+ meta.data_type != FieldType::TEXT) {
+ throw std::invalid_argument("Only STRING to TEXT migration is supported");
+ }
+ has_legacy_fields_ = true;
+ }
field_metas_[field.name] = meta;
field_orders_[field.id] = meta;
@@ -205,9 +214,14 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {
}
// Allocate buffer
- std::string buffer;
- buffer.resize(variable_region_offset);
- char* ptr = &buffer[0];
+ // Include the version in the original allocation; offsets remain relative
+ // to the row body so legacy and current schemas share the fixed layout.
+ const size_t header_size =
+ schema_->has_legacy_fields() ? VERSIONED_ROW_HEADER.size() : 0;
+ std::string buffer(variable_region_offset + header_size, '\0');
+ if (header_size != 0)
+ std::memcpy(buffer.data(), VERSIONED_ROW_HEADER.data(), header_size);
+ char* ptr = buffer.data() + header_size;
// Write header (field count)
// Be careful with alignment if we were doing raw casting, but we use memcpy
@@ -333,8 +347,17 @@ std::string BytesRow::serialize(const std::vector<Value>& row_data) const {
return buffer;
}
-Value BytesRow::deserialize_field(const std::string& serialized_data,
+Value BytesRow::deserialize_field(std::string_view serialized_data,
const std::string& field_name) const {
+ bool legacy = schema_->has_legacy_fields();
+ if (legacy && !serialized_data.empty() && serialized_data[0] == '\0') {
+ if (serialized_data.size() <= VERSIONED_ROW_HEADER.size() ||
+ serialized_data.substr(0, VERSIONED_ROW_HEADER.size()) != VERSIONED_ROW_HEADER) {
+ throw std::invalid_argument("Invalid or unsupported bytes_row record version");
+ }
+ serialized_data.remove_prefix(VERSIONED_ROW_HEADER.size());
+ legacy = false;
+ }
const FieldMeta* meta_ptr = schema_->get_field_meta(field_name);
if (!meta_ptr)
return std::monostate{};
@@ -354,7 +377,7 @@ Value BytesRow::deserialize_field(const std::string& serialized_data,
const char* field_ptr = ptr + meta.offset;
- switch (meta.data_type) {
+ switch (legacy ? meta.legacy_data_type : meta.data_type) {
case FieldType::INT64: {
int64_t v;
std::memcpy(&v, field_ptr, sizeof(v));
@@ -526,7 +549,7 @@ Value BytesRow::deserialize_field(const std::string& serialized_data,
}
std::map<std::string, Value> BytesRow::deserialize(
- const std::string& serialized_data) const {
+ std::string_view serialized_data) const {
std::map<std::string, Value> result;
const auto& order = schema_->get_field_order();
for (const auto& meta : order) {
diff --git a/src/store/bytes_row.h b/src/store/bytes_row.h
index 575870da0..319ffec9a 100644
--- a/src/store/bytes_row.h
+++ b/src/store/bytes_row.h
@@ -1,6 +1,8 @@
#pragma once
#include <string>
+#include <string_view>
+#include <optional>
#include <vector>
#include <map>
#include <variant>
@@ -12,6 +14,8 @@
namespace vectordb {
constexpr uint32_t STRING_MAX_UINT16_LENGTH = 0xFFFF;
+// A zero field count distinguishes versioned rows from legacy records.
+constexpr std::string_view VERSIONED_ROW_HEADER{"\0\1", 2};
enum class FieldType {
INT64 = 0,
@@ -37,6 +41,7 @@ struct FieldMeta {
int offset;
int id;
Value default_value;
+ FieldType legacy_data_type;
};
struct FieldDef {
@@ -44,6 +49,7 @@ struct FieldDef {
FieldType data_type;
int id;
Value default_value;
+ std::optional<FieldType> legacy_data_type;
};
class Schema {
@@ -56,12 +62,16 @@ class Schema {
int get_total_byte_length() const {
return total_byte_length_;
}
+ bool has_legacy_fields() const {
+ return has_legacy_fields_;
+ }
const FieldMeta* get_field_meta(const std::string& name) const;
private:
std::vector<FieldMeta> field_orders_;
std::map<std::string, FieldMeta> field_metas_;
int total_byte_length_;
+ bool has_legacy_fields_ = false;
};
class BytesRow {
@@ -272,9 +282,12 @@ class BytesRow {
}
}
- std::string buffer;
- buffer.resize(variable_region_offset);
- char* ptr = &buffer[0];
+ const size_t header_size =
+ schema_->has_legacy_fields() ? VERSIONED_ROW_HEADER.size() : 0;
+ std::string buffer(variable_region_offset + header_size, '\0');
+ if (header_size != 0)
+ std::memcpy(buffer.data(), VERSIONED_ROW_HEADER.data(), header_size);
+ char* ptr = buffer.data() + header_size;
// Header
ptr[0] = static_cast<uint8_t>(field_order.size());
@@ -406,10 +419,10 @@ class BytesRow {
// Deserialize to a map
std::map<std::string, Value> deserialize(
- const std::string& serialized_data) const;
+ std::string_view serialized_data) const;
// Deserialize a single field
- Value deserialize_field(const std::string& serialized_data,
+ Value deserialize_field(std::string_view serialized_data,
const std::string& field_name) const;
// Get schema
diff --git a/tests/vectordb/test_bytes_row.py b/tests/vectordb/test_bytes_row.py
index 8b197e2f1..25550c368 100644
--- a/tests/vectordb/test_bytes_row.py
+++ b/tests/vectordb/test_bytes_row.py
@@ -363,11 +363,27 @@ class TestTextFieldType(unittest.TestCase):
self.assertEqual(self.py_row.deserialize_field(cpp_bytes, "body"), data["body"])
def test_binary_consistency(self):
+ legacy_data = {"label": 42, "body": "升级前的记录"}
+ readers = []
+ for fields, field_type, schema, row in (
+ (self.cpp_fields, engine.FieldType, engine.Schema, engine.BytesRow),
+ (self.py_fields, _PyFieldType, _PySchema, _PyBytesRow),
+ ):
+ legacy_schema = schema([fields[0], {**fields[1], "data_type": field_type.string}])
+ legacy_bytes = row(legacy_schema).serialize(legacy_data)
+ reader = row(schema([fields[0], {**fields[1], "legacy_data_type": field_type.string}]))
+ self.assertEqual(reader.deserialize(legacy_bytes), legacy_data)
+ readers.append(reader)
+
data = self._make_data()
- py_bytes = self.py_row.serialize(data)
- cpp_bytes = self.cpp_row.serialize(data)
- self.assertEqual(len(py_bytes), len(cpp_bytes), "Binary length mismatch")
- self.assertEqual(py_bytes, cpp_bytes, "Binary content mismatch")
+ cpp_bytes, py_bytes = [reader.serialize(data) for reader in readers]
+ self.assertEqual(cpp_bytes, py_bytes, "Binary content mismatch")
+ for reader in readers:
+ self.assertEqual(reader.deserialize(cpp_bytes), data)
+ self.assertEqual(reader.deserialize_field(cpp_bytes, "label"), 42)
+ for invalid in (b"\x00\x01", b"\x00\x02\x02"):
+ with self.assertRaisesRegex((ValueError, RuntimeError), "record version"):
+ reader.deserialize(invalid)
def test_text_declared_via_metadata(self):
@serializable
diff --git a/tests/vectordb/test_crash_recovery.py b/tests/vectordb/test_crash_recovery.py
index d05a2dd7e..463821e7c 100644
--- a/tests/vectordb/test_crash_recovery.py
+++ b/tests/vectordb/test_crash_recovery.py
@@ -6,11 +6,16 @@ import shutil
import sys
import time
import unittest
+from contextlib import ExitStack
+from unittest.mock import patch
+from openviking.storage.vectordb import engine
from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection
+from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
DB_PATH_CRASH = "./test_data/test_db_crash_recovery"
DB_PATH_ROBUST = "./test_data/test_db_robust_crash"
+LARGE_TEXT = "旧数据升级后的长文本" * 4000
def worker_write_and_crash(path, start_id, count, event_ready):
@@ -52,8 +57,42 @@ def worker_write_and_crash(path, start_id, count, event_ready):
}
)
- print(f"[Subprocess] Upserting {count} items...")
- col.upsert_data(data)
+ # Seed real unversioned candidate and delta rows as an older writer did.
+ # Keep one legacy row untouched; update the other after switching writers.
+ legacy_fields = [
+ ("label", "uint64"),
+ ("vector", "list_float32"),
+ ("sparse_raw_terms", "list_string"),
+ ("sparse_values", "list_float32"),
+ ("fields", "string"),
+ ]
+ with ExitStack() as legacy_writer:
+ for record_type, definitions in (
+ (CandidateData, [*legacy_fields, ("expire_ns_ts", "uint64")]),
+ (DeltaRecord, [("type", "uint64"), *legacy_fields, ("old_fields", "string")]),
+ ):
+ legacy_schema = engine.Schema(
+ [
+ {
+ "name": name,
+ "id": index,
+ "data_type": getattr(engine.FieldType, field_type),
+ }
+ for index, (name, field_type) in enumerate(definitions)
+ ]
+ )
+ legacy_writer.enter_context(
+ patch.object(record_type, "bytes_row", engine.BytesRow(legacy_schema))
+ )
+ col.upsert_data(data[:2])
+
+ print(f"[Subprocess] Upserting {count} mixed-format items...")
+ col.upsert_data(data[2:])
+ col.update_data([{"id": start_id, "data": LARGE_TEXT}])
+ col.update_data([{"id": start_id, "data": LARGE_TEXT + " updated"}])
+ # Leave a large deletion in the log too, without changing the final count.
+ col.upsert_data([{"id": start_id + count, "vector": [0.1] * 4, "data": LARGE_TEXT}])
+ col.delete_data([start_id + count])
print("[Subprocess] Upsert done. Not closing.")
# Notify main process that write is done
@@ -208,6 +247,10 @@ class TestCrashRecovery(unittest.TestCase):
self.assertEqual(
len(res.items), data_count, f"Should find all {data_count} items in KV Store"
)
+ payloads = {item.id: item.fields["data"] for item in res.items}
+ self.assertEqual(payloads[0], LARGE_TEXT + " updated")
+ self.assertEqual(payloads[1], "crash_data_1")
+ self.assertEqual(col.fetch_data([data_count]).ids_not_exist, [data_count])
print("[Main] Data fetch verified.")
@@ -226,6 +269,7 @@ class TestCrashRecovery(unittest.TestCase):
self.assertEqual(
len(found_ids), data_count, "Index should contain all items after recovery"
)
+ self.assertEqual(set(found_ids), set(all_ids))
col.close()
print("[Main] Simple recovery test passed.")
diff --git a/tests/vectordb/test_filter_ops.py b/tests/vectordb/test_filter_ops.py
index 608d64520..f4aca36ab 100644
--- a/tests/vectordb/test_filter_ops.py
+++ b/tests/vectordb/test_filter_ops.py
@@ -14,7 +14,6 @@ db_path_basic = "./test_data/db_test_filters_basic/"
db_path_complex = "./test_data/db_test_filters_complex/"
db_path_lifecycle = "./test_data/db_test_filters_lifecycle/"
db_path_scale = "./test_data/db_test_filters_scale/"
-db_path_large_fields = "./test_data/db_test_filters_large_fields/"
def clean_dir(path):
@@ -116,45 +115,6 @@ class TestFilterOpsBasic(unittest.TestCase):
) # chERry, eldERbERry
-class TestFilterOpsLargeFields(unittest.TestCase):
- def setUp(self):
- clean_dir(db_path_large_fields)
- self.path = db_path_large_fields
- self.collection = self._create_collection()
-
- def tearDown(self):
- if self.collection:
- self.collection.drop()
- clean_dir(self.path)
-
- def _create_collection(self):
- collection_meta = {
- "CollectionName": "test_filters_large_fields",
- "Fields": [
- {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
- {"FieldName": "embedding", "FieldType": "vector", "Dim": 4},
- {"FieldName": "uri", "FieldType": "string"},
- {"FieldName": "abstract", "FieldType": "string"},
- ],
- }
- return get_or_create_local_collection(meta_data=collection_meta, path=self.path)
-
- def test_upsert_record_with_oversized_json_field_raises(self):
- large_abstract = 'prefix "quoted" \\\\ path\n' + ("x" * 66000)
- uri = "viking://user/memories/large.md"
- with self.assertRaisesRegex((RuntimeError, ValueError), "fields.*exceeds 65535 bytes"):
- self.collection.upsert_data(
- [
- {
- "id": 1,
- "embedding": [1.0, 0, 0, 0],
- "uri": uri,
- "abstract": large_abstract,
- }
- ]
- )
-
-
class TestFilterOpsComplex(unittest.TestCase):
"""Complex mixed logic tests"""