Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(vectordb): support safe local schema updates (#4700)

Qin Haojie37db9c834e95
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index 716e8c878..57cd34e66 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -319,6 +319,24 @@ async def init_context_collection(storage) -> bool:
"Existing collection metadata is unavailable; cannot validate embedding compatibility"
)
+ expected_fields = {field.get("FieldName") for field in schema["Fields"]}
+ existing_fields = {field.get("FieldName") for field in existing_meta.get("Fields", [])}
+ missing_fields = sorted(expected_fields - existing_fields)
+ expected_scalar_indexes = set(schema["ScalarIndex"])
+ existing_scalar_indexes = set(existing_meta.get("ScalarIndex", []))
+ missing_scalar_indexes = sorted(expected_scalar_indexes - existing_scalar_indexes)
+
+ async def _update_local_schema() -> None:
+ if vectordb_cfg.backend not in {"local", "cuvs"} or "Fields" not in existing_meta:
+ return
+ if not missing_fields and not missing_scalar_indexes:
+ return
+ if not hasattr(storage, "update_collection_schema"):
+ raise EmbeddingConfigurationError(
+ "Local context collection does not support automatic schema updates"
+ )
+ await storage.update_collection_schema(schema["Fields"], schema["ScalarIndex"])
+
base_description, existing_embedding_meta = _decode_collection_description(
existing_meta.get("Description")
)
@@ -337,6 +355,7 @@ async def init_context_collection(storage) -> bool:
)
if _embedding_metadata_compatible(existing_embedding_meta, embedding_meta):
+ await _update_local_schema()
return False
existing_count = await storage.count() if hasattr(storage, "count") else 0
@@ -348,6 +367,7 @@ async def init_context_collection(storage) -> bool:
embedding_meta,
)
)
+ await _update_local_schema()
return False
if existing_embedding_meta is None:
@@ -363,6 +383,7 @@ async def init_context_collection(storage) -> bool:
embedding_meta,
)
)
+ await _update_local_schema()
return False
if existing_count == 0 and hasattr(storage, "update_collection_description"):
@@ -372,6 +393,7 @@ async def init_context_collection(storage) -> bool:
embedding_meta,
)
)
+ await _update_local_schema()
return False
# Embedding metadata differs from current config and the collection is
@@ -407,6 +429,7 @@ async def init_context_collection(storage) -> bool:
embedding_meta,
)
)
+ await _update_local_schema()
return False
if dimension_changed:
diff --git a/openviking/storage/vectordb/collection/collection.py b/openviking/storage/vectordb/collection/collection.py
index b4499d08b..9910c7061 100644
--- a/openviking/storage/vectordb/collection/collection.py
+++ b/openviking/storage/vectordb/collection/collection.py
@@ -136,7 +136,7 @@ class ICollection(ABC):
def update_index(
self,
index_name: str,
- scalar_index: Optional[Dict[str, Any]] = None,
+ scalar_index: Optional[List[str]] = None,
description: Optional[str] = None,
):
raise NotImplementedError
@@ -535,7 +535,7 @@ class Collection:
def update_index(
self,
index_name: str,
- scalar_index: Optional[Dict[str, Any]] = None,
+ scalar_index: Optional[List[str]] = None,
description: Optional[str] = None,
):
"""
@@ -543,7 +543,7 @@ class Collection:
Args:
index_name (str): Name of the index to update.
- scalar_index (Optional[Dict[str, Any]]): Updated configuration for scalar indexes.
+ scalar_index (Optional[List[str]]): Complete list of scalar-indexed fields.
Defaults to None.
description (Optional[str]): New description for the index. Defaults to None.
"""
diff --git a/openviking/storage/vectordb/collection/http_collection.py b/openviking/storage/vectordb/collection/http_collection.py
index 2c9a85b86..7386a9c70 100644
--- a/openviking/storage/vectordb/collection/http_collection.py
+++ b/openviking/storage/vectordb/collection/http_collection.py
@@ -217,7 +217,7 @@ class HttpCollection(ICollection):
def update_index(
self,
index_name: str,
- scalar_index: Optional[Dict[str, Any]] = None,
+ scalar_index: Optional[List[str]] = None,
description: Optional[str] = None,
):
data = {
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.py
index 7bf80d019..b3ba5f136 100644
--- a/openviking/storage/vectordb/collection/local_collection.py
+++ b/openviking/storage/vectordb/collection/local_collection.py
@@ -475,15 +475,21 @@ class LocalCollection(ICollection):
def update_index(
self,
index_name: str,
- scalar_index: Optional[Dict[str, Any]] = None,
+ scalar_index: Optional[List[str]] = None,
description: Optional[str] = None,
) -> None:
- with self._index_mutation_barrier.mutation() as mutation:
+ with self._index_mutation_barrier.exclusive_mutation():
index = self.indexes.get(index_name)
if not index:
return
- index.update(scalar_index, description)
- mutation.mark_changed()
+ if scalar_index is not None:
+ if not self.store_mgr:
+ raise RuntimeError("Store manager is not initialized")
+ index.rebuild_scalar_index(
+ scalar_index, self.store_mgr.get_all_cands_data()
+ )
+ if description is not None:
+ index.update(None, description)
def get_index_meta_data(self, index_name: str) -> Optional[Dict[str, Any]]:
index = self.indexes.get(index_name)
diff --git a/openviking/storage/vectordb/collection/vikingdb_collection.py b/openviking/storage/vectordb/collection/vikingdb_collection.py
index 966bc7c9e..b36bc00e9 100644
--- a/openviking/storage/vectordb/collection/vikingdb_collection.py
+++ b/openviking/storage/vectordb/collection/vikingdb_collection.py
@@ -169,7 +169,7 @@ class VikingDBCollection(ICollection):
def update_index(
self,
index_name: str,
- scalar_index: Optional[Dict[str, Any]] = None,
+ scalar_index: Optional[List[str]] = None,
description: Optional[str] = None,
):
raise NotImplementedError("index should be managed manually")
diff --git a/openviking/storage/vectordb/collection/volcengine_api_key_collection.py b/openviking/storage/vectordb/collection/volcengine_api_key_collection.py
index 89ffbdfe4..7f5f6fbe5 100644
--- a/openviking/storage/vectordb/collection/volcengine_api_key_collection.py
+++ b/openviking/storage/vectordb/collection/volcengine_api_key_collection.py
@@ -298,7 +298,7 @@ class VolcengineApiKeyCollection(ICollection):
def update_index(
self,
index_name: str,
- scalar_index: Optional[Dict[str, Any]] = None,
+ scalar_index: Optional[List[str]] = None,
description: Optional[str] = None,
):
raise NotImplementedError(
diff --git a/openviking/storage/vectordb/collection/volcengine_collection.py b/openviking/storage/vectordb/collection/volcengine_collection.py
index 20196d2bf..1338f0146 100644
--- a/openviking/storage/vectordb/collection/volcengine_collection.py
+++ b/openviking/storage/vectordb/collection/volcengine_collection.py
@@ -363,7 +363,7 @@ class VolcengineCollection(ICollection):
def update_index(
self,
index_name: str,
- scalar_index: Optional[Dict[str, Any]] = None,
+ scalar_index: Optional[List[str]] = None,
description: Optional[str] = None,
):
data = {
diff --git a/openviking/storage/vectordb/engine/_python_api.py b/openviking/storage/vectordb/engine/_python_api.py
index d7c62dfc1..718892073 100644
--- a/openviking/storage/vectordb/engine/_python_api.py
+++ b/openviking/storage/vectordb/engine/_python_api.py
@@ -474,6 +474,17 @@ def build_abi3_exports(backend: Any) -> dict[str, Any]:
)
)
+ def rebuild_scalar_index(
+ self, scalar_index_json: str, data_list: list[AddDataRequest]
+ ) -> int:
+ return int(
+ self._backend._index_engine_rebuild_scalar_index(
+ self._handle,
+ scalar_index_json,
+ _request_list_to_backend(data_list),
+ )
+ )
+
def search(self, req: SearchRequest) -> SearchResult:
return SearchResult.from_backend(self._backend._index_engine_search(self._handle, req))
diff --git a/openviking/storage/vectordb/index/index.py b/openviking/storage/vectordb/index/index.py
index a0499dbf8..1be3b64dc 100644
--- a/openviking/storage/vectordb/index/index.py
+++ b/openviking/storage/vectordb/index/index.py
@@ -3,7 +3,7 @@
from abc import ABC, abstractmethod
from typing import Any, Dict, List, Optional, Tuple, Union
-from openviking.storage.vectordb.store.data import DeltaRecord
+from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
class IIndex(ABC):
@@ -189,6 +189,13 @@ class IIndex(ABC):
"""
raise NotImplementedError
+ @abstractmethod
+ def rebuild_scalar_index(
+ self, scalar_index: List[str], cands_list: List[CandidateData]
+ ) -> None:
+ """Replace the scalar index from a Store snapshot without rebuilding vectors."""
+ raise NotImplementedError
+
@abstractmethod
def get_meta_data(self):
"""Retrieve the complete metadata of the index.
diff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.py
index f33949099..e0e3fb293 100644
--- a/openviking/storage/vectordb/index/local_index.py
+++ b/openviking/storage/vectordb/index/local_index.py
@@ -24,11 +24,12 @@ from openviking.storage.vectordb.index.index import IIndex
from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
from openviking.storage.vectordb.utils.constants import IndexFileMarkers
from openviking.storage.vectordb.utils.data_processor import DataProcessor
+from openviking.storage.vectordb.utils.json_safety import safe_json_dumps
from openviking.storage.vectordb.utils.path_safety import (
safe_join,
safe_join_name,
)
-from openviking.storage.vectordb.utils.validation import validate_name_str
+from openviking.storage.vectordb.utils.validation import fix_fields_data, validate_name_str
from openviking_cli.utils.logger import default_logger as logger
_DENSE_REBUILD_MEMORY_RETRY_BASE_SECONDS = 1.0
@@ -199,6 +200,22 @@ class IndexEngineProxy:
add_req_list[i].fields_str = data.fields
self.index_engine.add_data(add_req_list)
+ def rebuild_scalar_index(
+ self, scalar_index_meta: List[Dict[str, str]], cands_list: List[CandidateData]
+ ) -> None:
+ if not self.index_engine:
+ raise RuntimeError("Index engine not initialized")
+
+ requests = [engine.AddDataRequest() for _ in range(len(cands_list))]
+ for request, data in zip(requests, cands_list, strict=False):
+ request.label = data.label
+ request.fields_str = data.fields
+ result = self.index_engine.rebuild_scalar_index(
+ json.dumps(scalar_index_meta), requests
+ )
+ if result != 0:
+ raise RuntimeError("Failed to rebuild native scalar index")
+
def upsert_data(self, delta_list: List[DeltaRecord]):
if not self.index_engine:
raise RuntimeError("Index engine not initialized")
@@ -465,6 +482,44 @@ class LocalIndex(IIndex):
return
self.meta.update(meta_data)
+ def rebuild_scalar_index(
+ self, scalar_index: List[str], cands_list: List[CandidateData]
+ ) -> None:
+ if not self.engine_proxy:
+ raise RuntimeError("Index engine not initialized")
+
+ self.field_type_converter = DataProcessor(self.meta.collection_meta.fields_dict)
+ converted_candidates: List[CandidateData] = []
+ vector_key = self.meta.collection_meta.vector_key
+ sparse_vector_key = self.meta.collection_meta.sparse_vector_key
+ for candidate in cands_list:
+ fields = fix_fields_data(
+ json.loads(candidate.fields), self.meta.collection_meta.fields_dict
+ )
+ fields.pop(vector_key, None)
+ if sparse_vector_key:
+ fields.pop(sparse_vector_key, None)
+ converted = CandidateData()
+ converted.label = candidate.label
+ converted.fields = safe_json_dumps(fields, ensure_ascii=False)
+ converted_candidates.append(converted)
+
+ engine_scalar_meta = self.field_type_converter.build_scalar_index_meta(scalar_index)
+ self.engine_proxy.rebuild_scalar_index(
+ engine_scalar_meta,
+ self._convert_candidate_list_for_index(converted_candidates),
+ )
+ if isinstance(self, PersistentIndex) and self.persist() <= 0:
+ raise RuntimeError("Failed to persist rebuilt scalar index")
+ if not self.meta.update({"ScalarIndex": scalar_index}):
+ raise RuntimeError("Failed to persist scalar index metadata")
+
+ if self.dense_search is not None:
+ self.dense_search.field_types = {
+ name: DataProcessor.normalize_field_type(field_meta.get("FieldType", ""))
+ for name, field_meta in self.meta.collection_meta.fields_dict.items()
+ }
+
def get_meta_data(self):
return self.meta.get_meta_data()
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index 2769b25be..4f1f20074 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -147,14 +147,51 @@ class _AsyncVectorAdapter:
async def run(self, func: Any, /, *args: Any, **kwargs: Any) -> Any:
return await asyncio.to_thread(func, *args, **kwargs)
- async def collection_meta(self) -> Dict[str, Any]:
- return await asyncio.to_thread(lambda: self._adapter.get_collection().get_meta_data() or {})
+ async def collection_meta(self, index_name: str) -> Dict[str, Any]:
+ def _get() -> Dict[str, Any]:
+ collection = self._adapter.get_collection()
+ meta = collection.get_meta_data() or {}
+ if self._adapter.mode in {"local", "cuvs"}:
+ index_meta = collection.get_index_meta_data(index_name) or {}
+ if "ScalarIndex" in index_meta:
+ meta["ScalarIndex"] = index_meta["ScalarIndex"]
+ return meta
+
+ return await asyncio.to_thread(_get)
async def update_collection_description(self, description: str) -> None:
await asyncio.to_thread(
lambda: self._adapter.get_collection().update(description=description)
)
+ async def update_collection_schema(
+ self, fields: List[Dict[str, Any]], scalar_index: List[str], index_name: str
+ ) -> None:
+ def _update() -> None:
+ collection = self._adapter.get_collection()
+ existing_fields = {
+ field.get("FieldName") for field in collection.get_meta_data().get("Fields", [])
+ }
+ missing_fields = [
+ field for field in fields if field.get("FieldName") not in existing_fields
+ ]
+ if missing_fields:
+ collection.update(fields=missing_fields)
+
+ index_meta = collection.get_index_meta_data(index_name) or {}
+ current_scalar_index = index_meta.get("ScalarIndex", [])
+ indexed_fields = set(current_scalar_index)
+ missing_scalar_fields = [
+ field for field in scalar_index if field not in indexed_fields
+ ]
+ if missing_scalar_fields:
+ collection.update_index(
+ index_name,
+ scalar_index=[*current_scalar_index, *missing_scalar_fields],
+ )
+
+ await asyncio.to_thread(_update)
+
class _SingleAccountBackend:
"""绑定单个 account 的后端实现(内部类)"""
@@ -274,7 +311,7 @@ class _SingleAccountBackend:
return "not found" in message or "does not exist" in message
async def _refresh_meta_data_async(self) -> None:
- self._meta_data_cache = await self._async_adapter.collection_meta()
+ self._meta_data_cache = await self._async_adapter.collection_meta(self._index_name)
# =========================================================================
# Collection Management
@@ -340,7 +377,7 @@ class _SingleAccountBackend:
async def get_collection_meta(self) -> Optional[Dict[str, Any]]:
if not await self.collection_exists():
return None
- return await self._async_adapter.collection_meta()
+ return await self._async_adapter.collection_meta(self._index_name)
async def update_collection_description(self, description: str) -> bool:
if not await self.collection_exists():
@@ -349,6 +386,14 @@ class _SingleAccountBackend:
await self._refresh_meta_data_async()
return True
+ async def update_collection_schema(
+ self, fields: List[Dict[str, Any]], scalar_index: List[str]
+ ) -> None:
+ await self._async_adapter.update_collection_schema(
+ fields, scalar_index, self._index_name
+ )
+ await self._refresh_meta_data_async()
+
# =========================================================================
# Data Operations (with tenant enforcement)
# =========================================================================
@@ -987,6 +1032,15 @@ class VikingVectorIndexBackend:
async def update_collection_description(self, description: str) -> bool:
return await self._get_default_backend().update_collection_description(description)
+ async def update_collection_schema(
+ self, fields: List[Dict[str, Any]], scalar_index: List[str]
+ ) -> None:
+ default_backend = self._get_default_backend()
+ await default_backend.update_collection_schema(fields, scalar_index)
+ for backend in [*self._account_backends.values(), self._root_backend]:
+ if backend is not None and backend is not default_backend:
+ await backend._refresh_meta_data_async()
+
# =========================================================================
# 公开数据操作 API(强制要求 ctx)
# =========================================================================
diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cpp
index f88f09d09..4417e77a4 100644
--- a/src/abi3_engine_backend.cpp
+++ b/src/abi3_engine_backend.cpp
@@ -1377,6 +1377,35 @@ PyObject* py_index_engine_delete_data(PyObject*, PyObject* args) {
}
}
+PyObject* py_index_engine_rebuild_scalar_index(PyObject*, PyObject* args) {
+ PyObject* capsule = nullptr;
+ const char* scalar_index_json = nullptr;
+ PyObject* items = nullptr;
+ if (!PyArg_ParseTuple(args, "OsO", &capsule, &scalar_index_json, &items)) {
+ return nullptr;
+ }
+
+ auto* engine = capsule_to_ptr<vdb::IndexEngine>(capsule, kIndexCapsuleName);
+ if (engine == nullptr) {
+ return nullptr;
+ }
+
+ std::vector<vdb::AddDataRequest> requests;
+ if (!parse_request_list(items, parse_add_request, &requests)) {
+ return nullptr;
+ }
+
+ try {
+ const int result = call_without_gil([&]() {
+ return engine->rebuild_scalar_index(scalar_index_json, requests);
+ });
+ return PyLong_FromLong(result);
+ } catch (const std::exception& exc) {
+ raise_runtime_error(exc.what());
+ return nullptr;
+ }
+}
+
PyObject* py_index_engine_search(PyObject*, PyObject* args) {
PyObject* capsule = nullptr;
PyObject* request_obj = nullptr;
@@ -1942,6 +1971,9 @@ PyMethodDef kModuleMethods[] = {
"Add data to the index engine."},
{"_index_engine_delete_data", py_index_engine_delete_data, METH_VARARGS,
"Delete data from the index engine."},
+ {"_index_engine_rebuild_scalar_index",
+ py_index_engine_rebuild_scalar_index, METH_VARARGS,
+ "Rebuild scalar fields without rebuilding the vector index."},
{"_index_engine_search", py_index_engine_search, METH_VARARGS,
"Search the index engine."},
{"_index_engine_search_with_filter_token",
diff --git a/src/index/detail/index_manager_impl.cpp b/src/index/detail/index_manager_impl.cpp
index a693e62d9..553b02ba6 100644
--- a/src/index/detail/index_manager_impl.cpp
+++ b/src/index/detail/index_manager_impl.cpp
@@ -623,6 +623,59 @@ int IndexManagerImpl::delete_data(
return 0;
}
+int IndexManagerImpl::rebuild_scalar_index(
+ const std::string& scalar_index_json,
+ const std::vector<AddDataRequest>& data_list) {
+ JsonDoc scalar_index_doc;
+ scalar_index_doc.Parse(scalar_index_json.c_str());
+ if (scalar_index_doc.HasParseError() || !scalar_index_doc.IsArray()) {
+ throw std::invalid_argument("Invalid scalar index metadata");
+ }
+
+ auto next_meta = std::make_shared<ScalarIndexMeta>();
+ if (next_meta->init_from_json(scalar_index_doc) != 0) {
+ throw std::invalid_argument("Invalid scalar index metadata");
+ }
+
+ std::vector<FieldsDict> parsed_fields(data_list.size());
+ for (size_t i = 0; i < data_list.size(); ++i) {
+ if (parsed_fields[i].parse_from_json(data_list[i].fields_str) != 0) {
+ throw std::runtime_error(
+ "Failed to parse scalar fields for label=" +
+ std::to_string(data_list[i].label));
+ }
+ }
+
+ auto next_scalar_index = std::make_shared<ScalarIndex>(next_meta);
+ std::unique_lock<std::shared_mutex> lock(rw_mutex_);
+ for (size_t i = 0; i < data_list.size(); ++i) {
+ const int offset = vector_index_->get_offset_by_label(data_list[i].label);
+ if (offset < 0) {
+ SPDLOG_WARN("IndexManagerImpl::rebuild_scalar_index label={} not found",
+ data_list[i].label);
+ continue;
+ }
+ if (next_scalar_index->add_row_data(offset, parsed_fields[i],
+ FieldsDict{}) != 0) {
+ throw std::runtime_error(
+ "Failed to rebuild scalar fields for label=" +
+ std::to_string(data_list[i].label));
+ }
+ }
+
+ scalar_index_ = std::move(next_scalar_index);
+ manager_meta_->scalar_index_meta = std::move(next_meta);
+ register_label_offset_converter_();
+ clear_filter_layout_();
+ clear_filter_token_cache_();
+ const auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(
+ std::chrono::system_clock::now().time_since_epoch())
+ .count();
+ manager_meta_->update_timestamp = std::max(
+ manager_meta_->update_timestamp + 1, static_cast<uint64_t>(now));
+ return 0;
+}
+
int64_t IndexManagerImpl::dump(const std::string& dir) {
std::filesystem::path dir_path(dir);
std::shared_lock<std::shared_mutex> lock(rw_mutex_);
diff --git a/src/index/detail/index_manager_impl.h b/src/index/detail/index_manager_impl.h
index 6180b0246..75778338b 100644
--- a/src/index/detail/index_manager_impl.h
+++ b/src/index/detail/index_manager_impl.h
@@ -52,6 +52,10 @@ class IndexManagerImpl : public IndexManager {
int delete_data(const std::vector<DeleteDataRequest>& data_list) override;
+ int rebuild_scalar_index(
+ const std::string& scalar_index_json,
+ const std::vector<AddDataRequest>& data_list) override;
+
int64_t dump(const std::string& dir) override;
int get_state(StateResult& state_result) override;
diff --git a/src/index/index_engine.cpp b/src/index/index_engine.cpp
index 0ebd15e1f..1712f1044 100644
--- a/src/index/index_engine.cpp
+++ b/src/index/index_engine.cpp
@@ -70,6 +70,12 @@ int IndexEngine::delete_data(const std::vector<DeleteDataRequest>& data_list) {
return impl_->delete_data(data_list);
}
+int IndexEngine::rebuild_scalar_index(
+ const std::string& scalar_index_json,
+ const std::vector<AddDataRequest>& data_list) {
+ return impl_->rebuild_scalar_index(scalar_index_json, data_list);
+}
+
int64_t IndexEngine::dump(const std::string& dir) {
return impl_->dump(dir);
}
diff --git a/src/index/index_engine.h b/src/index/index_engine.h
index 1c82a9244..e177afbdb 100644
--- a/src/index/index_engine.h
+++ b/src/index/index_engine.h
@@ -23,6 +23,10 @@ class IndexEngine {
int delete_data(const std::vector<DeleteDataRequest>& data_list);
+ int rebuild_scalar_index(
+ const std::string& scalar_index_json,
+ const std::vector<AddDataRequest>& data_list);
+
SearchResult search(const SearchRequest& req);
std::optional<SearchResult> search_with_filter_token(
diff --git a/src/index/index_manager.h b/src/index/index_manager.h
index 9b6e1f92f..344241d4c 100644
--- a/src/index/index_manager.h
+++ b/src/index/index_manager.h
@@ -36,6 +36,10 @@ class IndexManager {
virtual int delete_data(const std::vector<DeleteDataRequest>& data_list) = 0;
+ virtual int rebuild_scalar_index(
+ const std::string& scalar_index_json,
+ const std::vector<AddDataRequest>& data_list) = 0;
+
virtual int64_t dump(const std::string& dir) = 0;
virtual int get_state(StateResult& state_result) = 0;
diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.py
index b0a0d09cc..29a763e12 100644
--- a/tests/storage/test_collection_schemas.py
+++ b/tests/storage/test_collection_schemas.py
@@ -186,6 +186,15 @@ async def test_init_context_collection_writes_embedding_metadata(monkeypatch):
@pytest.mark.asyncio
async def test_init_context_collection_backfills_metadata_for_empty_legacy_collection(monkeypatch):
updates = []
+ schema_updates = []
+ config = _DummyConfig(_DummyEmbedder(), backend="local")
+ existing_schema = CollectionSchemas.context_collection("context", config.embedding.dimension)
+ existing_fields = [
+ field for field in existing_schema["Fields"] if field["FieldName"] != "tags"
+ ]
+ existing_scalar_index = [
+ field for field in existing_schema["ScalarIndex"] if field != "tags"
+ ]
class _FakeStorage:
async def create_collection(self, name, schema):
@@ -193,7 +202,11 @@ async def test_init_context_collection_backfills_metadata_for_empty_legacy_colle
return False
async def get_collection_meta(self):
- return {"Description": "Unified context collection"}
+ return {
+ "Description": "Unified context collection",
+ "Fields": existing_fields,
+ "ScalarIndex": existing_scalar_index,
+ }
async def count(self):
return 0
@@ -202,7 +215,9 @@ async def test_init_context_collection_backfills_metadata_for_empty_legacy_colle
updates.append(description)
return True
- config = _DummyConfig(_DummyEmbedder())
+ async def update_collection_schema(self, fields, scalar_index):
+ schema_updates.append((fields, scalar_index))
+
monkeypatch.setattr(
"openviking_cli.utils.config.get_openviking_config",
lambda: config,
@@ -213,6 +228,12 @@ async def test_init_context_collection_backfills_metadata_for_empty_legacy_colle
assert created is False
assert len(updates) == 1
assert '"provider": "local"' in updates[0]
+ assert len(schema_updates) == 1
+ fields, scalar_index = schema_updates[0]
+ assert {field["FieldName"] for field in fields} - {
+ field["FieldName"] for field in existing_fields
+ } == {"tags"}
+ assert set(scalar_index) - set(existing_scalar_index) == {"tags"}
@pytest.mark.asyncio
diff --git a/tests/vectordb/test_engine_filter_packed_native_abi.py b/tests/vectordb/test_engine_filter_packed_native_abi.py
index 426b957b2..dde81d9b3 100644
--- a/tests/vectordb/test_engine_filter_packed_native_abi.py
+++ b/tests/vectordb/test_engine_filter_packed_native_abi.py
@@ -75,3 +75,23 @@ def test_native_packed_filter_abi_has_exact_little_endian_layout_and_shapes():
assert routed_wide.native_filter_token == 0
assert empty.bitset_words == b"\x00" * 12
assert empty.eligible_count == 0
+
+ for row, request in enumerate(requests):
+ request.fields_str = json.dumps(
+ {
+ "uri": f"/keep/item-{row}" if row in selected_rows else f"/drop/item-{row}",
+ "schema_flag": row in selected_rows,
+ }
+ )
+ scalar_meta = [
+ {"FieldName": "uri", "FieldType": "path"},
+ {"FieldName": "schema_flag", "FieldType": "bool"},
+ ]
+ assert index.rebuild_scalar_index(json.dumps(scalar_meta), requests) == 0
+ assert index.set_filter_layout(labels) == 0
+ rebuilt = index.evaluate_filter(
+ json.dumps({"op": "must", "field": "schema_flag", "conds": [True]})
+ )
+ assert rebuilt.bitset_words == expected_words
+ assert rebuilt.eligible_count == len(selected_rows)
+ assert index.evaluate_filter(keep_filter).bitset_words == expected_words
diff --git a/tests/vectordb/test_openviking_vectordb.py b/tests/vectordb/test_openviking_vectordb.py
index dae80c7fc..b75f80882 100644
--- a/tests/vectordb/test_openviking_vectordb.py
+++ b/tests/vectordb/test_openviking_vectordb.py
@@ -4,6 +4,7 @@
import json
import shutil
import unittest
+from pathlib import Path
from unittest.mock import patch
from openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection
@@ -254,6 +255,28 @@ class TestOpenVikingVectorDB(unittest.TestCase):
index_meta = collection.get_index_meta_data("idx_filters") or {}
self.assertIn("type", index_meta.get("ScalarIndex", []))
+ index = collection.get_index("idx_filters")
+ previous_version = index.get_newest_version()
+ collection.update(
+ fields=[
+ {"FieldName": "schema_flag", "FieldType": "bool", "DefaultValue": False}
+ ]
+ )
+ collection.update_index(
+ "idx_filters",
+ scalar_index=[*index_meta["ScalarIndex"], "schema_flag"],
+ )
+ current_version = index.get_newest_version()
+ self.assertGreater(current_version, previous_version)
+ self.assertTrue(Path(index.version_dir, str(previous_version)).is_dir())
+ self.assertTrue(Path(f"{index.version_dir}/{previous_version}.write_done").is_file())
+ self.assertEqual(
+ self._search_ids(
+ collection,
+ {"op": "must", "field": "schema_flag", "conds": [False]},
+ ),
+ sorted(item["id"] for item in self.data),
+ )
fetched = collection.fetch_data(["res_1"])
self.assertEqual(fetched.items[0].fields.get("type"), "file")