Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(vectordb): support safe local schema updates (#4700)
Qin Haojie37db9c834e95
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex 716e8c878..57cd34e66 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -319,6 +319,24 @@ async def init_context_collection(storage) -> bool:"Existing collection metadata is unavailable; cannot validate embedding compatibility")+ expected_fields = {field.get("FieldName") for field in schema["Fields"]}+ existing_fields = {field.get("FieldName") for field in existing_meta.get("Fields", [])}+ missing_fields = sorted(expected_fields - existing_fields)+ expected_scalar_indexes = set(schema["ScalarIndex"])+ existing_scalar_indexes = set(existing_meta.get("ScalarIndex", []))+ missing_scalar_indexes = sorted(expected_scalar_indexes - existing_scalar_indexes)++ async def _update_local_schema() -> None:+ if vectordb_cfg.backend not in {"local", "cuvs"} or "Fields" not in existing_meta:+ return+ if not missing_fields and not missing_scalar_indexes:+ return+ if not hasattr(storage, "update_collection_schema"):+ raise EmbeddingConfigurationError(+ "Local context collection does not support automatic schema updates"+ )+ await storage.update_collection_schema(schema["Fields"], schema["ScalarIndex"])+base_description, existing_embedding_meta = _decode_collection_description(existing_meta.get("Description"))@@ -337,6 +355,7 @@ async def init_context_collection(storage) -> bool:)if _embedding_metadata_compatible(existing_embedding_meta, embedding_meta):+ await _update_local_schema()return Falseexisting_count = await storage.count() if hasattr(storage, "count") else 0@@ -348,6 +367,7 @@ async def init_context_collection(storage) -> bool:embedding_meta,))+ await _update_local_schema()return Falseif existing_embedding_meta is None:@@ -363,6 +383,7 @@ async def init_context_collection(storage) -> bool:embedding_meta,))+ await _update_local_schema()return Falseif existing_count == 0 and hasattr(storage, "update_collection_description"):@@ -372,6 +393,7 @@ async def init_context_collection(storage) -> bool:embedding_meta,))+ await _update_local_schema()return False# Embedding metadata differs from current config and the collection is@@ -407,6 +429,7 @@ async def init_context_collection(storage) -> bool:embedding_meta,))+ await _update_local_schema()return Falseif dimension_changed:diff --git a/openviking/storage/vectordb/collection/collection.py b/openviking/storage/vectordb/collection/collection.pyindex b4499d08b..9910c7061 100644--- a/openviking/storage/vectordb/collection/collection.py+++ b/openviking/storage/vectordb/collection/collection.py@@ -136,7 +136,7 @@ class ICollection(ABC):def update_index(self,index_name: str,- scalar_index: Optional[Dict[str, Any]] = None,+ scalar_index: Optional[List[str]] = None,description: Optional[str] = None,):raise NotImplementedError@@ -535,7 +535,7 @@ class Collection:def update_index(self,index_name: str,- scalar_index: Optional[Dict[str, Any]] = None,+ scalar_index: Optional[List[str]] = None,description: Optional[str] = None,):"""@@ -543,7 +543,7 @@ class Collection:Args:index_name (str): Name of the index to update.- scalar_index (Optional[Dict[str, Any]]): Updated configuration for scalar indexes.+ scalar_index (Optional[List[str]]): Complete list of scalar-indexed fields.Defaults to None.description (Optional[str]): New description for the index. Defaults to None."""diff --git a/openviking/storage/vectordb/collection/http_collection.py b/openviking/storage/vectordb/collection/http_collection.pyindex 2c9a85b86..7386a9c70 100644--- a/openviking/storage/vectordb/collection/http_collection.py+++ b/openviking/storage/vectordb/collection/http_collection.py@@ -217,7 +217,7 @@ class HttpCollection(ICollection):def update_index(self,index_name: str,- scalar_index: Optional[Dict[str, Any]] = None,+ scalar_index: Optional[List[str]] = None,description: Optional[str] = None,):data = {diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.pyindex 7bf80d019..b3ba5f136 100644--- a/openviking/storage/vectordb/collection/local_collection.py+++ b/openviking/storage/vectordb/collection/local_collection.py@@ -475,15 +475,21 @@ class LocalCollection(ICollection):def update_index(self,index_name: str,- scalar_index: Optional[Dict[str, Any]] = None,+ scalar_index: Optional[List[str]] = None,description: Optional[str] = None,) -> None:- with self._index_mutation_barrier.mutation() as mutation:+ with self._index_mutation_barrier.exclusive_mutation():index = self.indexes.get(index_name)if not index:return- index.update(scalar_index, description)- mutation.mark_changed()+ if scalar_index is not None:+ if not self.store_mgr:+ raise RuntimeError("Store manager is not initialized")+ index.rebuild_scalar_index(+ scalar_index, self.store_mgr.get_all_cands_data()+ )+ if description is not None:+ index.update(None, description)def get_index_meta_data(self, index_name: str) -> Optional[Dict[str, Any]]:index = self.indexes.get(index_name)diff --git a/openviking/storage/vectordb/collection/vikingdb_collection.py b/openviking/storage/vectordb/collection/vikingdb_collection.pyindex 966bc7c9e..b36bc00e9 100644--- a/openviking/storage/vectordb/collection/vikingdb_collection.py+++ b/openviking/storage/vectordb/collection/vikingdb_collection.py@@ -169,7 +169,7 @@ class VikingDBCollection(ICollection):def update_index(self,index_name: str,- scalar_index: Optional[Dict[str, Any]] = None,+ scalar_index: Optional[List[str]] = None,description: Optional[str] = None,):raise NotImplementedError("index should be managed manually")diff --git a/openviking/storage/vectordb/collection/volcengine_api_key_collection.py b/openviking/storage/vectordb/collection/volcengine_api_key_collection.pyindex 89ffbdfe4..7f5f6fbe5 100644--- a/openviking/storage/vectordb/collection/volcengine_api_key_collection.py+++ b/openviking/storage/vectordb/collection/volcengine_api_key_collection.py@@ -298,7 +298,7 @@ class VolcengineApiKeyCollection(ICollection):def update_index(self,index_name: str,- scalar_index: Optional[Dict[str, Any]] = None,+ scalar_index: Optional[List[str]] = None,description: Optional[str] = None,):raise NotImplementedError(diff --git a/openviking/storage/vectordb/collection/volcengine_collection.py b/openviking/storage/vectordb/collection/volcengine_collection.pyindex 20196d2bf..1338f0146 100644--- a/openviking/storage/vectordb/collection/volcengine_collection.py+++ b/openviking/storage/vectordb/collection/volcengine_collection.py@@ -363,7 +363,7 @@ class VolcengineCollection(ICollection):def update_index(self,index_name: str,- scalar_index: Optional[Dict[str, Any]] = None,+ scalar_index: Optional[List[str]] = None,description: Optional[str] = None,):data = {diff --git a/openviking/storage/vectordb/engine/_python_api.py b/openviking/storage/vectordb/engine/_python_api.pyindex d7c62dfc1..718892073 100644--- a/openviking/storage/vectordb/engine/_python_api.py+++ b/openviking/storage/vectordb/engine/_python_api.py@@ -474,6 +474,17 @@ def build_abi3_exports(backend: Any) -> dict[str, Any]:))+ def rebuild_scalar_index(+ self, scalar_index_json: str, data_list: list[AddDataRequest]+ ) -> int:+ return int(+ self._backend._index_engine_rebuild_scalar_index(+ self._handle,+ scalar_index_json,+ _request_list_to_backend(data_list),+ )+ )+def search(self, req: SearchRequest) -> SearchResult:return SearchResult.from_backend(self._backend._index_engine_search(self._handle, req))diff --git a/openviking/storage/vectordb/index/index.py b/openviking/storage/vectordb/index/index.pyindex a0499dbf8..1be3b64dc 100644--- a/openviking/storage/vectordb/index/index.py+++ b/openviking/storage/vectordb/index/index.py@@ -3,7 +3,7 @@from abc import ABC, abstractmethodfrom typing import Any, Dict, List, Optional, Tuple, Union-from openviking.storage.vectordb.store.data import DeltaRecord+from openviking.storage.vectordb.store.data import CandidateData, DeltaRecordclass IIndex(ABC):@@ -189,6 +189,13 @@ class IIndex(ABC):"""raise NotImplementedError+ @abstractmethod+ def rebuild_scalar_index(+ self, scalar_index: List[str], cands_list: List[CandidateData]+ ) -> None:+ """Replace the scalar index from a Store snapshot without rebuilding vectors."""+ raise NotImplementedError+@abstractmethoddef get_meta_data(self):"""Retrieve the complete metadata of the index.diff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.pyindex f33949099..e0e3fb293 100644--- a/openviking/storage/vectordb/index/local_index.py+++ b/openviking/storage/vectordb/index/local_index.py@@ -24,11 +24,12 @@ from openviking.storage.vectordb.index.index import IIndexfrom openviking.storage.vectordb.store.data import CandidateData, DeltaRecordfrom openviking.storage.vectordb.utils.constants import IndexFileMarkersfrom openviking.storage.vectordb.utils.data_processor import DataProcessor+from openviking.storage.vectordb.utils.json_safety import safe_json_dumpsfrom openviking.storage.vectordb.utils.path_safety import (safe_join,safe_join_name,)-from openviking.storage.vectordb.utils.validation import validate_name_str+from openviking.storage.vectordb.utils.validation import fix_fields_data, validate_name_strfrom openviking_cli.utils.logger import default_logger as logger_DENSE_REBUILD_MEMORY_RETRY_BASE_SECONDS = 1.0@@ -199,6 +200,22 @@ class IndexEngineProxy:add_req_list[i].fields_str = data.fieldsself.index_engine.add_data(add_req_list)+ def rebuild_scalar_index(+ self, scalar_index_meta: List[Dict[str, str]], cands_list: List[CandidateData]+ ) -> None:+ if not self.index_engine:+ raise RuntimeError("Index engine not initialized")++ requests = [engine.AddDataRequest() for _ in range(len(cands_list))]+ for request, data in zip(requests, cands_list, strict=False):+ request.label = data.label+ request.fields_str = data.fields+ result = self.index_engine.rebuild_scalar_index(+ json.dumps(scalar_index_meta), requests+ )+ if result != 0:+ raise RuntimeError("Failed to rebuild native scalar index")+def upsert_data(self, delta_list: List[DeltaRecord]):if not self.index_engine:raise RuntimeError("Index engine not initialized")@@ -465,6 +482,44 @@ class LocalIndex(IIndex):returnself.meta.update(meta_data)+ def rebuild_scalar_index(+ self, scalar_index: List[str], cands_list: List[CandidateData]+ ) -> None:+ if not self.engine_proxy:+ raise RuntimeError("Index engine not initialized")++ self.field_type_converter = DataProcessor(self.meta.collection_meta.fields_dict)+ converted_candidates: List[CandidateData] = []+ vector_key = self.meta.collection_meta.vector_key+ sparse_vector_key = self.meta.collection_meta.sparse_vector_key+ for candidate in cands_list:+ fields = fix_fields_data(+ json.loads(candidate.fields), self.meta.collection_meta.fields_dict+ )+ fields.pop(vector_key, None)+ if sparse_vector_key:+ fields.pop(sparse_vector_key, None)+ converted = CandidateData()+ converted.label = candidate.label+ converted.fields = safe_json_dumps(fields, ensure_ascii=False)+ converted_candidates.append(converted)++ engine_scalar_meta = self.field_type_converter.build_scalar_index_meta(scalar_index)+ self.engine_proxy.rebuild_scalar_index(+ engine_scalar_meta,+ self._convert_candidate_list_for_index(converted_candidates),+ )+ if isinstance(self, PersistentIndex) and self.persist() <= 0:+ raise RuntimeError("Failed to persist rebuilt scalar index")+ if not self.meta.update({"ScalarIndex": scalar_index}):+ raise RuntimeError("Failed to persist scalar index metadata")++ if self.dense_search is not None:+ self.dense_search.field_types = {+ name: DataProcessor.normalize_field_type(field_meta.get("FieldType", ""))+ for name, field_meta in self.meta.collection_meta.fields_dict.items()+ }+def get_meta_data(self):return self.meta.get_meta_data()diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex 2769b25be..4f1f20074 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -147,14 +147,51 @@ class _AsyncVectorAdapter:async def run(self, func: Any, /, *args: Any, **kwargs: Any) -> Any:return await asyncio.to_thread(func, *args, **kwargs)- async def collection_meta(self) -> Dict[str, Any]:- return await asyncio.to_thread(lambda: self._adapter.get_collection().get_meta_data() or {})+ async def collection_meta(self, index_name: str) -> Dict[str, Any]:+ def _get() -> Dict[str, Any]:+ collection = self._adapter.get_collection()+ meta = collection.get_meta_data() or {}+ if self._adapter.mode in {"local", "cuvs"}:+ index_meta = collection.get_index_meta_data(index_name) or {}+ if "ScalarIndex" in index_meta:+ meta["ScalarIndex"] = index_meta["ScalarIndex"]+ return meta++ return await asyncio.to_thread(_get)async def update_collection_description(self, description: str) -> None:await asyncio.to_thread(lambda: self._adapter.get_collection().update(description=description))+ async def update_collection_schema(+ self, fields: List[Dict[str, Any]], scalar_index: List[str], index_name: str+ ) -> None:+ def _update() -> None:+ collection = self._adapter.get_collection()+ existing_fields = {+ field.get("FieldName") for field in collection.get_meta_data().get("Fields", [])+ }+ missing_fields = [+ field for field in fields if field.get("FieldName") not in existing_fields+ ]+ if missing_fields:+ collection.update(fields=missing_fields)++ index_meta = collection.get_index_meta_data(index_name) or {}+ current_scalar_index = index_meta.get("ScalarIndex", [])+ indexed_fields = set(current_scalar_index)+ missing_scalar_fields = [+ field for field in scalar_index if field not in indexed_fields+ ]+ if missing_scalar_fields:+ collection.update_index(+ index_name,+ scalar_index=[*current_scalar_index, *missing_scalar_fields],+ )++ await asyncio.to_thread(_update)+class _SingleAccountBackend:"""绑定单个 account 的后端实现(内部类)"""@@ -274,7 +311,7 @@ class _SingleAccountBackend:return "not found" in message or "does not exist" in messageasync def _refresh_meta_data_async(self) -> None:- self._meta_data_cache = await self._async_adapter.collection_meta()+ self._meta_data_cache = await self._async_adapter.collection_meta(self._index_name)# =========================================================================# Collection Management@@ -340,7 +377,7 @@ class _SingleAccountBackend:async def get_collection_meta(self) -> Optional[Dict[str, Any]]:if not await self.collection_exists():return None- return await self._async_adapter.collection_meta()+ return await self._async_adapter.collection_meta(self._index_name)async def update_collection_description(self, description: str) -> bool:if not await self.collection_exists():@@ -349,6 +386,14 @@ class _SingleAccountBackend:await self._refresh_meta_data_async()return True+ async def update_collection_schema(+ self, fields: List[Dict[str, Any]], scalar_index: List[str]+ ) -> None:+ await self._async_adapter.update_collection_schema(+ fields, scalar_index, self._index_name+ )+ await self._refresh_meta_data_async()+# =========================================================================# Data Operations (with tenant enforcement)# =========================================================================@@ -987,6 +1032,15 @@ class VikingVectorIndexBackend:async def update_collection_description(self, description: str) -> bool:return await self._get_default_backend().update_collection_description(description)+ async def update_collection_schema(+ self, fields: List[Dict[str, Any]], scalar_index: List[str]+ ) -> None:+ default_backend = self._get_default_backend()+ await default_backend.update_collection_schema(fields, scalar_index)+ for backend in [*self._account_backends.values(), self._root_backend]:+ if backend is not None and backend is not default_backend:+ await backend._refresh_meta_data_async()+# =========================================================================# 公开数据操作 API(强制要求 ctx)# =========================================================================diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cppindex f88f09d09..4417e77a4 100644--- a/src/abi3_engine_backend.cpp+++ b/src/abi3_engine_backend.cpp@@ -1377,6 +1377,35 @@ PyObject* py_index_engine_delete_data(PyObject*, PyObject* args) {}}+PyObject* py_index_engine_rebuild_scalar_index(PyObject*, PyObject* args) {+ PyObject* capsule = nullptr;+ const char* scalar_index_json = nullptr;+ PyObject* items = nullptr;+ if (!PyArg_ParseTuple(args, "OsO", &capsule, &scalar_index_json, &items)) {+ return nullptr;+ }++ auto* engine = capsule_to_ptr<vdb::IndexEngine>(capsule, kIndexCapsuleName);+ if (engine == nullptr) {+ return nullptr;+ }++ std::vector<vdb::AddDataRequest> requests;+ if (!parse_request_list(items, parse_add_request, &requests)) {+ return nullptr;+ }++ try {+ const int result = call_without_gil([&]() {+ return engine->rebuild_scalar_index(scalar_index_json, requests);+ });+ return PyLong_FromLong(result);+ } catch (const std::exception& exc) {+ raise_runtime_error(exc.what());+ return nullptr;+ }+}+PyObject* py_index_engine_search(PyObject*, PyObject* args) {PyObject* capsule = nullptr;PyObject* request_obj = nullptr;@@ -1942,6 +1971,9 @@ PyMethodDef kModuleMethods[] = {"Add data to the index engine."},{"_index_engine_delete_data", py_index_engine_delete_data, METH_VARARGS,"Delete data from the index engine."},+ {"_index_engine_rebuild_scalar_index",+ py_index_engine_rebuild_scalar_index, METH_VARARGS,+ "Rebuild scalar fields without rebuilding the vector index."},{"_index_engine_search", py_index_engine_search, METH_VARARGS,"Search the index engine."},{"_index_engine_search_with_filter_token",diff --git a/src/index/detail/index_manager_impl.cpp b/src/index/detail/index_manager_impl.cppindex a693e62d9..553b02ba6 100644--- a/src/index/detail/index_manager_impl.cpp+++ b/src/index/detail/index_manager_impl.cpp@@ -623,6 +623,59 @@ int IndexManagerImpl::delete_data(return 0;}+int IndexManagerImpl::rebuild_scalar_index(+ const std::string& scalar_index_json,+ const std::vector<AddDataRequest>& data_list) {+ JsonDoc scalar_index_doc;+ scalar_index_doc.Parse(scalar_index_json.c_str());+ if (scalar_index_doc.HasParseError() || !scalar_index_doc.IsArray()) {+ throw std::invalid_argument("Invalid scalar index metadata");+ }++ auto next_meta = std::make_shared<ScalarIndexMeta>();+ if (next_meta->init_from_json(scalar_index_doc) != 0) {+ throw std::invalid_argument("Invalid scalar index metadata");+ }++ std::vector<FieldsDict> parsed_fields(data_list.size());+ for (size_t i = 0; i < data_list.size(); ++i) {+ if (parsed_fields[i].parse_from_json(data_list[i].fields_str) != 0) {+ throw std::runtime_error(+ "Failed to parse scalar fields for label=" ++ std::to_string(data_list[i].label));+ }+ }++ auto next_scalar_index = std::make_shared<ScalarIndex>(next_meta);+ std::unique_lock<std::shared_mutex> lock(rw_mutex_);+ for (size_t i = 0; i < data_list.size(); ++i) {+ const int offset = vector_index_->get_offset_by_label(data_list[i].label);+ if (offset < 0) {+ SPDLOG_WARN("IndexManagerImpl::rebuild_scalar_index label={} not found",+ data_list[i].label);+ continue;+ }+ if (next_scalar_index->add_row_data(offset, parsed_fields[i],+ FieldsDict{}) != 0) {+ throw std::runtime_error(+ "Failed to rebuild scalar fields for label=" ++ std::to_string(data_list[i].label));+ }+ }++ scalar_index_ = std::move(next_scalar_index);+ manager_meta_->scalar_index_meta = std::move(next_meta);+ register_label_offset_converter_();+ clear_filter_layout_();+ clear_filter_token_cache_();+ const auto now = std::chrono::duration_cast<std::chrono::nanoseconds>(+ std::chrono::system_clock::now().time_since_epoch())+ .count();+ manager_meta_->update_timestamp = std::max(+ manager_meta_->update_timestamp + 1, static_cast<uint64_t>(now));+ return 0;+}+int64_t IndexManagerImpl::dump(const std::string& dir) {std::filesystem::path dir_path(dir);std::shared_lock<std::shared_mutex> lock(rw_mutex_);diff --git a/src/index/detail/index_manager_impl.h b/src/index/detail/index_manager_impl.hindex 6180b0246..75778338b 100644--- a/src/index/detail/index_manager_impl.h+++ b/src/index/detail/index_manager_impl.h@@ -52,6 +52,10 @@ class IndexManagerImpl : public IndexManager {int delete_data(const std::vector<DeleteDataRequest>& data_list) override;+ int rebuild_scalar_index(+ const std::string& scalar_index_json,+ const std::vector<AddDataRequest>& data_list) override;+int64_t dump(const std::string& dir) override;int get_state(StateResult& state_result) override;diff --git a/src/index/index_engine.cpp b/src/index/index_engine.cppindex 0ebd15e1f..1712f1044 100644--- a/src/index/index_engine.cpp+++ b/src/index/index_engine.cpp@@ -70,6 +70,12 @@ int IndexEngine::delete_data(const std::vector<DeleteDataRequest>& data_list) {return impl_->delete_data(data_list);}+int IndexEngine::rebuild_scalar_index(+ const std::string& scalar_index_json,+ const std::vector<AddDataRequest>& data_list) {+ return impl_->rebuild_scalar_index(scalar_index_json, data_list);+}+int64_t IndexEngine::dump(const std::string& dir) {return impl_->dump(dir);}diff --git a/src/index/index_engine.h b/src/index/index_engine.hindex 1c82a9244..e177afbdb 100644--- a/src/index/index_engine.h+++ b/src/index/index_engine.h@@ -23,6 +23,10 @@ class IndexEngine {int delete_data(const std::vector<DeleteDataRequest>& data_list);+ int rebuild_scalar_index(+ const std::string& scalar_index_json,+ const std::vector<AddDataRequest>& data_list);+SearchResult search(const SearchRequest& req);std::optional<SearchResult> search_with_filter_token(diff --git a/src/index/index_manager.h b/src/index/index_manager.hindex 9b6e1f92f..344241d4c 100644--- a/src/index/index_manager.h+++ b/src/index/index_manager.h@@ -36,6 +36,10 @@ class IndexManager {virtual int delete_data(const std::vector<DeleteDataRequest>& data_list) = 0;+ virtual int rebuild_scalar_index(+ const std::string& scalar_index_json,+ const std::vector<AddDataRequest>& data_list) = 0;+virtual int64_t dump(const std::string& dir) = 0;virtual int get_state(StateResult& state_result) = 0;diff --git a/tests/storage/test_collection_schemas.py b/tests/storage/test_collection_schemas.pyindex b0a0d09cc..29a763e12 100644--- a/tests/storage/test_collection_schemas.py+++ b/tests/storage/test_collection_schemas.py@@ -186,6 +186,15 @@ async def test_init_context_collection_writes_embedding_metadata(monkeypatch):@pytest.mark.asyncioasync def test_init_context_collection_backfills_metadata_for_empty_legacy_collection(monkeypatch):updates = []+ schema_updates = []+ config = _DummyConfig(_DummyEmbedder(), backend="local")+ existing_schema = CollectionSchemas.context_collection("context", config.embedding.dimension)+ existing_fields = [+ field for field in existing_schema["Fields"] if field["FieldName"] != "tags"+ ]+ existing_scalar_index = [+ field for field in existing_schema["ScalarIndex"] if field != "tags"+ ]class _FakeStorage:async def create_collection(self, name, schema):@@ -193,7 +202,11 @@ async def test_init_context_collection_backfills_metadata_for_empty_legacy_collereturn Falseasync def get_collection_meta(self):- return {"Description": "Unified context collection"}+ return {+ "Description": "Unified context collection",+ "Fields": existing_fields,+ "ScalarIndex": existing_scalar_index,+ }async def count(self):return 0@@ -202,7 +215,9 @@ async def test_init_context_collection_backfills_metadata_for_empty_legacy_colleupdates.append(description)return True- config = _DummyConfig(_DummyEmbedder())+ async def update_collection_schema(self, fields, scalar_index):+ schema_updates.append((fields, scalar_index))+monkeypatch.setattr("openviking_cli.utils.config.get_openviking_config",lambda: config,@@ -213,6 +228,12 @@ async def test_init_context_collection_backfills_metadata_for_empty_legacy_colleassert created is Falseassert len(updates) == 1assert '"provider": "local"' in updates[0]+ assert len(schema_updates) == 1+ fields, scalar_index = schema_updates[0]+ assert {field["FieldName"] for field in fields} - {+ field["FieldName"] for field in existing_fields+ } == {"tags"}+ assert set(scalar_index) - set(existing_scalar_index) == {"tags"}@pytest.mark.asynciodiff --git a/tests/vectordb/test_engine_filter_packed_native_abi.py b/tests/vectordb/test_engine_filter_packed_native_abi.pyindex 426b957b2..dde81d9b3 100644--- a/tests/vectordb/test_engine_filter_packed_native_abi.py+++ b/tests/vectordb/test_engine_filter_packed_native_abi.py@@ -75,3 +75,23 @@ def test_native_packed_filter_abi_has_exact_little_endian_layout_and_shapes():assert routed_wide.native_filter_token == 0assert empty.bitset_words == b"\x00" * 12assert empty.eligible_count == 0++ for row, request in enumerate(requests):+ request.fields_str = json.dumps(+ {+ "uri": f"/keep/item-{row}" if row in selected_rows else f"/drop/item-{row}",+ "schema_flag": row in selected_rows,+ }+ )+ scalar_meta = [+ {"FieldName": "uri", "FieldType": "path"},+ {"FieldName": "schema_flag", "FieldType": "bool"},+ ]+ assert index.rebuild_scalar_index(json.dumps(scalar_meta), requests) == 0+ assert index.set_filter_layout(labels) == 0+ rebuilt = index.evaluate_filter(+ json.dumps({"op": "must", "field": "schema_flag", "conds": [True]})+ )+ assert rebuilt.bitset_words == expected_words+ assert rebuilt.eligible_count == len(selected_rows)+ assert index.evaluate_filter(keep_filter).bitset_words == expected_wordsdiff --git a/tests/vectordb/test_openviking_vectordb.py b/tests/vectordb/test_openviking_vectordb.pyindex dae80c7fc..b75f80882 100644--- a/tests/vectordb/test_openviking_vectordb.py+++ b/tests/vectordb/test_openviking_vectordb.py@@ -4,6 +4,7 @@import jsonimport shutilimport unittest+from pathlib import Pathfrom unittest.mock import patchfrom openviking.storage.vectordb.collection.local_collection import get_or_create_local_collection@@ -254,6 +255,28 @@ class TestOpenVikingVectorDB(unittest.TestCase):index_meta = collection.get_index_meta_data("idx_filters") or {}self.assertIn("type", index_meta.get("ScalarIndex", []))+ index = collection.get_index("idx_filters")+ previous_version = index.get_newest_version()+ collection.update(+ fields=[+ {"FieldName": "schema_flag", "FieldType": "bool", "DefaultValue": False}+ ]+ )+ collection.update_index(+ "idx_filters",+ scalar_index=[*index_meta["ScalarIndex"], "schema_flag"],+ )+ current_version = index.get_newest_version()+ self.assertGreater(current_version, previous_version)+ self.assertTrue(Path(index.version_dir, str(previous_version)).is_dir())+ self.assertTrue(Path(f"{index.version_dir}/{previous_version}.write_done").is_file())+ self.assertEqual(+ self._search_ids(+ collection,+ {"op": "must", "field": "schema_flag", "conds": [False]},+ ),+ sorted(item["id"] for item in self.data),+ )fetched = collection.fetch_data(["res_1"])self.assertEqual(fetched.items[0].fields.get("type"), "file")