Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(vectordb): bound memory during local schema upgrades (#4819)
Qin Haojiea58f0f5ec6c1
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.pyindex b3ba5f136..9385504f5 100644--- a/openviking/storage/vectordb/collection/local_collection.py+++ b/openviking/storage/vectordb/collection/local_collection.py@@ -485,9 +485,7 @@ class LocalCollection(ICollection):if scalar_index is not None:if not self.store_mgr:raise RuntimeError("Store manager is not initialized")- index.rebuild_scalar_index(- scalar_index, self.store_mgr.get_all_cands_data()- )+ index.rebuild_scalar_index(scalar_index, self.store_mgr.iter_all_cands_fields())if description is not None:index.update(None, description)diff --git a/openviking/storage/vectordb/engine/_python_api.py b/openviking/storage/vectordb/engine/_python_api.pyindex 718892073..866219c6b 100644--- a/openviking/storage/vectordb/engine/_python_api.py+++ b/openviking/storage/vectordb/engine/_python_api.py@@ -475,13 +475,13 @@ def build_abi3_exports(backend: Any) -> dict[str, Any]:)def rebuild_scalar_index(- self, scalar_index_json: str, data_list: list[AddDataRequest]+ self, scalar_index_json: str, data_list: Iterable[AddDataRequest]) -> int:return int(self._backend._index_engine_rebuild_scalar_index(self._handle,scalar_index_json,- _request_list_to_backend(data_list),+ data_list,))diff --git a/openviking/storage/vectordb/index/index.py b/openviking/storage/vectordb/index/index.pyindex 1be3b64dc..07dc39c7e 100644--- a/openviking/storage/vectordb/index/index.py+++ b/openviking/storage/vectordb/index/index.py@@ -1,9 +1,9 @@# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.# SPDX-License-Identifier: AGPL-3.0from abc import ABC, abstractmethod-from typing import Any, Dict, List, Optional, Tuple, Union+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union-from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord+from openviking.storage.vectordb.store.data import DeltaRecordclass IIndex(ABC):@@ -191,7 +191,7 @@ class IIndex(ABC):@abstractmethoddef rebuild_scalar_index(- self, scalar_index: List[str], cands_list: List[CandidateData]+ self, scalar_index: List[str], cands_fields: Iterable[Tuple[int, str]]) -> None:"""Replace the scalar index from a Store snapshot without rebuilding vectors."""raise NotImplementedErrordiff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.pyindex e0e3fb293..6b4c8adfa 100644--- a/openviking/storage/vectordb/index/local_index.py+++ b/openviking/storage/vectordb/index/local_index.py@@ -201,18 +201,12 @@ class IndexEngineProxy:self.index_engine.add_data(add_req_list)def rebuild_scalar_index(- self, scalar_index_meta: List[Dict[str, str]], cands_list: List[CandidateData]+ self, scalar_index_meta: List[Dict[str, str]], requests: Iterable[engine.AddDataRequest]) -> None:if not self.index_engine:raise RuntimeError("Index engine not initialized")- requests = [engine.AddDataRequest() for _ in range(len(cands_list))]- for request, data in zip(requests, cands_list, strict=False):- request.label = data.label- request.fields_str = data.fields- result = self.index_engine.rebuild_scalar_index(- json.dumps(scalar_index_meta), requests- )+ result = self.index_engine.rebuild_scalar_index(json.dumps(scalar_index_meta), requests)if result != 0:raise RuntimeError("Failed to rebuild native scalar index")@@ -483,32 +477,31 @@ class LocalIndex(IIndex):self.meta.update(meta_data)def rebuild_scalar_index(- self, scalar_index: List[str], cands_list: List[CandidateData]+ self, scalar_index: List[str], cands_fields: Iterable[Tuple[int, str]]) -> None:if not self.engine_proxy:raise RuntimeError("Index engine not initialized")self.field_type_converter = DataProcessor(self.meta.collection_meta.fields_dict)- converted_candidates: List[CandidateData] = []- vector_key = self.meta.collection_meta.vector_key- sparse_vector_key = self.meta.collection_meta.sparse_vector_key- for candidate in cands_list:- fields = fix_fields_data(- json.loads(candidate.fields), self.meta.collection_meta.fields_dict- )- fields.pop(vector_key, None)- if sparse_vector_key:- fields.pop(sparse_vector_key, None)- converted = CandidateData()- converted.label = candidate.label- converted.fields = safe_json_dumps(fields, ensure_ascii=False)- converted_candidates.append(converted)+ indexed_fields = {+ name: self.meta.collection_meta.fields_dict[name] for name in scalar_index+ }++ def requests() -> Iterator[engine.AddDataRequest]:+ for label, fields_json in cands_fields:+ fields = json.loads(fields_json)+ fields = {name: fields[name] for name in indexed_fields if name in fields}+ fix_fields_data(fields, indexed_fields)+ request = engine.AddDataRequest()+ request.label = label+ request.fields_str = safe_json_dumps(+ self.field_type_converter.convert_fields_dict_for_index(fields),+ ensure_ascii=False,+ )+ yield requestengine_scalar_meta = self.field_type_converter.build_scalar_index_meta(scalar_index)- self.engine_proxy.rebuild_scalar_index(- engine_scalar_meta,- self._convert_candidate_list_for_index(converted_candidates),- )+ self.engine_proxy.rebuild_scalar_index(engine_scalar_meta, requests())if isinstance(self, PersistentIndex) and self.persist() <= 0:raise RuntimeError("Failed to persist rebuilt scalar index")if not self.meta.update({"ScalarIndex": scalar_index}):diff --git a/openviking/storage/vectordb/store/store_manager.py b/openviking/storage/vectordb/store/store_manager.pyindex 10c6b4c3a..b27dafac9 100644--- a/openviking/storage/vectordb/store/store_manager.py+++ b/openviking/storage/vectordb/store/store_manager.py@@ -227,6 +227,14 @@ class StoreManager:"""return list(self.iter_all_cands_data())+ def iter_all_cands_fields(self) -> Iterator[Tuple[int, str]]:+ """Scan labels and scalar fields without decoding stored vectors."""+ for _, bytes_data in self.storage.iter_all(StoreManager.CandsTable):+ yield (+ CandidateData.bytes_row.deserialize_field(bytes_data, "label"),+ CandidateData.bytes_row.deserialize_field(bytes_data, "fields"),+ )+def iter_all_cands_data(self) -> Iterator[CandidateData]:"""Iterate candidates without retaining deserialized full tables.diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cppindex 4417e77a4..0b366b722 100644--- a/src/abi3_engine_backend.cpp+++ b/src/abi3_engine_backend.cpp@@ -1390,18 +1390,53 @@ PyObject* py_index_engine_rebuild_scalar_index(PyObject*, PyObject* args) {return nullptr;}- std::vector<vdb::AddDataRequest> requests;- if (!parse_request_list(items, parse_add_request, &requests)) {+ std::unique_ptr<PyObject, decltype(&Py_DecRef)> iterator(PyObject_GetIter(items),+ Py_DecRef);+ if (!iterator) {return nullptr;}try {+ // Bound the native input independently of the Store's encoded page size.+ // A single oversized row is allowed so every nonempty batch makes progress.+ auto read_batch = [&](std::vector<vdb::AddDataRequest>& batch) {+ constexpr size_t kMaxRows = 1024;+ constexpr size_t kMaxBytes = 1024 * 1024;+ const auto gil = PyGILState_Ensure();+ try {+ batch.clear();+ size_t bytes = 0;+ while (batch.size() < kMaxRows && bytes < kMaxBytes) {+ std::unique_ptr<PyObject, decltype(&Py_DecRef)> item(+ PyIter_Next(iterator.get()), Py_DecRef);+ if (!item) {+ if (PyErr_Occurred()) {+ throw std::runtime_error("Failed to read scalar index rows");+ }+ break;+ }+ vdb::AddDataRequest request;+ if (!parse_add_request(item.get(), &request)) {+ throw std::runtime_error("Invalid scalar index row");+ }+ bytes += request.fields_str.size();+ batch.push_back(std::move(request));+ }+ } catch (...) {+ PyGILState_Release(gil);+ throw;+ }+ PyGILState_Release(gil);+ return !batch.empty();+ };const int result = call_without_gil([&]() {- return engine->rebuild_scalar_index(scalar_index_json, requests);+ return engine->rebuild_scalar_index(scalar_index_json, read_batch);});return PyLong_FromLong(result);} catch (const std::exception& exc) {- raise_runtime_error(exc.what());+ if (!PyErr_Occurred()) {+ raise_runtime_error(exc.what());+ }return nullptr;}}diff --git a/src/index/detail/index_manager_impl.cpp b/src/index/detail/index_manager_impl.cppindex 553b02ba6..5b67dc440 100644--- a/src/index/detail/index_manager_impl.cpp+++ b/src/index/detail/index_manager_impl.cpp@@ -625,7 +625,7 @@ int IndexManagerImpl::delete_data(int IndexManagerImpl::rebuild_scalar_index(const std::string& scalar_index_json,- const std::vector<AddDataRequest>& data_list) {+ const std::function<bool(std::vector<AddDataRequest>&)>& read_batch) {JsonDoc scalar_index_doc;scalar_index_doc.Parse(scalar_index_json.c_str());if (scalar_index_doc.HasParseError() || !scalar_index_doc.IsArray()) {@@ -637,30 +637,30 @@ int IndexManagerImpl::rebuild_scalar_index(throw std::invalid_argument("Invalid scalar index metadata");}- std::vector<FieldsDict> parsed_fields(data_list.size());- for (size_t i = 0; i < data_list.size(); ++i) {- if (parsed_fields[i].parse_from_json(data_list[i].fields_str) != 0) {- throw std::runtime_error(- "Failed to parse scalar fields for label=" +- std::to_string(data_list[i].label));- }- }-auto next_scalar_index = std::make_shared<ScalarIndex>(next_meta);std::unique_lock<std::shared_mutex> lock(rw_mutex_);- for (size_t i = 0; i < data_list.size(); ++i) {- const int offset = vector_index_->get_offset_by_label(data_list[i].label);- if (offset < 0) {- SPDLOG_WARN("IndexManagerImpl::rebuild_scalar_index label={} not found",- data_list[i].label);- continue;- }- if (next_scalar_index->add_row_data(offset, parsed_fields[i],- FieldsDict{}) != 0) {- throw std::runtime_error(- "Failed to rebuild scalar fields for label=" +- std::to_string(data_list[i].label));+ std::vector<AddDataRequest> batch;+ while (read_batch(batch)) {+ for (const auto& data : batch) {+ FieldsDict fields;+ if (fields.parse_from_json(data.fields_str) != 0) {+ throw std::runtime_error(+ "Failed to parse scalar fields for label=" ++ std::to_string(data.label));+ }+ const int offset = vector_index_->get_offset_by_label(data.label);+ if (offset < 0) {+ SPDLOG_WARN("IndexManagerImpl::rebuild_scalar_index label={} not found",+ data.label);+ continue;+ }+ if (next_scalar_index->add_row_data(offset, fields, FieldsDict{}) != 0) {+ throw std::runtime_error(+ "Failed to rebuild scalar fields for label=" ++ std::to_string(data.label));+ }}+ batch.clear();}scalar_index_ = std::move(next_scalar_index);diff --git a/src/index/detail/index_manager_impl.h b/src/index/detail/index_manager_impl.hindex 75778338b..578f4e990 100644--- a/src/index/detail/index_manager_impl.h+++ b/src/index/detail/index_manager_impl.h@@ -54,7 +54,7 @@ class IndexManagerImpl : public IndexManager {int rebuild_scalar_index(const std::string& scalar_index_json,- const std::vector<AddDataRequest>& data_list) override;+ const std::function<bool(std::vector<AddDataRequest>&)>& read_batch) override;int64_t dump(const std::string& dir) override;diff --git a/src/index/index_engine.cpp b/src/index/index_engine.cppindex 1712f1044..2db71d1f0 100644--- a/src/index/index_engine.cpp+++ b/src/index/index_engine.cpp@@ -72,8 +72,8 @@ int IndexEngine::delete_data(const std::vector<DeleteDataRequest>& data_list) {int IndexEngine::rebuild_scalar_index(const std::string& scalar_index_json,- const std::vector<AddDataRequest>& data_list) {- return impl_->rebuild_scalar_index(scalar_index_json, data_list);+ const std::function<bool(std::vector<AddDataRequest>&)>& read_batch) {+ return impl_->rebuild_scalar_index(scalar_index_json, read_batch);}int64_t IndexEngine::dump(const std::string& dir) {diff --git a/src/index/index_engine.h b/src/index/index_engine.hindex e177afbdb..f363e51fd 100644--- a/src/index/index_engine.h+++ b/src/index/index_engine.h@@ -25,7 +25,7 @@ class IndexEngine {int rebuild_scalar_index(const std::string& scalar_index_json,- const std::vector<AddDataRequest>& data_list);+ const std::function<bool(std::vector<AddDataRequest>&)>& read_batch);SearchResult search(const SearchRequest& req);diff --git a/src/index/index_manager.h b/src/index/index_manager.hindex 344241d4c..72b7dfa64 100644--- a/src/index/index_manager.h+++ b/src/index/index_manager.h@@ -1,6 +1,7 @@// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.// SPDX-License-Identifier: AGPL-3.0#pragma once+#include <functional>#include <string>#include <vector>#include "index/common_structs.h"@@ -38,7 +39,7 @@ class IndexManager {virtual int rebuild_scalar_index(const std::string& scalar_index_json,- const std::vector<AddDataRequest>& data_list) = 0;+ const std::function<bool(std::vector<AddDataRequest>&)>& read_batch) = 0;virtual int64_t dump(const std::string& dir) = 0;diff --git a/tests/vectordb/test_engine_filter_packed_native_abi.py b/tests/vectordb/test_engine_filter_packed_native_abi.pyindex dde81d9b3..4af24bead 100644--- a/tests/vectordb/test_engine_filter_packed_native_abi.py+++ b/tests/vectordb/test_engine_filter_packed_native_abi.py@@ -87,7 +87,21 @@ def test_native_packed_filter_abi_has_exact_little_endian_layout_and_shapes():{"FieldName": "uri", "FieldType": "path"},{"FieldName": "schema_flag", "FieldType": "bool"},]- assert index.rebuild_scalar_index(json.dumps(scalar_meta), requests) == 0++ def interrupted_rows():+ for row in range(1100):+ request = engine.AddDataRequest()+ request.label = labels[row % len(labels)]+ request.fields_str = '{"uri":"/replacement","schema_flag":true}'+ yield request+ raise ValueError("Store scan interrupted")++ # Even after a batch has been built, a failed scan must keep the old index.+ with pytest.raises(ValueError, match="Store scan interrupted"):+ index.rebuild_scalar_index(json.dumps(scalar_meta), interrupted_rows())+ assert index.evaluate_filter(keep_filter).bitset_words == expected_words++ assert index.rebuild_scalar_index(json.dumps(scalar_meta), iter(requests)) == 0assert index.set_filter_layout(labels) == 0rebuilt = index.evaluate_filter(json.dumps({"op": "must", "field": "schema_flag", "conds": [True]})