Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(vectordb): bound memory during local schema upgrades (#4819)

Qin Haojiea58f0f5ec6c1
diff --git a/openviking/storage/vectordb/collection/local_collection.py b/openviking/storage/vectordb/collection/local_collection.py
index b3ba5f136..9385504f5 100644
--- a/openviking/storage/vectordb/collection/local_collection.py
+++ b/openviking/storage/vectordb/collection/local_collection.py
@@ -485,9 +485,7 @@ class LocalCollection(ICollection):
if scalar_index is not None:
if not self.store_mgr:
raise RuntimeError("Store manager is not initialized")
- index.rebuild_scalar_index(
- scalar_index, self.store_mgr.get_all_cands_data()
- )
+ index.rebuild_scalar_index(scalar_index, self.store_mgr.iter_all_cands_fields())
if description is not None:
index.update(None, description)
diff --git a/openviking/storage/vectordb/engine/_python_api.py b/openviking/storage/vectordb/engine/_python_api.py
index 718892073..866219c6b 100644
--- a/openviking/storage/vectordb/engine/_python_api.py
+++ b/openviking/storage/vectordb/engine/_python_api.py
@@ -475,13 +475,13 @@ def build_abi3_exports(backend: Any) -> dict[str, Any]:
)
def rebuild_scalar_index(
- self, scalar_index_json: str, data_list: list[AddDataRequest]
+ self, scalar_index_json: str, data_list: Iterable[AddDataRequest]
) -> int:
return int(
self._backend._index_engine_rebuild_scalar_index(
self._handle,
scalar_index_json,
- _request_list_to_backend(data_list),
+ data_list,
)
)
diff --git a/openviking/storage/vectordb/index/index.py b/openviking/storage/vectordb/index/index.py
index 1be3b64dc..07dc39c7e 100644
--- a/openviking/storage/vectordb/index/index.py
+++ b/openviking/storage/vectordb/index/index.py
@@ -1,9 +1,9 @@
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
from abc import ABC, abstractmethod
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Dict, Iterable, List, Optional, Tuple, Union
-from openviking.storage.vectordb.store.data import CandidateData, DeltaRecord
+from openviking.storage.vectordb.store.data import DeltaRecord
class IIndex(ABC):
@@ -191,7 +191,7 @@ class IIndex(ABC):
@abstractmethod
def rebuild_scalar_index(
- self, scalar_index: List[str], cands_list: List[CandidateData]
+ self, scalar_index: List[str], cands_fields: Iterable[Tuple[int, str]]
) -> None:
"""Replace the scalar index from a Store snapshot without rebuilding vectors."""
raise NotImplementedError
diff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.py
index e0e3fb293..6b4c8adfa 100644
--- a/openviking/storage/vectordb/index/local_index.py
+++ b/openviking/storage/vectordb/index/local_index.py
@@ -201,18 +201,12 @@ class IndexEngineProxy:
self.index_engine.add_data(add_req_list)
def rebuild_scalar_index(
- self, scalar_index_meta: List[Dict[str, str]], cands_list: List[CandidateData]
+ self, scalar_index_meta: List[Dict[str, str]], requests: Iterable[engine.AddDataRequest]
) -> None:
if not self.index_engine:
raise RuntimeError("Index engine not initialized")
- requests = [engine.AddDataRequest() for _ in range(len(cands_list))]
- for request, data in zip(requests, cands_list, strict=False):
- request.label = data.label
- request.fields_str = data.fields
- result = self.index_engine.rebuild_scalar_index(
- json.dumps(scalar_index_meta), requests
- )
+ result = self.index_engine.rebuild_scalar_index(json.dumps(scalar_index_meta), requests)
if result != 0:
raise RuntimeError("Failed to rebuild native scalar index")
@@ -483,32 +477,31 @@ class LocalIndex(IIndex):
self.meta.update(meta_data)
def rebuild_scalar_index(
- self, scalar_index: List[str], cands_list: List[CandidateData]
+ self, scalar_index: List[str], cands_fields: Iterable[Tuple[int, str]]
) -> None:
if not self.engine_proxy:
raise RuntimeError("Index engine not initialized")
self.field_type_converter = DataProcessor(self.meta.collection_meta.fields_dict)
- converted_candidates: List[CandidateData] = []
- vector_key = self.meta.collection_meta.vector_key
- sparse_vector_key = self.meta.collection_meta.sparse_vector_key
- for candidate in cands_list:
- fields = fix_fields_data(
- json.loads(candidate.fields), self.meta.collection_meta.fields_dict
- )
- fields.pop(vector_key, None)
- if sparse_vector_key:
- fields.pop(sparse_vector_key, None)
- converted = CandidateData()
- converted.label = candidate.label
- converted.fields = safe_json_dumps(fields, ensure_ascii=False)
- converted_candidates.append(converted)
+ indexed_fields = {
+ name: self.meta.collection_meta.fields_dict[name] for name in scalar_index
+ }
+
+ def requests() -> Iterator[engine.AddDataRequest]:
+ for label, fields_json in cands_fields:
+ fields = json.loads(fields_json)
+ fields = {name: fields[name] for name in indexed_fields if name in fields}
+ fix_fields_data(fields, indexed_fields)
+ request = engine.AddDataRequest()
+ request.label = label
+ request.fields_str = safe_json_dumps(
+ self.field_type_converter.convert_fields_dict_for_index(fields),
+ ensure_ascii=False,
+ )
+ yield request
engine_scalar_meta = self.field_type_converter.build_scalar_index_meta(scalar_index)
- self.engine_proxy.rebuild_scalar_index(
- engine_scalar_meta,
- self._convert_candidate_list_for_index(converted_candidates),
- )
+ self.engine_proxy.rebuild_scalar_index(engine_scalar_meta, requests())
if isinstance(self, PersistentIndex) and self.persist() <= 0:
raise RuntimeError("Failed to persist rebuilt scalar index")
if not self.meta.update({"ScalarIndex": scalar_index}):
diff --git a/openviking/storage/vectordb/store/store_manager.py b/openviking/storage/vectordb/store/store_manager.py
index 10c6b4c3a..b27dafac9 100644
--- a/openviking/storage/vectordb/store/store_manager.py
+++ b/openviking/storage/vectordb/store/store_manager.py
@@ -227,6 +227,14 @@ class StoreManager:
"""
return list(self.iter_all_cands_data())
+ def iter_all_cands_fields(self) -> Iterator[Tuple[int, str]]:
+ """Scan labels and scalar fields without decoding stored vectors."""
+ for _, bytes_data in self.storage.iter_all(StoreManager.CandsTable):
+ yield (
+ CandidateData.bytes_row.deserialize_field(bytes_data, "label"),
+ CandidateData.bytes_row.deserialize_field(bytes_data, "fields"),
+ )
+
def iter_all_cands_data(self) -> Iterator[CandidateData]:
"""Iterate candidates without retaining deserialized full tables.
diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cpp
index 4417e77a4..0b366b722 100644
--- a/src/abi3_engine_backend.cpp
+++ b/src/abi3_engine_backend.cpp
@@ -1390,18 +1390,53 @@ PyObject* py_index_engine_rebuild_scalar_index(PyObject*, PyObject* args) {
return nullptr;
}
- std::vector<vdb::AddDataRequest> requests;
- if (!parse_request_list(items, parse_add_request, &requests)) {
+ std::unique_ptr<PyObject, decltype(&Py_DecRef)> iterator(PyObject_GetIter(items),
+ Py_DecRef);
+ if (!iterator) {
return nullptr;
}
try {
+ // Bound the native input independently of the Store's encoded page size.
+ // A single oversized row is allowed so every nonempty batch makes progress.
+ auto read_batch = [&](std::vector<vdb::AddDataRequest>& batch) {
+ constexpr size_t kMaxRows = 1024;
+ constexpr size_t kMaxBytes = 1024 * 1024;
+ const auto gil = PyGILState_Ensure();
+ try {
+ batch.clear();
+ size_t bytes = 0;
+ while (batch.size() < kMaxRows && bytes < kMaxBytes) {
+ std::unique_ptr<PyObject, decltype(&Py_DecRef)> item(
+ PyIter_Next(iterator.get()), Py_DecRef);
+ if (!item) {
+ if (PyErr_Occurred()) {
+ throw std::runtime_error("Failed to read scalar index rows");
+ }
+ break;
+ }
+ vdb::AddDataRequest request;
+ if (!parse_add_request(item.get(), &request)) {
+ throw std::runtime_error("Invalid scalar index row");
+ }
+ bytes += request.fields_str.size();
+ batch.push_back(std::move(request));
+ }
+ } catch (...) {
+ PyGILState_Release(gil);
+ throw;
+ }
+ PyGILState_Release(gil);
+ return !batch.empty();
+ };
const int result = call_without_gil([&]() {
- return engine->rebuild_scalar_index(scalar_index_json, requests);
+ return engine->rebuild_scalar_index(scalar_index_json, read_batch);
});
return PyLong_FromLong(result);
} catch (const std::exception& exc) {
- raise_runtime_error(exc.what());
+ if (!PyErr_Occurred()) {
+ raise_runtime_error(exc.what());
+ }
return nullptr;
}
}
diff --git a/src/index/detail/index_manager_impl.cpp b/src/index/detail/index_manager_impl.cpp
index 553b02ba6..5b67dc440 100644
--- a/src/index/detail/index_manager_impl.cpp
+++ b/src/index/detail/index_manager_impl.cpp
@@ -625,7 +625,7 @@ int IndexManagerImpl::delete_data(
int IndexManagerImpl::rebuild_scalar_index(
const std::string& scalar_index_json,
- const std::vector<AddDataRequest>& data_list) {
+ const std::function<bool(std::vector<AddDataRequest>&)>& read_batch) {
JsonDoc scalar_index_doc;
scalar_index_doc.Parse(scalar_index_json.c_str());
if (scalar_index_doc.HasParseError() || !scalar_index_doc.IsArray()) {
@@ -637,30 +637,30 @@ int IndexManagerImpl::rebuild_scalar_index(
throw std::invalid_argument("Invalid scalar index metadata");
}
- std::vector<FieldsDict> parsed_fields(data_list.size());
- for (size_t i = 0; i < data_list.size(); ++i) {
- if (parsed_fields[i].parse_from_json(data_list[i].fields_str) != 0) {
- throw std::runtime_error(
- "Failed to parse scalar fields for label=" +
- std::to_string(data_list[i].label));
- }
- }
-
auto next_scalar_index = std::make_shared<ScalarIndex>(next_meta);
std::unique_lock<std::shared_mutex> lock(rw_mutex_);
- for (size_t i = 0; i < data_list.size(); ++i) {
- const int offset = vector_index_->get_offset_by_label(data_list[i].label);
- if (offset < 0) {
- SPDLOG_WARN("IndexManagerImpl::rebuild_scalar_index label={} not found",
- data_list[i].label);
- continue;
- }
- if (next_scalar_index->add_row_data(offset, parsed_fields[i],
- FieldsDict{}) != 0) {
- throw std::runtime_error(
- "Failed to rebuild scalar fields for label=" +
- std::to_string(data_list[i].label));
+ std::vector<AddDataRequest> batch;
+ while (read_batch(batch)) {
+ for (const auto& data : batch) {
+ FieldsDict fields;
+ if (fields.parse_from_json(data.fields_str) != 0) {
+ throw std::runtime_error(
+ "Failed to parse scalar fields for label=" +
+ std::to_string(data.label));
+ }
+ const int offset = vector_index_->get_offset_by_label(data.label);
+ if (offset < 0) {
+ SPDLOG_WARN("IndexManagerImpl::rebuild_scalar_index label={} not found",
+ data.label);
+ continue;
+ }
+ if (next_scalar_index->add_row_data(offset, fields, FieldsDict{}) != 0) {
+ throw std::runtime_error(
+ "Failed to rebuild scalar fields for label=" +
+ std::to_string(data.label));
+ }
}
+ batch.clear();
}
scalar_index_ = std::move(next_scalar_index);
diff --git a/src/index/detail/index_manager_impl.h b/src/index/detail/index_manager_impl.h
index 75778338b..578f4e990 100644
--- a/src/index/detail/index_manager_impl.h
+++ b/src/index/detail/index_manager_impl.h
@@ -54,7 +54,7 @@ class IndexManagerImpl : public IndexManager {
int rebuild_scalar_index(
const std::string& scalar_index_json,
- const std::vector<AddDataRequest>& data_list) override;
+ const std::function<bool(std::vector<AddDataRequest>&)>& read_batch) override;
int64_t dump(const std::string& dir) override;
diff --git a/src/index/index_engine.cpp b/src/index/index_engine.cpp
index 1712f1044..2db71d1f0 100644
--- a/src/index/index_engine.cpp
+++ b/src/index/index_engine.cpp
@@ -72,8 +72,8 @@ int IndexEngine::delete_data(const std::vector<DeleteDataRequest>& data_list) {
int IndexEngine::rebuild_scalar_index(
const std::string& scalar_index_json,
- const std::vector<AddDataRequest>& data_list) {
- return impl_->rebuild_scalar_index(scalar_index_json, data_list);
+ const std::function<bool(std::vector<AddDataRequest>&)>& read_batch) {
+ return impl_->rebuild_scalar_index(scalar_index_json, read_batch);
}
int64_t IndexEngine::dump(const std::string& dir) {
diff --git a/src/index/index_engine.h b/src/index/index_engine.h
index e177afbdb..f363e51fd 100644
--- a/src/index/index_engine.h
+++ b/src/index/index_engine.h
@@ -25,7 +25,7 @@ class IndexEngine {
int rebuild_scalar_index(
const std::string& scalar_index_json,
- const std::vector<AddDataRequest>& data_list);
+ const std::function<bool(std::vector<AddDataRequest>&)>& read_batch);
SearchResult search(const SearchRequest& req);
diff --git a/src/index/index_manager.h b/src/index/index_manager.h
index 344241d4c..72b7dfa64 100644
--- a/src/index/index_manager.h
+++ b/src/index/index_manager.h
@@ -1,6 +1,7 @@
// Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
// SPDX-License-Identifier: AGPL-3.0
#pragma once
+#include <functional>
#include <string>
#include <vector>
#include "index/common_structs.h"
@@ -38,7 +39,7 @@ class IndexManager {
virtual int rebuild_scalar_index(
const std::string& scalar_index_json,
- const std::vector<AddDataRequest>& data_list) = 0;
+ const std::function<bool(std::vector<AddDataRequest>&)>& read_batch) = 0;
virtual int64_t dump(const std::string& dir) = 0;
diff --git a/tests/vectordb/test_engine_filter_packed_native_abi.py b/tests/vectordb/test_engine_filter_packed_native_abi.py
index dde81d9b3..4af24bead 100644
--- a/tests/vectordb/test_engine_filter_packed_native_abi.py
+++ b/tests/vectordb/test_engine_filter_packed_native_abi.py
@@ -87,7 +87,21 @@ def test_native_packed_filter_abi_has_exact_little_endian_layout_and_shapes():
{"FieldName": "uri", "FieldType": "path"},
{"FieldName": "schema_flag", "FieldType": "bool"},
]
- assert index.rebuild_scalar_index(json.dumps(scalar_meta), requests) == 0
+
+ def interrupted_rows():
+ for row in range(1100):
+ request = engine.AddDataRequest()
+ request.label = labels[row % len(labels)]
+ request.fields_str = '{"uri":"/replacement","schema_flag":true}'
+ yield request
+ raise ValueError("Store scan interrupted")
+
+ # Even after a batch has been built, a failed scan must keep the old index.
+ with pytest.raises(ValueError, match="Store scan interrupted"):
+ index.rebuild_scalar_index(json.dumps(scalar_meta), interrupted_rows())
+ assert index.evaluate_filter(keep_filter).bitset_words == expected_words
+
+ assert index.rebuild_scalar_index(json.dumps(scalar_meta), iter(requests)) == 0
assert index.set_filter_layout(labels) == 0
rebuilt = index.evaluate_filter(
json.dumps({"op": "must", "field": "schema_flag", "conds": [True]})