Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(vectordb): 在检索引擎中归一化 cosine 分数 (#5358)
Qin Haojie707a6da62d8e
diff --git a/openviking/storage/vectordb/engine/_python_api.py b/openviking/storage/vectordb/engine/_python_api.pyindex 866219c6b..6fc6a654c 100644--- a/openviking/storage/vectordb/engine/_python_api.py+++ b/openviking/storage/vectordb/engine/_python_api.py@@ -456,9 +456,9 @@ def build_abi3_exports(backend: Any) -> dict[str, Any]:SchemaExport, BytesRowExport = Schema, BytesRowclass IndexEngine:- def __init__(self, path_or_json: str):+ def __init__(self, path_or_json: str, normalize_vector: bool = False):self._backend = backend- self._handle = backend._new_index_engine(path_or_json)+ self._handle = backend._new_index_engine(path_or_json, normalize_vector)def add_data(self, data_list: list[AddDataRequest]) -> int:return int(diff --git a/openviking/storage/vectordb/index/cuvs_index.py b/openviking/storage/vectordb/index/cuvs_index.pyindex 2e202af54..6c8d525b2 100644--- a/openviking/storage/vectordb/index/cuvs_index.py+++ b/openviking/storage/vectordb/index/cuvs_index.py@@ -2035,6 +2035,13 @@ class CuVSDenseIndex:assert results is not Nonerequest.future.set_result(results[index])+ def _score_from_distance(self, distance: float) -> float:+ if self.distance == "l2":+ return 1.0 - distance+ if self.normalize_vectors:+ return min(1.0, max(0.0, (distance + 1.0) / 2.0))+ return distance+def _dispatch_micro_batch(self, requests: List[_CuVSMicroBatchRequest]) -> None:runnable = [request for request in requests if request.future.set_running_or_notify_cancel()@@ -2108,7 +2115,7 @@ class CuVSDenseIndex:if offset < 0 or offset >= len(request_snapshot.labels):continuelabels.append(request_snapshot.labels[offset])- scores.append(1.0 - distance if self.distance == "l2" else distance)+ scores.append(self._score_from_distance(distance))results.append((labels, scores))except BaseException as exc:error = exc@@ -2475,7 +2482,7 @@ class CuVSDenseIndex:if offset < 0 or offset >= len(snapshot.labels):continuelabels.append(snapshot.labels[offset])- scores.append(1.0 - distance if self.distance == "l2" else distance)+ scores.append(self._score_from_distance(distance))finally:try:# A concurrent rebuild can remove the index/cache's owningdiff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.pyindex 81917e440..77d9e378a 100644--- a/openviking/storage/vectordb/index/local_index.py+++ b/openviking/storage/vectordb/index/local_index.py@@ -129,7 +129,9 @@ class IndexEngineProxy:normalize_vector_flag (bool): If True, all vectors will be L2-normalizedbefore being added to the index or used for search. Defaults to False."""- self.index_engine: Optional[engine.IndexEngine] = engine.IndexEngine(index_path_or_json)+ self.index_engine: Optional[engine.IndexEngine] = engine.IndexEngine(+ index_path_or_json, normalize_vector_flag+ )self.normalize_vector_flag = normalize_vector_flagdef search(diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cppindex 8b25bdd50..7ed708ad2 100644--- a/src/abi3_engine_backend.cpp+++ b/src/abi3_engine_backend.cpp@@ -1325,13 +1325,14 @@ PyObject* py_init_logging(PyObject*, PyObject* args, PyObject* kwargs) {PyObject* py_new_index_engine(PyObject*, PyObject* args) {const char* path_or_json = nullptr;- if (!PyArg_ParseTuple(args, "s", &path_or_json)) {+ int normalize_vector = 0;+ if (!PyArg_ParseTuple(args, "s|p", &path_or_json, &normalize_vector)) {return nullptr;}try {- return PyCapsule_New(new vdb::IndexEngine(path_or_json), kIndexCapsuleName,- index_capsule_destructor);+ return PyCapsule_New(new vdb::IndexEngine(path_or_json, normalize_vector != 0),+ kIndexCapsuleName, index_capsule_destructor);} catch (const std::exception& exc) {raise_runtime_error(exc.what());return nullptr;diff --git a/src/index/detail/index_manager_impl.cpp b/src/index/detail/index_manager_impl.cppindex 5b67dc440..001e94464 100644--- a/src/index/detail/index_manager_impl.cpp+++ b/src/index/detail/index_manager_impl.cpp@@ -24,22 +24,21 @@ constexpr uint64_t kFilterLayoutInverseMaxSpanFactor = 4;constexpr uint32_t kMissingFilterLayoutOffset =std::numeric_limits<uint32_t>::max();-IndexManagerImpl::IndexManagerImpl(const std::string& path_or_json) {- int ret = 0;+IndexManagerImpl::IndexManagerImpl(const std::string& path_or_json,+ bool normalize_vector) {std::filesystem::path dir(path_or_json);std::error_code ec;if (std::filesystem::exists(dir, ec)) {load_from_path(dir);- return;- }-- JsonDoc json;- json.Parse(path_or_json.c_str());- if (!json.HasParseError()) {+ } else {+ JsonDoc json;+ json.Parse(path_or_json.c_str());+ if (json.HasParseError()) {+ return;+ }init_from_json(json);- return;}- return;+ manager_meta_->vector_index_meta->normalize_vector = normalize_vector;}void IndexManagerImpl::init_from_json(const JsonDoc& json) {diff --git a/src/index/detail/index_manager_impl.h b/src/index/detail/index_manager_impl.hindex 578f4e990..15398dd40 100644--- a/src/index/detail/index_manager_impl.h+++ b/src/index/detail/index_manager_impl.h@@ -22,7 +22,7 @@ namespace vectordb {class IndexManagerImpl : public IndexManager {public:- IndexManagerImpl(const std::string& path_or_json);+ IndexManagerImpl(const std::string& path_or_json, bool normalize_vector = false);~IndexManagerImpl() {scalar_index_.reset();diff --git a/src/index/detail/meta/vector_index_meta.h b/src/index/detail/meta/vector_index_meta.hindex 4c4a728fa..a7ba196a7 100644--- a/src/index/detail/meta/vector_index_meta.h+++ b/src/index/detail/meta/vector_index_meta.h@@ -9,6 +9,8 @@ namespace vectordb {class VectorIndexMeta {public:std::string distance_type = "ip";+ // Runtime configuration from the caller; not serialized in native snapshots.+ bool normalize_vector = false;std::string index_type;std::string quantization_type = "float"; // "float" | "int8"uint64_t element_count = 0;diff --git a/src/index/detail/vector/common/bruteforce.h b/src/index/detail/vector/common/bruteforce.hindex 3d0a45371..8589580f6 100644--- a/src/index/detail/vector/common/bruteforce.h+++ b/src/index/detail/vector/common/bruteforce.h@@ -262,9 +262,15 @@ class BruteforceSearch {labels.resize(result_size);scores.resize(result_size);+ const bool cosine_score =+ meta_->distance_type == "ip" && meta_->normalize_vector &&+ (!query_sparse_view || meta_->search_with_sparse_logit_alpha <= 0.0f);for (int i = static_cast<int>(result_size) - 1; i >= 0; --i) {const auto& top = pq.top();- scores[i] = top.first;+ // Map after ranking so clamping quantization error cannot change top-k.+ scores[i] = cosine_score+ ? std::clamp((top.first + 1.0f) * 0.5f, 0.0f, 1.0f)+ : top.first;labels[i] = top.second;pq.pop();}diff --git a/src/index/index_engine.cpp b/src/index/index_engine.cppindex 2db71d1f0..6b7230be0 100644--- a/src/index/index_engine.cpp+++ b/src/index/index_engine.cpp@@ -7,8 +7,8 @@#include <unistd.h>namespace vectordb {-IndexEngine::IndexEngine(const std::string& path_or_json) {- impl_ = std::make_shared<IndexManagerImpl>(path_or_json);+IndexEngine::IndexEngine(const std::string& path_or_json, bool normalize_vector) {+ impl_ = std::make_shared<IndexManagerImpl>(path_or_json, normalize_vector);}SearchResult IndexEngine::search(const SearchRequest& req) {diff --git a/src/index/index_engine.h b/src/index/index_engine.hindex f363e51fd..3a0cd7b1a 100644--- a/src/index/index_engine.h+++ b/src/index/index_engine.h@@ -14,7 +14,8 @@ namespace vectordb {class IndexEngine {public:- IndexEngine(const std::string& path_or_json);+ // The caller normalizes vectors and supplies the same flag on every load.+ IndexEngine(const std::string& path_or_json, bool normalize_vector = false);bool is_valid() const {return impl_ != nullptr;diff --git a/tests/engine/test_index_engine.cpp b/tests/engine/test_index_engine.cppindex 9bd4186dd..db666d6b2 100644--- a/tests/engine/test_index_engine.cpp+++ b/tests/engine/test_index_engine.cpp@@ -54,7 +54,7 @@ void test_basic_workflow() {"ElementCount": 0,"MaxElementCount": 2,"Dimension": 4,- "Distance": "l2",+ "Distance": "ip","Quant": "float"},"ScalarIndex": [@@ -65,7 +65,7 @@ void test_basic_workflow() {]})";- IndexEngine engine(config);+ IndexEngine engine(config, true);if (!engine.is_valid()) {SPDLOG_ERROR("Engine initialization failed");exit(1);@@ -76,14 +76,14 @@ void test_basic_workflow() {AddDataRequest req1;req1.label = 1001;- req1.vector = {0.1, 0.1, 0.1, 0.1};+ req1.vector = {1.0, 0.0, 0.0, 0.0};req1.fields_str =R"({"title": "apple", "count": 10, "price": 5.5, "uri": "/docs/one"})";add_reqs.push_back(req1);AddDataRequest req2;req2.label = 1002;- req2.vector = {0.2, 0.2, 0.2, 0.2};+ req2.vector = {-1.0, 0.0, 0.0, 0.0};req2.fields_str =R"({"title": "banana", "count": 20, "price": 3.0, "uri": "/other/two"})";add_reqs.push_back(req2);@@ -96,7 +96,7 @@ void test_basic_workflow() {// 3. Search (Vector only)SearchRequest search_req;- search_req.query = {0.1, 0.1, 0.1, 0.1};+ search_req.query = {1.0, 0.0, 0.0, 0.0};search_req.topk = 5;SearchResult res = engine.search(search_req);@@ -109,6 +109,26 @@ void test_basic_workflow() {exit(1);}+ if (res.result_num != 2 || !is_close(res.scores[0], 1.0f) ||+ !is_close(res.scores[1], 0.0f)) {+ SPDLOG_ERROR("Cosine endpoints were not mapped to [0, 1]");+ exit(1);+ }+ search_req.query = {0.0, 1.0, 0.0, 0.0};+ res = engine.search(search_req);+ if (!is_close(res.scores[0], 0.5f) || !is_close(res.scores[1], 0.5f)) {+ SPDLOG_ERROR("Orthogonal cosine scores were not mapped to 0.5");+ exit(1);+ }+ search_req.query = {-0.6, 0.8, 0.0, 0.0};+ res = engine.search(search_req);+ if (res.labels != std::vector<uint64_t>{1002, 1001} ||+ !is_close(res.scores[0], 0.8f) || !is_close(res.scores[1], 0.2f)) {+ SPDLOG_ERROR("Cosine mapping changed the scores or ranking");+ exit(1);+ }+ search_req.query = {1.0, 0.0, 0.0, 0.0};+// Native scalar filters can be projected into any external row order. This// is the bridge used by external dense indexes such as cuVS.if (engine.set_filter_layout({1002, 9999, 1001}) != 0) {@@ -168,7 +188,8 @@ void test_basic_workflow() {auto token_search_res = engine.search_with_filter_token(search_req, cached_filter_res.native_filter_token);if (!token_search_res || token_search_res->result_num != 1 ||- token_search_res->labels[0] != 1001) {+ token_search_res->labels[0] != 1001 ||+ !is_close(token_search_res->scores[0], 1.0f)) {SPDLOG_ERROR("Search with native filter token failed");exit(1);}@@ -211,6 +232,17 @@ void test_basic_workflow() {exit(1);}+ // The same snapshot is readable as raw IP or cosine without rebuilding.+ for (bool normalize_vector : {false, true}) {+ IndexEngine reloaded(db_path, normalize_vector);+ res = reloaded.search(search_req);+ if (res.result_num != 1 || res.labels[0] != 1002 ||+ !is_close(res.scores[0], normalize_vector ? 0.0f : -1.0f)) {+ SPDLOG_ERROR("Reload did not apply the caller's cosine configuration");+ exit(1);+ }+ }+std::filesystem::remove_all(db_path);SPDLOG_INFO("[Passed] test_basic_workflow");}diff --git a/tests/vectordb/test_cuvs_index.py b/tests/vectordb/test_cuvs_index.pyindex 07b705ca4..22c4e8bf2 100644--- a/tests/vectordb/test_cuvs_index.py+++ b/tests/vectordb/test_cuvs_index.py@@ -848,20 +848,25 @@ def test_cuvs_registrar_failure_consumes_candidate_without_marking_clean():assert index.search([1.0, 0.0], 1, None)[0] == [1]-def test_cuvs_dense_search_handles_filter_upsert_delete_and_lazy_rebuild():[email protected]("micro_batching_enabled", [False, True])+def test_cuvs_dense_search_handles_filter_upsert_delete_and_lazy_rebuild(micro_batching_enabled):runtime = FakeCuVSRuntime()index = CuVSDenseIndex(dimension=2,distance="ip",normalize_vectors=True,field_types={"account_id": "string", "uri": "path"},- config={"algorithm": "brute_force"},+ config={+ "algorithm": "brute_force",+ "micro_batching_enabled": micro_batching_enabled,+ "micro_batching_max_wait_ms": 0.1,+ },runtime=runtime,)index.add_candidates([candidate(10, [1.0, 0.0], account_id="a", uri="/docs/one"),- candidate(20, [0.8, 0.2], account_id="a", uri="/docs/deep/two"),+ candidate(20, [-3.0, 4.0], account_id="a", uri="/docs/deep/two"),candidate(30, [0.0, 1.0], account_id="b", uri="/other/three"),])@@ -881,6 +886,13 @@ def test_cuvs_dense_search_handles_filter_upsert_delete_and_lazy_rebuild():assert scores == [1.0]assert runtime.build_count == 1+ labels, scores = index.search([1.0, 0.0], 3, None)+ assert labels == [10, 30, 20]+ assert scores == pytest.approx([1.0, 0.5, 0.2])+ labels, scores = index.search([-1.0, 0.0], 3, None)+ assert labels == [20, 30, 10]+ assert scores == pytest.approx([0.8, 0.5, 0.0])+# Repeated reads reuse the GPU index; a mutation invalidates it exactly once.assert index.search([1.0, 0.0], 1, None)[0] == [10]assert runtime.build_count == 1@@ -890,6 +902,7 @@ def test_cuvs_dense_search_handles_filter_upsert_delete_and_lazy_rebuild():index.delete([DeltaRecord(label=10)])assert index.search([1.0, 0.0], 3, None)[0] == [30, 20]assert runtime.build_count == 3+ index.close()def test_empty_filter_does_not_allocate_a_device_bitset():diff --git a/tests/vectordb/test_engine_filter_routing_abi.py b/tests/vectordb/test_engine_filter_routing_abi.pyindex b07d7ea66..c086e5738 100644--- a/tests/vectordb/test_engine_filter_routing_abi.py+++ b/tests/vectordb/test_engine_filter_routing_abi.py@@ -10,7 +10,7 @@ class _CurrentBackend:def __init__(self):self.calls = []- def _new_index_engine(self, path_or_json):+ def _new_index_engine(self, path_or_json, normalize_vector=False):self.calls.append(("new", path_or_json))return "engine-handle"@@ -33,7 +33,7 @@ class _LegacyBackend:def __init__(self):self.calls = []- def _new_index_engine(self, path_or_json):+ def _new_index_engine(self, path_or_json, normalize_vector=False):return "legacy-handle"def _index_engine_evaluate_filter(self, handle, dsl):