Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(vectordb): 在检索引擎中归一化 cosine 分数 (#5358)

Qin Haojie707a6da62d8e
diff --git a/openviking/storage/vectordb/engine/_python_api.py b/openviking/storage/vectordb/engine/_python_api.py
index 866219c6b..6fc6a654c 100644
--- a/openviking/storage/vectordb/engine/_python_api.py
+++ b/openviking/storage/vectordb/engine/_python_api.py
@@ -456,9 +456,9 @@ def build_abi3_exports(backend: Any) -> dict[str, Any]:
SchemaExport, BytesRowExport = Schema, BytesRow
class IndexEngine:
- def __init__(self, path_or_json: str):
+ def __init__(self, path_or_json: str, normalize_vector: bool = False):
self._backend = backend
- self._handle = backend._new_index_engine(path_or_json)
+ self._handle = backend._new_index_engine(path_or_json, normalize_vector)
def add_data(self, data_list: list[AddDataRequest]) -> int:
return int(
diff --git a/openviking/storage/vectordb/index/cuvs_index.py b/openviking/storage/vectordb/index/cuvs_index.py
index 2e202af54..6c8d525b2 100644
--- a/openviking/storage/vectordb/index/cuvs_index.py
+++ b/openviking/storage/vectordb/index/cuvs_index.py
@@ -2035,6 +2035,13 @@ class CuVSDenseIndex:
assert results is not None
request.future.set_result(results[index])
+ def _score_from_distance(self, distance: float) -> float:
+ if self.distance == "l2":
+ return 1.0 - distance
+ if self.normalize_vectors:
+ return min(1.0, max(0.0, (distance + 1.0) / 2.0))
+ return distance
+
def _dispatch_micro_batch(self, requests: List[_CuVSMicroBatchRequest]) -> None:
runnable = [
request for request in requests if request.future.set_running_or_notify_cancel()
@@ -2108,7 +2115,7 @@ class CuVSDenseIndex:
if offset < 0 or offset >= len(request_snapshot.labels):
continue
labels.append(request_snapshot.labels[offset])
- scores.append(1.0 - distance if self.distance == "l2" else distance)
+ scores.append(self._score_from_distance(distance))
results.append((labels, scores))
except BaseException as exc:
error = exc
@@ -2475,7 +2482,7 @@ class CuVSDenseIndex:
if offset < 0 or offset >= len(snapshot.labels):
continue
labels.append(snapshot.labels[offset])
- scores.append(1.0 - distance if self.distance == "l2" else distance)
+ scores.append(self._score_from_distance(distance))
finally:
try:
# A concurrent rebuild can remove the index/cache's owning
diff --git a/openviking/storage/vectordb/index/local_index.py b/openviking/storage/vectordb/index/local_index.py
index 81917e440..77d9e378a 100644
--- a/openviking/storage/vectordb/index/local_index.py
+++ b/openviking/storage/vectordb/index/local_index.py
@@ -129,7 +129,9 @@ class IndexEngineProxy:
normalize_vector_flag (bool): If True, all vectors will be L2-normalized
before being added to the index or used for search. Defaults to False.
"""
- self.index_engine: Optional[engine.IndexEngine] = engine.IndexEngine(index_path_or_json)
+ self.index_engine: Optional[engine.IndexEngine] = engine.IndexEngine(
+ index_path_or_json, normalize_vector_flag
+ )
self.normalize_vector_flag = normalize_vector_flag
def search(
diff --git a/src/abi3_engine_backend.cpp b/src/abi3_engine_backend.cpp
index 8b25bdd50..7ed708ad2 100644
--- a/src/abi3_engine_backend.cpp
+++ b/src/abi3_engine_backend.cpp
@@ -1325,13 +1325,14 @@ PyObject* py_init_logging(PyObject*, PyObject* args, PyObject* kwargs) {
PyObject* py_new_index_engine(PyObject*, PyObject* args) {
const char* path_or_json = nullptr;
- if (!PyArg_ParseTuple(args, "s", &path_or_json)) {
+ int normalize_vector = 0;
+ if (!PyArg_ParseTuple(args, "s|p", &path_or_json, &normalize_vector)) {
return nullptr;
}
try {
- return PyCapsule_New(new vdb::IndexEngine(path_or_json), kIndexCapsuleName,
- index_capsule_destructor);
+ return PyCapsule_New(new vdb::IndexEngine(path_or_json, normalize_vector != 0),
+ kIndexCapsuleName, index_capsule_destructor);
} catch (const std::exception& exc) {
raise_runtime_error(exc.what());
return nullptr;
diff --git a/src/index/detail/index_manager_impl.cpp b/src/index/detail/index_manager_impl.cpp
index 5b67dc440..001e94464 100644
--- a/src/index/detail/index_manager_impl.cpp
+++ b/src/index/detail/index_manager_impl.cpp
@@ -24,22 +24,21 @@ constexpr uint64_t kFilterLayoutInverseMaxSpanFactor = 4;
constexpr uint32_t kMissingFilterLayoutOffset =
std::numeric_limits<uint32_t>::max();
-IndexManagerImpl::IndexManagerImpl(const std::string& path_or_json) {
- int ret = 0;
+IndexManagerImpl::IndexManagerImpl(const std::string& path_or_json,
+ bool normalize_vector) {
std::filesystem::path dir(path_or_json);
std::error_code ec;
if (std::filesystem::exists(dir, ec)) {
load_from_path(dir);
- return;
- }
-
- JsonDoc json;
- json.Parse(path_or_json.c_str());
- if (!json.HasParseError()) {
+ } else {
+ JsonDoc json;
+ json.Parse(path_or_json.c_str());
+ if (json.HasParseError()) {
+ return;
+ }
init_from_json(json);
- return;
}
- return;
+ manager_meta_->vector_index_meta->normalize_vector = normalize_vector;
}
void IndexManagerImpl::init_from_json(const JsonDoc& json) {
diff --git a/src/index/detail/index_manager_impl.h b/src/index/detail/index_manager_impl.h
index 578f4e990..15398dd40 100644
--- a/src/index/detail/index_manager_impl.h
+++ b/src/index/detail/index_manager_impl.h
@@ -22,7 +22,7 @@ namespace vectordb {
class IndexManagerImpl : public IndexManager {
public:
- IndexManagerImpl(const std::string& path_or_json);
+ IndexManagerImpl(const std::string& path_or_json, bool normalize_vector = false);
~IndexManagerImpl() {
scalar_index_.reset();
diff --git a/src/index/detail/meta/vector_index_meta.h b/src/index/detail/meta/vector_index_meta.h
index 4c4a728fa..a7ba196a7 100644
--- a/src/index/detail/meta/vector_index_meta.h
+++ b/src/index/detail/meta/vector_index_meta.h
@@ -9,6 +9,8 @@ namespace vectordb {
class VectorIndexMeta {
public:
std::string distance_type = "ip";
+ // Runtime configuration from the caller; not serialized in native snapshots.
+ bool normalize_vector = false;
std::string index_type;
std::string quantization_type = "float"; // "float" | "int8"
uint64_t element_count = 0;
diff --git a/src/index/detail/vector/common/bruteforce.h b/src/index/detail/vector/common/bruteforce.h
index 3d0a45371..8589580f6 100644
--- a/src/index/detail/vector/common/bruteforce.h
+++ b/src/index/detail/vector/common/bruteforce.h
@@ -262,9 +262,15 @@ class BruteforceSearch {
labels.resize(result_size);
scores.resize(result_size);
+ const bool cosine_score =
+ meta_->distance_type == "ip" && meta_->normalize_vector &&
+ (!query_sparse_view || meta_->search_with_sparse_logit_alpha <= 0.0f);
for (int i = static_cast<int>(result_size) - 1; i >= 0; --i) {
const auto& top = pq.top();
- scores[i] = top.first;
+ // Map after ranking so clamping quantization error cannot change top-k.
+ scores[i] = cosine_score
+ ? std::clamp((top.first + 1.0f) * 0.5f, 0.0f, 1.0f)
+ : top.first;
labels[i] = top.second;
pq.pop();
}
diff --git a/src/index/index_engine.cpp b/src/index/index_engine.cpp
index 2db71d1f0..6b7230be0 100644
--- a/src/index/index_engine.cpp
+++ b/src/index/index_engine.cpp
@@ -7,8 +7,8 @@
#include <unistd.h>
namespace vectordb {
-IndexEngine::IndexEngine(const std::string& path_or_json) {
- impl_ = std::make_shared<IndexManagerImpl>(path_or_json);
+IndexEngine::IndexEngine(const std::string& path_or_json, bool normalize_vector) {
+ impl_ = std::make_shared<IndexManagerImpl>(path_or_json, normalize_vector);
}
SearchResult IndexEngine::search(const SearchRequest& req) {
diff --git a/src/index/index_engine.h b/src/index/index_engine.h
index f363e51fd..3a0cd7b1a 100644
--- a/src/index/index_engine.h
+++ b/src/index/index_engine.h
@@ -14,7 +14,8 @@ namespace vectordb {
class IndexEngine {
public:
- IndexEngine(const std::string& path_or_json);
+ // The caller normalizes vectors and supplies the same flag on every load.
+ IndexEngine(const std::string& path_or_json, bool normalize_vector = false);
bool is_valid() const {
return impl_ != nullptr;
diff --git a/tests/engine/test_index_engine.cpp b/tests/engine/test_index_engine.cpp
index 9bd4186dd..db666d6b2 100644
--- a/tests/engine/test_index_engine.cpp
+++ b/tests/engine/test_index_engine.cpp
@@ -54,7 +54,7 @@ void test_basic_workflow() {
"ElementCount": 0,
"MaxElementCount": 2,
"Dimension": 4,
- "Distance": "l2",
+ "Distance": "ip",
"Quant": "float"
},
"ScalarIndex": [
@@ -65,7 +65,7 @@ void test_basic_workflow() {
]
})";
- IndexEngine engine(config);
+ IndexEngine engine(config, true);
if (!engine.is_valid()) {
SPDLOG_ERROR("Engine initialization failed");
exit(1);
@@ -76,14 +76,14 @@ void test_basic_workflow() {
AddDataRequest req1;
req1.label = 1001;
- req1.vector = {0.1, 0.1, 0.1, 0.1};
+ req1.vector = {1.0, 0.0, 0.0, 0.0};
req1.fields_str =
R"({"title": "apple", "count": 10, "price": 5.5, "uri": "/docs/one"})";
add_reqs.push_back(req1);
AddDataRequest req2;
req2.label = 1002;
- req2.vector = {0.2, 0.2, 0.2, 0.2};
+ req2.vector = {-1.0, 0.0, 0.0, 0.0};
req2.fields_str =
R"({"title": "banana", "count": 20, "price": 3.0, "uri": "/other/two"})";
add_reqs.push_back(req2);
@@ -96,7 +96,7 @@ void test_basic_workflow() {
// 3. Search (Vector only)
SearchRequest search_req;
- search_req.query = {0.1, 0.1, 0.1, 0.1};
+ search_req.query = {1.0, 0.0, 0.0, 0.0};
search_req.topk = 5;
SearchResult res = engine.search(search_req);
@@ -109,6 +109,26 @@ void test_basic_workflow() {
exit(1);
}
+ if (res.result_num != 2 || !is_close(res.scores[0], 1.0f) ||
+ !is_close(res.scores[1], 0.0f)) {
+ SPDLOG_ERROR("Cosine endpoints were not mapped to [0, 1]");
+ exit(1);
+ }
+ search_req.query = {0.0, 1.0, 0.0, 0.0};
+ res = engine.search(search_req);
+ if (!is_close(res.scores[0], 0.5f) || !is_close(res.scores[1], 0.5f)) {
+ SPDLOG_ERROR("Orthogonal cosine scores were not mapped to 0.5");
+ exit(1);
+ }
+ search_req.query = {-0.6, 0.8, 0.0, 0.0};
+ res = engine.search(search_req);
+ if (res.labels != std::vector<uint64_t>{1002, 1001} ||
+ !is_close(res.scores[0], 0.8f) || !is_close(res.scores[1], 0.2f)) {
+ SPDLOG_ERROR("Cosine mapping changed the scores or ranking");
+ exit(1);
+ }
+ search_req.query = {1.0, 0.0, 0.0, 0.0};
+
// Native scalar filters can be projected into any external row order. This
// is the bridge used by external dense indexes such as cuVS.
if (engine.set_filter_layout({1002, 9999, 1001}) != 0) {
@@ -168,7 +188,8 @@ void test_basic_workflow() {
auto token_search_res = engine.search_with_filter_token(
search_req, cached_filter_res.native_filter_token);
if (!token_search_res || token_search_res->result_num != 1 ||
- token_search_res->labels[0] != 1001) {
+ token_search_res->labels[0] != 1001 ||
+ !is_close(token_search_res->scores[0], 1.0f)) {
SPDLOG_ERROR("Search with native filter token failed");
exit(1);
}
@@ -211,6 +232,17 @@ void test_basic_workflow() {
exit(1);
}
+ // The same snapshot is readable as raw IP or cosine without rebuilding.
+ for (bool normalize_vector : {false, true}) {
+ IndexEngine reloaded(db_path, normalize_vector);
+ res = reloaded.search(search_req);
+ if (res.result_num != 1 || res.labels[0] != 1002 ||
+ !is_close(res.scores[0], normalize_vector ? 0.0f : -1.0f)) {
+ SPDLOG_ERROR("Reload did not apply the caller's cosine configuration");
+ exit(1);
+ }
+ }
+
std::filesystem::remove_all(db_path);
SPDLOG_INFO("[Passed] test_basic_workflow");
}
diff --git a/tests/vectordb/test_cuvs_index.py b/tests/vectordb/test_cuvs_index.py
index 07b705ca4..22c4e8bf2 100644
--- a/tests/vectordb/test_cuvs_index.py
+++ b/tests/vectordb/test_cuvs_index.py
@@ -848,20 +848,25 @@ def test_cuvs_registrar_failure_consumes_candidate_without_marking_clean():
assert index.search([1.0, 0.0], 1, None)[0] == [1]
-def test_cuvs_dense_search_handles_filter_upsert_delete_and_lazy_rebuild():
[email protected]("micro_batching_enabled", [False, True])
+def test_cuvs_dense_search_handles_filter_upsert_delete_and_lazy_rebuild(micro_batching_enabled):
runtime = FakeCuVSRuntime()
index = CuVSDenseIndex(
dimension=2,
distance="ip",
normalize_vectors=True,
field_types={"account_id": "string", "uri": "path"},
- config={"algorithm": "brute_force"},
+ config={
+ "algorithm": "brute_force",
+ "micro_batching_enabled": micro_batching_enabled,
+ "micro_batching_max_wait_ms": 0.1,
+ },
runtime=runtime,
)
index.add_candidates(
[
candidate(10, [1.0, 0.0], account_id="a", uri="/docs/one"),
- candidate(20, [0.8, 0.2], account_id="a", uri="/docs/deep/two"),
+ candidate(20, [-3.0, 4.0], account_id="a", uri="/docs/deep/two"),
candidate(30, [0.0, 1.0], account_id="b", uri="/other/three"),
]
)
@@ -881,6 +886,13 @@ def test_cuvs_dense_search_handles_filter_upsert_delete_and_lazy_rebuild():
assert scores == [1.0]
assert runtime.build_count == 1
+ labels, scores = index.search([1.0, 0.0], 3, None)
+ assert labels == [10, 30, 20]
+ assert scores == pytest.approx([1.0, 0.5, 0.2])
+ labels, scores = index.search([-1.0, 0.0], 3, None)
+ assert labels == [20, 30, 10]
+ assert scores == pytest.approx([0.8, 0.5, 0.0])
+
# Repeated reads reuse the GPU index; a mutation invalidates it exactly once.
assert index.search([1.0, 0.0], 1, None)[0] == [10]
assert runtime.build_count == 1
@@ -890,6 +902,7 @@ def test_cuvs_dense_search_handles_filter_upsert_delete_and_lazy_rebuild():
index.delete([DeltaRecord(label=10)])
assert index.search([1.0, 0.0], 3, None)[0] == [30, 20]
assert runtime.build_count == 3
+ index.close()
def test_empty_filter_does_not_allocate_a_device_bitset():
diff --git a/tests/vectordb/test_engine_filter_routing_abi.py b/tests/vectordb/test_engine_filter_routing_abi.py
index b07d7ea66..c086e5738 100644
--- a/tests/vectordb/test_engine_filter_routing_abi.py
+++ b/tests/vectordb/test_engine_filter_routing_abi.py
@@ -10,7 +10,7 @@ class _CurrentBackend:
def __init__(self):
self.calls = []
- def _new_index_engine(self, path_or_json):
+ def _new_index_engine(self, path_or_json, normalize_vector=False):
self.calls.append(("new", path_or_json))
return "engine-handle"
@@ -33,7 +33,7 @@ class _LegacyBackend:
def __init__(self):
self.calls = []
- def _new_index_engine(self, path_or_json):
+ def _new_index_engine(self, path_or_json, normalize_vector=False):
return "legacy-handle"
def _index_engine_evaluate_filter(self, handle, dsl):