Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: add search_with_sparse_logit_alpha (#71)

kkkwjx9368d838f794
diff --git a/docs/en/configuration/configuration.md b/docs/en/configuration/configuration.md
index d15e0f9fb..5b29e8cc0 100644
--- a/docs/en/configuration/configuration.md
+++ b/docs/en/configuration/configuration.md
@@ -207,6 +207,9 @@ client = ov.AsyncOpenViking(config=config)
}
```
+Notes:
+- `storage.vectordb.sparse_weight` controls hybrid (dense + sparse) indexing/search. It only takes effect when you use a hybrid index; set it > 0 to enable sparse signals.
+
## Related Documentation
- [Embedding Configuration](./embedding.md) - Embedding setup
diff --git a/docs/zh/configuration/configuration.md b/docs/zh/configuration/configuration.md
index 93f5e21ff..bef6327b5 100644
--- a/docs/zh/configuration/configuration.md
+++ b/docs/zh/configuration/configuration.md
@@ -294,6 +294,9 @@ client = ov.AsyncOpenViking(config=config)
}
```
+说明:
+- `storage.vectordb.sparse_weight` 用于混合(dense + sparse)索引/检索的权重,仅在使用 hybrid 索引时生效;设置为 > 0 才会启用 sparse 信号。
+
## 相关文档
- [Embedding 配置](./embedding.md) - Embedding 设置
diff --git a/openviking/storage/queuefs/embedding_queue.py b/openviking/storage/queuefs/embedding_queue.py
index 6af3d3d0b..b6dc771ae 100644
--- a/openviking/storage/queuefs/embedding_queue.py
+++ b/openviking/storage/queuefs/embedding_queue.py
@@ -16,7 +16,7 @@ class EmbeddingQueue(NamedQueue):
Supports direct enqueue and dequeue of EmbeddingMsg objects.
"""
- async def enqueue(self, msg: EmbeddingMsg | None) -> str:
+ async def enqueue(self, msg: Optional[EmbeddingMsg]) -> str:
"""Serialize EmbeddingMsg object and store in queue."""
if msg is None:
logger.warning("Embedding message is None, skipping enqueuing")
@@ -28,17 +28,26 @@ class EmbeddingQueue(NamedQueue):
data_dict = await super().dequeue()
if not data_dict:
return None
- if "data" in data_dict and isinstance(data_dict["data"], str):
- try:
- return EmbeddingMsg.from_json(data_dict["data"])
- except Exception as e:
- logger.debug(f"[EmbeddingQueue] Failed to parse message data: {e}")
- return None
+ if "data" in data_dict:
+ if isinstance(data_dict["data"], str):
+ try:
+ return EmbeddingMsg.from_json(data_dict["data"])
+ except Exception as e:
+ logger.debug(f"[EmbeddingQueue] Failed to parse message data: {e}")
+ return None
+ elif isinstance(data_dict["data"], dict):
+ try:
+ return EmbeddingMsg.from_dict(data_dict["data"])
+ except Exception as e:
+ logger.debug(
+ f"[EmbeddingQueue] Failed to create EmbeddingMsg from data dict: {e}"
+ )
+ return None
+
# Otherwise try to convert directly from dict
try:
return EmbeddingMsg.from_dict(data_dict)
- except Exception as e:
- logger.debug(f"[EmbeddingQueue] Failed to create EmbeddingMsg from dict: {e}")
+ except Exception:
return None
async def peek(self) -> Optional[EmbeddingMsg]:
@@ -47,11 +56,17 @@ class EmbeddingQueue(NamedQueue):
if not data_dict:
return None
- if "data" in data_dict and isinstance(data_dict["data"], str):
- try:
- return EmbeddingMsg.from_json(data_dict["data"])
- except Exception:
- return None
+ if "data" in data_dict:
+ if isinstance(data_dict["data"], str):
+ try:
+ return EmbeddingMsg.from_json(data_dict["data"])
+ except Exception:
+ return None
+ elif isinstance(data_dict["data"], dict):
+ try:
+ return EmbeddingMsg.from_dict(data_dict["data"])
+ except Exception:
+ return None
try:
return EmbeddingMsg.from_dict(data_dict)
diff --git a/openviking/storage/vectordb/meta/index_meta.py b/openviking/storage/vectordb/meta/index_meta.py
index 3c2253e72..a7635e0f9 100644
--- a/openviking/storage/vectordb/meta/index_meta.py
+++ b/openviking/storage/vectordb/meta/index_meta.py
@@ -112,9 +112,6 @@ class IndexMeta:
vector_index["SearchWithSparseLogitAlpha"] = inner_meta["VectorIndex"].get(
"SearchWithSparseLogitAlpha", 0.5
)
- vector_index["IndexWithSparseLogitAlpha"] = inner_meta["VectorIndex"].get(
- "IndexWithSparseLogitAlpha", 0.5
- )
if "flat" in inner_meta["VectorIndex"]["IndexType"]:
vector_index["IndexType"] = "flat"
if "EnableSparse" in inner_meta["VectorIndex"]:
@@ -123,10 +120,6 @@ class IndexMeta:
vector_index["SearchWithSparseLogitAlpha"] = inner_meta["VectorIndex"][
"SearchWithSparseLogitAlpha"
]
- if "IndexWithSparseLogitAlpha" in inner_meta["VectorIndex"]:
- vector_index["IndexWithSparseLogitAlpha"] = inner_meta["VectorIndex"][
- "IndexWithSparseLogitAlpha"
- ]
inner_meta["VectorIndex"] = vector_index
inner_meta["CollectionName"] = collection_meta.collection_name
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index ff656f0bc..91c266c02 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -77,8 +77,9 @@ class VikingVectorIndexBackend(VikingDBInterface):
"""
init_cpp_logging()
- self.vector_dim = config.vector_dim
+ self.vector_dim = config.dimension
self.distance_metric = config.distance_metric
+ self.sparse_weight = config.sparse_weight
if config.backend == "volcengine":
if not (
@@ -242,15 +243,19 @@ class VikingVectorIndexBackend(VikingDBInterface):
scalar_index_fields.append(field_name)
# Create default index for the collection
+ use_sparse = self.sparse_weight > 0.0
index_meta = {
"IndexName": self.DEFAULT_INDEX_NAME,
"VectorIndex": {
- "IndexType": "flat_hybrid",
+ "IndexType": "flat_hybrid" if use_sparse else "flat",
"Distance": distance,
"Quant": "int8",
},
"ScalarIndex": scalar_index_fields,
}
+ if use_sparse:
+ index_meta["VectorIndex"]["EnableSparse"] = True
+ index_meta["VectorIndex"]["SearchWithSparseLogitAlpha"] = self.sparse_weight
logger.info(f"Creating index with meta: {index_meta}")
collection.create_index(self.DEFAULT_INDEX_NAME, index_meta)
diff --git a/openviking/utils/config/open_viking_config.py b/openviking/utils/config/open_viking_config.py
index ada70a23d..6d6c646d0 100644
--- a/openviking/utils/config/open_viking_config.py
+++ b/openviking/utils/config/open_viking_config.py
@@ -301,6 +301,10 @@ def initialize_openviking_config(
config.storage.vectordb.backend = "http"
config.storage.vectordb.url = vectordb_url
+ # Ensure vector dimension is synced if not set in storage
+ if config.storage.vectordb.dimension == 0:
+ config.storage.vectordb.dimension = config.embedding.dimension
+
# Validate configuration
if not is_valid_openviking_config(config):
raise ValueError("Invalid OpenViking configuration")
diff --git a/openviking/utils/config/vectordb_config.py b/openviking/utils/config/vectordb_config.py
index dc32c7e42..9af43fdea 100644
--- a/openviking/utils/config/vectordb_config.py
+++ b/openviking/utils/config/vectordb_config.py
@@ -43,11 +43,19 @@ class VectorDBBackendConfig(BaseModel):
description="Distance metric for vector similarity search (e.g., 'cosine', 'l2', 'ip')",
)
- vector_dim: int = Field(
+ dimension: int = Field(
default=0,
description="Dimension of vector embeddings",
)
+ sparse_weight: float = Field(
+ default=0.0,
+ description=(
+ "Sparse weight for hybrid vector search. "
+ "When > 0, sparse vectors are used for index build and search."
+ ),
+ )
+
volcengine: Optional[VolcengineConfig] = Field(
default_factory=lambda: VolcengineConfig(),
description="Volcengine VikingDB configuration for 'volcengine' type",
diff --git a/src/index/detail/meta/vector_index_meta.cpp b/src/index/detail/meta/vector_index_meta.cpp
index 238b1d11b..a182bde0d 100644
--- a/src/index/detail/meta/vector_index_meta.cpp
+++ b/src/index/detail/meta/vector_index_meta.cpp
@@ -48,10 +48,6 @@ int VectorIndexMeta::init_from_json(const JsonValue& json) {
search_with_sparse_logit_alpha =
json["SearchWithSparseLogitAlpha"].GetFloat();
}
- if (json.HasMember("IndexWithSparseLogitAlpha")) {
- index_with_sparse_logit_alpha =
- json["IndexWithSparseLogitAlpha"].GetFloat();
- }
return 0;
}
@@ -102,9 +98,7 @@ int VectorIndexMeta::save_to_json(JsonPrettyWriter& writer) {
writer.Bool(enable_sparse);
writer.Key("SearchWithSparseLogitAlpha");
writer.Double(search_with_sparse_logit_alpha);
- writer.Key("IndexWithSparseLogitAlpha");
- writer.Double(index_with_sparse_logit_alpha);
return 0;
}
-} // namespace vectordb
\ No newline at end of file
+} // namespace vectordb
diff --git a/src/index/detail/vector/common/bruteforce.h b/src/index/detail/vector/common/bruteforce.h
index 7069f5558..2a065acfb 100644
--- a/src/index/detail/vector/common/bruteforce.h
+++ b/src/index/detail/vector/common/bruteforce.h
@@ -178,6 +178,11 @@ class BruteforceSearch {
std::vector<float>& scores) const {
if (!query_data)
return;
+ if (k == 0) {
+ labels.clear();
+ scores.clear();
+ return;
+ }
if (current_count_ == 0)
return;
@@ -204,17 +209,8 @@ class BruteforceSearch {
continue;
}
- float dist = dist_func(encoded_query.data(), ptr, dist_params);
-
- if (sparse_index_) {
- float sparse_dist = 0;
- if (query_sparse_view) {
- sparse_dist =
- sparse_index_->sparse_head_output(*query_sparse_view, i);
- }
- dist = dist * (1 - meta_->search_with_sparse_logit_alpha) +
- sparse_dist * meta_->search_with_sparse_logit_alpha;
- }
+ float dist = compute_score(encoded_query.data(), ptr, query_sparse_view,
+ i, dist_func, dist_params);
uint64_t label;
std::memcpy(&label, ptr + vector_byte_size_, sizeof(uint64_t));
@@ -307,6 +303,7 @@ class BruteforceSearch {
private:
void setup_metric() {
+ reverse_query_score_ = (meta_->distance_type == "l2");
if (meta_->quantization_type == "int8") {
if (meta_->distance_type == "l2")
space_ = std::make_unique<L2SpaceInt8>(meta_->dimension);
@@ -357,6 +354,26 @@ class BruteforceSearch {
return view;
}
+ float compute_score(
+ const void* encoded_query, const char* data_ptr,
+ const std::shared_ptr<SparseDatapointView>& query_sparse_view,
+ size_t idx, MetricFunc<float> dist_func,
+ void* dist_params) const {
+ float dense_raw = dist_func(encoded_query, data_ptr, dist_params);
+ float dense_score =
+ reverse_query_score_ ? (1.0f - dense_raw) : dense_raw;
+ if (!sparse_index_ || !query_sparse_view ||
+ meta_->search_with_sparse_logit_alpha <= 0.0f) {
+ return dense_score;
+ }
+ float sparse_raw =
+ sparse_index_->sparse_head_output(*query_sparse_view, idx);
+ float sparse_score =
+ reverse_query_score_ ? (1.0f - sparse_raw) : sparse_raw;
+ float alpha = meta_->search_with_sparse_logit_alpha;
+ return dense_score * (1.0f - alpha) + sparse_score * alpha;
+ }
+
std::shared_ptr<BruteForceMeta> meta_;
char* data_buffer_ = nullptr;
size_t capacity_ = 0;
@@ -370,6 +387,7 @@ class BruteforceSearch {
std::unique_ptr<VectorSpace<float>> space_;
std::unique_ptr<VectorQuantizer> quantizer_;
std::unique_ptr<SparseDataHolder> sparse_index_;
+ bool reverse_query_score_ = false;
};
} // namespace vectordb
diff --git a/src/index/detail/vector/common/space_int8.h b/src/index/detail/vector/common/space_int8.h
index 9e15acf5f..36dda0e9e 100644
--- a/src/index/detail/vector/common/space_int8.h
+++ b/src/index/detail/vector/common/space_int8.h
@@ -153,7 +153,7 @@ static float l2_distance_int8(const void* v1, const void* v2,
float real_ip = static_cast<float>(ip) * scale1 * scale2;
float dist = norm_sq1 + norm_sq2 - 2.0f * real_ip;
- return 1.0f - std::max(0.0f, dist);
+ return std::max(0.0f, dist);
}
class InnerProductSpaceInt8 : public VectorSpace<float> {
diff --git a/src/index/detail/vector/common/space_l2.h b/src/index/detail/vector/common/space_l2.h
index 7b0e50b42..a6bef28d6 100644
--- a/src/index/detail/vector/common/space_l2.h
+++ b/src/index/detail/vector/common/space_l2.h
@@ -18,7 +18,7 @@ static float l2_sqr_ref(const void* v1, const void* v2, const void* params) {
float diff = pv1[i] - pv2[i];
res += diff * diff;
}
- return 1.0f - res;
+ return res;
}
#if defined(OV_SIMD_AVX512)
@@ -46,7 +46,7 @@ static float l2_sqr_avx512(const void* v1, const void* v2, const void* params) {
res += diff * diff;
}
- return 1.0f - res;
+ return res;
}
#endif
@@ -84,7 +84,7 @@ static float l2_sqr_avx(const void* v1, const void* v2, const void* params) {
res += diff * diff;
}
- return 1.0f - res;
+ return res;
}
#endif
@@ -117,7 +117,7 @@ static float l2_sqr_sse(const void* v1, const void* v2, const void* params) {
res += diff * diff;
}
- return 1.0f - res;
+ return res;
}
#endif
diff --git a/src/index/detail/vector/sparse_retrieval/sparse_data_holder.h b/src/index/detail/vector/sparse_retrieval/sparse_data_holder.h
index 382fcd73b..03720bc1d 100644
--- a/src/index/detail/vector/sparse_retrieval/sparse_data_holder.h
+++ b/src/index/detail/vector/sparse_retrieval/sparse_data_holder.h
@@ -142,7 +142,7 @@ class SparseDataHolder {
float sparse_head_dot_product_logit(const SparseDatapointView& x,
const DocID docid) {
- return 1.0f - sparse_holder_.sparse_dot_product_reduce(x, docid);
+ return sparse_holder_.sparse_dot_product_reduce(x, docid);
}
size_t rows() {
diff --git a/tests/vectordb/test_recall.py b/tests/vectordb/test_recall.py
index cdc5c39e7..b32300694 100644
--- a/tests/vectordb/test_recall.py
+++ b/tests/vectordb/test_recall.py
@@ -236,6 +236,189 @@ class TestRecall(unittest.TestCase):
)
print("✓ IP Recall verified")
+ def test_search_limit_zero(self):
+ """Test search with limit=0 returns empty result without error"""
+ print("\n=== Test: Search limit=0 ===")
+
+ dim = 8
+ meta_data = {
+ "CollectionName": "test_limit_zero",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},
+ ],
+ }
+
+ collection = self.register_collection(
+ get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)
+ )
+
+ data = [{"id": 0, "vector": [0.1] * dim}, {"id": 1, "vector": [0.2] * dim}]
+ collection.upsert_data(data)
+
+ collection.create_index(
+ "idx_limit_zero",
+ {
+ "IndexName": "idx_limit_zero",
+ "VectorIndex": {"IndexType": "flat", "Distance": "l2"},
+ },
+ )
+
+ result = collection.search_by_vector("idx_limit_zero", dense_vector=[0.1] * dim, limit=0)
+
+ self.assertEqual(len(result.data), 0, "limit=0 should return empty results")
+ print("✓ limit=0 returns empty results")
+
+ def test_sparse_vector_recall(self):
+ """Test sparse vector recall in hybrid index"""
+ print("\n=== Test: Sparse Vector Recall ===")
+
+ dim = 4
+ meta_data = {
+ "CollectionName": "test_sparse_recall",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},
+ {"FieldName": "sparse_vector", "FieldType": "sparse_vector"},
+ ],
+ }
+
+ collection = self.register_collection(
+ get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)
+ )
+
+ dense_vec = [0.1] * dim
+ data = [
+ {"id": 0, "vector": dense_vec, "sparse_vector": {"t1": 1.0}},
+ {"id": 1, "vector": dense_vec, "sparse_vector": {"t1": 0.5}},
+ {"id": 2, "vector": dense_vec, "sparse_vector": {"t2": 1.0}},
+ ]
+ collection.upsert_data(data)
+
+ collection.create_index(
+ "idx_sparse",
+ {
+ "IndexName": "idx_sparse",
+ "VectorIndex": {
+ "IndexType": "flat_hybrid",
+ "Distance": "ip",
+ "SearchWithSparseLogitAlpha": 1.0,
+ },
+ },
+ )
+
+ result = collection.search_by_vector(
+ "idx_sparse",
+ dense_vector=dense_vec,
+ sparse_vector={"t1": 1.0},
+ limit=3,
+ )
+ result_ids = [item.id for item in result.data]
+
+ self.assertEqual(result_ids, [0, 1, 2], "Sparse ranking should match dot product order")
+ print("✓ Sparse vector recall verified", result)
+
+ def test_sparse_vector_recall_l2(self):
+ """Test sparse vector recall with L2 distance in hybrid index"""
+ print("\n=== Test: Sparse Vector Recall (L2) ===")
+
+ dim = 4
+ meta_data = {
+ "CollectionName": "test_sparse_recall_l2",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},
+ {"FieldName": "sparse_vector", "FieldType": "sparse_vector"},
+ ],
+ }
+
+ collection = self.register_collection(
+ get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)
+ )
+
+ dense_vec = [0.1] * dim
+ data = [
+ {"id": 0, "vector": dense_vec, "sparse_vector": {"t1": 1.0}},
+ {"id": 1, "vector": dense_vec, "sparse_vector": {"t1": 0.5}},
+ {"id": 2, "vector": dense_vec, "sparse_vector": {"t2": 1.0}},
+ ]
+ collection.upsert_data(data)
+
+ collection.create_index(
+ "idx_sparse_l2",
+ {
+ "IndexName": "idx_sparse_l2",
+ "VectorIndex": {
+ "IndexType": "flat_hybrid",
+ "Distance": "l2",
+ "SearchWithSparseLogitAlpha": 1.0,
+ },
+ },
+ )
+
+ result = collection.search_by_vector(
+ "idx_sparse_l2",
+ dense_vector=dense_vec,
+ sparse_vector={"t1": 1.0},
+ limit=3,
+ )
+ result_ids = [item.id for item in result.data]
+
+ self.assertEqual(result_ids, [0, 1, 2], "Sparse L2 ranking should favor closest match")
+ print("✓ Sparse vector recall (L2) verified", result)
+
+ def test_hybrid_dense_sparse_mix(self):
+ """Test hybrid scoring combines dense and sparse signals"""
+ print("\n=== Test: Hybrid Dense+Sparse Mix ===")
+
+ dim = 4
+ meta_data = {
+ "CollectionName": "test_hybrid_mix",
+ "Fields": [
+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},
+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},
+ {"FieldName": "sparse_vector", "FieldType": "sparse_vector"},
+ ],
+ }
+
+ collection = self.register_collection(
+ get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)
+ )
+
+ data = [
+ {"id": 0, "vector": [0.9, 0.0, 0.0, 0.0], "sparse_vector": {"t1": 0.1}},
+ {"id": 1, "vector": [0.2, 0.0, 0.0, 0.0], "sparse_vector": {"t1": 1.0}},
+ {"id": 2, "vector": [0.1, 0.0, 0.0, 0.0], "sparse_vector": {"t1": 0.8}},
+ ]
+ collection.upsert_data(data)
+
+ collection.create_index(
+ "idx_hybrid_mix",
+ {
+ "IndexName": "idx_hybrid_mix",
+ "VectorIndex": {
+ "IndexType": "flat_hybrid",
+ "Distance": "ip",
+ "SearchWithSparseLogitAlpha": 0.5,
+ },
+ },
+ )
+
+ result = collection.search_by_vector(
+ "idx_hybrid_mix",
+ dense_vector=[1.0, 0.0, 0.0, 0.0],
+ sparse_vector={"t1": 1.0},
+ limit=3,
+ )
+ result_ids = [item.id for item in result.data]
+
+ self.assertEqual(
+ result_ids,
+ [1, 0, 2],
+ "Hybrid ranking should reflect combined dense and sparse scores",
+ )
+ print("✓ Hybrid dense+sparse mix verified")
+
if __name__ == "__main__":
unittest.main()