Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: add search_with_sparse_logit_alpha (#71)
kkkwjx9368d838f794
diff --git a/docs/en/configuration/configuration.md b/docs/en/configuration/configuration.mdindex d15e0f9fb..5b29e8cc0 100644--- a/docs/en/configuration/configuration.md+++ b/docs/en/configuration/configuration.md@@ -207,6 +207,9 @@ client = ov.AsyncOpenViking(config=config)}```+Notes:+- `storage.vectordb.sparse_weight` controls hybrid (dense + sparse) indexing/search. It only takes effect when you use a hybrid index; set it > 0 to enable sparse signals.+## Related Documentation- [Embedding Configuration](./embedding.md) - Embedding setupdiff --git a/docs/zh/configuration/configuration.md b/docs/zh/configuration/configuration.mdindex 93f5e21ff..bef6327b5 100644--- a/docs/zh/configuration/configuration.md+++ b/docs/zh/configuration/configuration.md@@ -294,6 +294,9 @@ client = ov.AsyncOpenViking(config=config)}```+说明:+- `storage.vectordb.sparse_weight` 用于混合(dense + sparse)索引/检索的权重,仅在使用 hybrid 索引时生效;设置为 > 0 才会启用 sparse 信号。+## 相关文档- [Embedding 配置](./embedding.md) - Embedding 设置diff --git a/openviking/storage/queuefs/embedding_queue.py b/openviking/storage/queuefs/embedding_queue.pyindex 6af3d3d0b..b6dc771ae 100644--- a/openviking/storage/queuefs/embedding_queue.py+++ b/openviking/storage/queuefs/embedding_queue.py@@ -16,7 +16,7 @@ class EmbeddingQueue(NamedQueue):Supports direct enqueue and dequeue of EmbeddingMsg objects."""- async def enqueue(self, msg: EmbeddingMsg | None) -> str:+ async def enqueue(self, msg: Optional[EmbeddingMsg]) -> str:"""Serialize EmbeddingMsg object and store in queue."""if msg is None:logger.warning("Embedding message is None, skipping enqueuing")@@ -28,17 +28,26 @@ class EmbeddingQueue(NamedQueue):data_dict = await super().dequeue()if not data_dict:return None- if "data" in data_dict and isinstance(data_dict["data"], str):- try:- return EmbeddingMsg.from_json(data_dict["data"])- except Exception as e:- logger.debug(f"[EmbeddingQueue] Failed to parse message data: {e}")- return None+ if "data" in data_dict:+ if isinstance(data_dict["data"], str):+ try:+ return EmbeddingMsg.from_json(data_dict["data"])+ except Exception as e:+ logger.debug(f"[EmbeddingQueue] Failed to parse message data: {e}")+ return None+ elif isinstance(data_dict["data"], dict):+ try:+ return EmbeddingMsg.from_dict(data_dict["data"])+ except Exception as e:+ logger.debug(+ f"[EmbeddingQueue] Failed to create EmbeddingMsg from data dict: {e}"+ )+ return None+# Otherwise try to convert directly from dicttry:return EmbeddingMsg.from_dict(data_dict)- except Exception as e:- logger.debug(f"[EmbeddingQueue] Failed to create EmbeddingMsg from dict: {e}")+ except Exception:return Noneasync def peek(self) -> Optional[EmbeddingMsg]:@@ -47,11 +56,17 @@ class EmbeddingQueue(NamedQueue):if not data_dict:return None- if "data" in data_dict and isinstance(data_dict["data"], str):- try:- return EmbeddingMsg.from_json(data_dict["data"])- except Exception:- return None+ if "data" in data_dict:+ if isinstance(data_dict["data"], str):+ try:+ return EmbeddingMsg.from_json(data_dict["data"])+ except Exception:+ return None+ elif isinstance(data_dict["data"], dict):+ try:+ return EmbeddingMsg.from_dict(data_dict["data"])+ except Exception:+ return Nonetry:return EmbeddingMsg.from_dict(data_dict)diff --git a/openviking/storage/vectordb/meta/index_meta.py b/openviking/storage/vectordb/meta/index_meta.pyindex 3c2253e72..a7635e0f9 100644--- a/openviking/storage/vectordb/meta/index_meta.py+++ b/openviking/storage/vectordb/meta/index_meta.py@@ -112,9 +112,6 @@ class IndexMeta:vector_index["SearchWithSparseLogitAlpha"] = inner_meta["VectorIndex"].get("SearchWithSparseLogitAlpha", 0.5)- vector_index["IndexWithSparseLogitAlpha"] = inner_meta["VectorIndex"].get(- "IndexWithSparseLogitAlpha", 0.5- )if "flat" in inner_meta["VectorIndex"]["IndexType"]:vector_index["IndexType"] = "flat"if "EnableSparse" in inner_meta["VectorIndex"]:@@ -123,10 +120,6 @@ class IndexMeta:vector_index["SearchWithSparseLogitAlpha"] = inner_meta["VectorIndex"]["SearchWithSparseLogitAlpha"]- if "IndexWithSparseLogitAlpha" in inner_meta["VectorIndex"]:- vector_index["IndexWithSparseLogitAlpha"] = inner_meta["VectorIndex"][- "IndexWithSparseLogitAlpha"- ]inner_meta["VectorIndex"] = vector_indexinner_meta["CollectionName"] = collection_meta.collection_namediff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex ff656f0bc..91c266c02 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -77,8 +77,9 @@ class VikingVectorIndexBackend(VikingDBInterface):"""init_cpp_logging()- self.vector_dim = config.vector_dim+ self.vector_dim = config.dimensionself.distance_metric = config.distance_metric+ self.sparse_weight = config.sparse_weightif config.backend == "volcengine":if not (@@ -242,15 +243,19 @@ class VikingVectorIndexBackend(VikingDBInterface):scalar_index_fields.append(field_name)# Create default index for the collection+ use_sparse = self.sparse_weight > 0.0index_meta = {"IndexName": self.DEFAULT_INDEX_NAME,"VectorIndex": {- "IndexType": "flat_hybrid",+ "IndexType": "flat_hybrid" if use_sparse else "flat","Distance": distance,"Quant": "int8",},"ScalarIndex": scalar_index_fields,}+ if use_sparse:+ index_meta["VectorIndex"]["EnableSparse"] = True+ index_meta["VectorIndex"]["SearchWithSparseLogitAlpha"] = self.sparse_weightlogger.info(f"Creating index with meta: {index_meta}")collection.create_index(self.DEFAULT_INDEX_NAME, index_meta)diff --git a/openviking/utils/config/open_viking_config.py b/openviking/utils/config/open_viking_config.pyindex ada70a23d..6d6c646d0 100644--- a/openviking/utils/config/open_viking_config.py+++ b/openviking/utils/config/open_viking_config.py@@ -301,6 +301,10 @@ def initialize_openviking_config(config.storage.vectordb.backend = "http"config.storage.vectordb.url = vectordb_url+ # Ensure vector dimension is synced if not set in storage+ if config.storage.vectordb.dimension == 0:+ config.storage.vectordb.dimension = config.embedding.dimension+# Validate configurationif not is_valid_openviking_config(config):raise ValueError("Invalid OpenViking configuration")diff --git a/openviking/utils/config/vectordb_config.py b/openviking/utils/config/vectordb_config.pyindex dc32c7e42..9af43fdea 100644--- a/openviking/utils/config/vectordb_config.py+++ b/openviking/utils/config/vectordb_config.py@@ -43,11 +43,19 @@ class VectorDBBackendConfig(BaseModel):description="Distance metric for vector similarity search (e.g., 'cosine', 'l2', 'ip')",)- vector_dim: int = Field(+ dimension: int = Field(default=0,description="Dimension of vector embeddings",)+ sparse_weight: float = Field(+ default=0.0,+ description=(+ "Sparse weight for hybrid vector search. "+ "When > 0, sparse vectors are used for index build and search."+ ),+ )+volcengine: Optional[VolcengineConfig] = Field(default_factory=lambda: VolcengineConfig(),description="Volcengine VikingDB configuration for 'volcengine' type",diff --git a/src/index/detail/meta/vector_index_meta.cpp b/src/index/detail/meta/vector_index_meta.cppindex 238b1d11b..a182bde0d 100644--- a/src/index/detail/meta/vector_index_meta.cpp+++ b/src/index/detail/meta/vector_index_meta.cpp@@ -48,10 +48,6 @@ int VectorIndexMeta::init_from_json(const JsonValue& json) {search_with_sparse_logit_alpha =json["SearchWithSparseLogitAlpha"].GetFloat();}- if (json.HasMember("IndexWithSparseLogitAlpha")) {- index_with_sparse_logit_alpha =- json["IndexWithSparseLogitAlpha"].GetFloat();- }return 0;}@@ -102,9 +98,7 @@ int VectorIndexMeta::save_to_json(JsonPrettyWriter& writer) {writer.Bool(enable_sparse);writer.Key("SearchWithSparseLogitAlpha");writer.Double(search_with_sparse_logit_alpha);- writer.Key("IndexWithSparseLogitAlpha");- writer.Double(index_with_sparse_logit_alpha);return 0;}-} // namespace vectordb\ No newline at end of file+} // namespace vectordbdiff --git a/src/index/detail/vector/common/bruteforce.h b/src/index/detail/vector/common/bruteforce.hindex 7069f5558..2a065acfb 100644--- a/src/index/detail/vector/common/bruteforce.h+++ b/src/index/detail/vector/common/bruteforce.h@@ -178,6 +178,11 @@ class BruteforceSearch {std::vector<float>& scores) const {if (!query_data)return;+ if (k == 0) {+ labels.clear();+ scores.clear();+ return;+ }if (current_count_ == 0)return;@@ -204,17 +209,8 @@ class BruteforceSearch {continue;}- float dist = dist_func(encoded_query.data(), ptr, dist_params);-- if (sparse_index_) {- float sparse_dist = 0;- if (query_sparse_view) {- sparse_dist =- sparse_index_->sparse_head_output(*query_sparse_view, i);- }- dist = dist * (1 - meta_->search_with_sparse_logit_alpha) +- sparse_dist * meta_->search_with_sparse_logit_alpha;- }+ float dist = compute_score(encoded_query.data(), ptr, query_sparse_view,+ i, dist_func, dist_params);uint64_t label;std::memcpy(&label, ptr + vector_byte_size_, sizeof(uint64_t));@@ -307,6 +303,7 @@ class BruteforceSearch {private:void setup_metric() {+ reverse_query_score_ = (meta_->distance_type == "l2");if (meta_->quantization_type == "int8") {if (meta_->distance_type == "l2")space_ = std::make_unique<L2SpaceInt8>(meta_->dimension);@@ -357,6 +354,26 @@ class BruteforceSearch {return view;}+ float compute_score(+ const void* encoded_query, const char* data_ptr,+ const std::shared_ptr<SparseDatapointView>& query_sparse_view,+ size_t idx, MetricFunc<float> dist_func,+ void* dist_params) const {+ float dense_raw = dist_func(encoded_query, data_ptr, dist_params);+ float dense_score =+ reverse_query_score_ ? (1.0f - dense_raw) : dense_raw;+ if (!sparse_index_ || !query_sparse_view ||+ meta_->search_with_sparse_logit_alpha <= 0.0f) {+ return dense_score;+ }+ float sparse_raw =+ sparse_index_->sparse_head_output(*query_sparse_view, idx);+ float sparse_score =+ reverse_query_score_ ? (1.0f - sparse_raw) : sparse_raw;+ float alpha = meta_->search_with_sparse_logit_alpha;+ return dense_score * (1.0f - alpha) + sparse_score * alpha;+ }+std::shared_ptr<BruteForceMeta> meta_;char* data_buffer_ = nullptr;size_t capacity_ = 0;@@ -370,6 +387,7 @@ class BruteforceSearch {std::unique_ptr<VectorSpace<float>> space_;std::unique_ptr<VectorQuantizer> quantizer_;std::unique_ptr<SparseDataHolder> sparse_index_;+ bool reverse_query_score_ = false;};} // namespace vectordbdiff --git a/src/index/detail/vector/common/space_int8.h b/src/index/detail/vector/common/space_int8.hindex 9e15acf5f..36dda0e9e 100644--- a/src/index/detail/vector/common/space_int8.h+++ b/src/index/detail/vector/common/space_int8.h@@ -153,7 +153,7 @@ static float l2_distance_int8(const void* v1, const void* v2,float real_ip = static_cast<float>(ip) * scale1 * scale2;float dist = norm_sq1 + norm_sq2 - 2.0f * real_ip;- return 1.0f - std::max(0.0f, dist);+ return std::max(0.0f, dist);}class InnerProductSpaceInt8 : public VectorSpace<float> {diff --git a/src/index/detail/vector/common/space_l2.h b/src/index/detail/vector/common/space_l2.hindex 7b0e50b42..a6bef28d6 100644--- a/src/index/detail/vector/common/space_l2.h+++ b/src/index/detail/vector/common/space_l2.h@@ -18,7 +18,7 @@ static float l2_sqr_ref(const void* v1, const void* v2, const void* params) {float diff = pv1[i] - pv2[i];res += diff * diff;}- return 1.0f - res;+ return res;}#if defined(OV_SIMD_AVX512)@@ -46,7 +46,7 @@ static float l2_sqr_avx512(const void* v1, const void* v2, const void* params) {res += diff * diff;}- return 1.0f - res;+ return res;}#endif@@ -84,7 +84,7 @@ static float l2_sqr_avx(const void* v1, const void* v2, const void* params) {res += diff * diff;}- return 1.0f - res;+ return res;}#endif@@ -117,7 +117,7 @@ static float l2_sqr_sse(const void* v1, const void* v2, const void* params) {res += diff * diff;}- return 1.0f - res;+ return res;}#endifdiff --git a/src/index/detail/vector/sparse_retrieval/sparse_data_holder.h b/src/index/detail/vector/sparse_retrieval/sparse_data_holder.hindex 382fcd73b..03720bc1d 100644--- a/src/index/detail/vector/sparse_retrieval/sparse_data_holder.h+++ b/src/index/detail/vector/sparse_retrieval/sparse_data_holder.h@@ -142,7 +142,7 @@ class SparseDataHolder {float sparse_head_dot_product_logit(const SparseDatapointView& x,const DocID docid) {- return 1.0f - sparse_holder_.sparse_dot_product_reduce(x, docid);+ return sparse_holder_.sparse_dot_product_reduce(x, docid);}size_t rows() {diff --git a/tests/vectordb/test_recall.py b/tests/vectordb/test_recall.pyindex cdc5c39e7..b32300694 100644--- a/tests/vectordb/test_recall.py+++ b/tests/vectordb/test_recall.py@@ -236,6 +236,189 @@ class TestRecall(unittest.TestCase):)print("✓ IP Recall verified")+ def test_search_limit_zero(self):+ """Test search with limit=0 returns empty result without error"""+ print("\n=== Test: Search limit=0 ===")++ dim = 8+ meta_data = {+ "CollectionName": "test_limit_zero",+ "Fields": [+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},+ ],+ }++ collection = self.register_collection(+ get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)+ )++ data = [{"id": 0, "vector": [0.1] * dim}, {"id": 1, "vector": [0.2] * dim}]+ collection.upsert_data(data)++ collection.create_index(+ "idx_limit_zero",+ {+ "IndexName": "idx_limit_zero",+ "VectorIndex": {"IndexType": "flat", "Distance": "l2"},+ },+ )++ result = collection.search_by_vector("idx_limit_zero", dense_vector=[0.1] * dim, limit=0)++ self.assertEqual(len(result.data), 0, "limit=0 should return empty results")+ print("✓ limit=0 returns empty results")++ def test_sparse_vector_recall(self):+ """Test sparse vector recall in hybrid index"""+ print("\n=== Test: Sparse Vector Recall ===")++ dim = 4+ meta_data = {+ "CollectionName": "test_sparse_recall",+ "Fields": [+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},+ {"FieldName": "sparse_vector", "FieldType": "sparse_vector"},+ ],+ }++ collection = self.register_collection(+ get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)+ )++ dense_vec = [0.1] * dim+ data = [+ {"id": 0, "vector": dense_vec, "sparse_vector": {"t1": 1.0}},+ {"id": 1, "vector": dense_vec, "sparse_vector": {"t1": 0.5}},+ {"id": 2, "vector": dense_vec, "sparse_vector": {"t2": 1.0}},+ ]+ collection.upsert_data(data)++ collection.create_index(+ "idx_sparse",+ {+ "IndexName": "idx_sparse",+ "VectorIndex": {+ "IndexType": "flat_hybrid",+ "Distance": "ip",+ "SearchWithSparseLogitAlpha": 1.0,+ },+ },+ )++ result = collection.search_by_vector(+ "idx_sparse",+ dense_vector=dense_vec,+ sparse_vector={"t1": 1.0},+ limit=3,+ )+ result_ids = [item.id for item in result.data]++ self.assertEqual(result_ids, [0, 1, 2], "Sparse ranking should match dot product order")+ print("✓ Sparse vector recall verified", result)++ def test_sparse_vector_recall_l2(self):+ """Test sparse vector recall with L2 distance in hybrid index"""+ print("\n=== Test: Sparse Vector Recall (L2) ===")++ dim = 4+ meta_data = {+ "CollectionName": "test_sparse_recall_l2",+ "Fields": [+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},+ {"FieldName": "sparse_vector", "FieldType": "sparse_vector"},+ ],+ }++ collection = self.register_collection(+ get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)+ )++ dense_vec = [0.1] * dim+ data = [+ {"id": 0, "vector": dense_vec, "sparse_vector": {"t1": 1.0}},+ {"id": 1, "vector": dense_vec, "sparse_vector": {"t1": 0.5}},+ {"id": 2, "vector": dense_vec, "sparse_vector": {"t2": 1.0}},+ ]+ collection.upsert_data(data)++ collection.create_index(+ "idx_sparse_l2",+ {+ "IndexName": "idx_sparse_l2",+ "VectorIndex": {+ "IndexType": "flat_hybrid",+ "Distance": "l2",+ "SearchWithSparseLogitAlpha": 1.0,+ },+ },+ )++ result = collection.search_by_vector(+ "idx_sparse_l2",+ dense_vector=dense_vec,+ sparse_vector={"t1": 1.0},+ limit=3,+ )+ result_ids = [item.id for item in result.data]++ self.assertEqual(result_ids, [0, 1, 2], "Sparse L2 ranking should favor closest match")+ print("✓ Sparse vector recall (L2) verified", result)++ def test_hybrid_dense_sparse_mix(self):+ """Test hybrid scoring combines dense and sparse signals"""+ print("\n=== Test: Hybrid Dense+Sparse Mix ===")++ dim = 4+ meta_data = {+ "CollectionName": "test_hybrid_mix",+ "Fields": [+ {"FieldName": "id", "FieldType": "int64", "IsPrimaryKey": True},+ {"FieldName": "vector", "FieldType": "vector", "Dim": dim},+ {"FieldName": "sparse_vector", "FieldType": "sparse_vector"},+ ],+ }++ collection = self.register_collection(+ get_or_create_local_collection(meta_data=meta_data, path=TEST_DB_PATH)+ )++ data = [+ {"id": 0, "vector": [0.9, 0.0, 0.0, 0.0], "sparse_vector": {"t1": 0.1}},+ {"id": 1, "vector": [0.2, 0.0, 0.0, 0.0], "sparse_vector": {"t1": 1.0}},+ {"id": 2, "vector": [0.1, 0.0, 0.0, 0.0], "sparse_vector": {"t1": 0.8}},+ ]+ collection.upsert_data(data)++ collection.create_index(+ "idx_hybrid_mix",+ {+ "IndexName": "idx_hybrid_mix",+ "VectorIndex": {+ "IndexType": "flat_hybrid",+ "Distance": "ip",+ "SearchWithSparseLogitAlpha": 0.5,+ },+ },+ )++ result = collection.search_by_vector(+ "idx_hybrid_mix",+ dense_vector=[1.0, 0.0, 0.0, 0.0],+ sparse_vector={"t1": 1.0},+ limit=3,+ )+ result_ids = [item.id for item in result.data]++ self.assertEqual(+ result_ids,+ [1, 0, 2],+ "Hybrid ranking should reflect combined dense and sparse scores",+ )+ print("✓ Hybrid dense+sparse mix verified")+if __name__ == "__main__":unittest.main()