Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Review changes + unit test for rank fusion

ozanarmaganb10508a876c3
diff --git a/src/index.cpp b/src/index.cpp
index e4feba90..faf0d973 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -3021,6 +3021,10 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
if(has_text_match) {
+ // For hybrid search, we need to give weight to text match and vector search
+ constexpr float TEXT_MATCH_WEIGHT = 0.7;
+ constexpr float VECTOR_SEARCH_WEIGHT = 1.0 - TEXT_MATCH_WEIGHT;
+
VectorFilterFunctor filterFunctor(filter_result.docs, filter_result.count);
auto& field_vector_index = vector_index.at(vector_query.field_name);
std::vector<std::pair<float, size_t>> dist_labels;
@@ -3040,22 +3044,23 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
auto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :
dist_label.first;
- auto score = (1.0 - vec_dist_score) * 100.0;
- vec_results.emplace_back(seq_id, score);
+ vec_results.emplace_back(seq_id, vec_dist_score);
}
std::sort(vec_results.begin(), vec_results.end(), [](const auto& a, const auto& b) {
- return a.second > b.second;
+ return a.second < b.second;
});
topster->sort();
// Reciprocal rank fusion
+ // Score is sum of (1 / rank_of_document) * WEIGHT from each list (text match and vector search)
for(uint32_t i = 0; i < topster->size; i++) {
auto result = topster->getKV(i);
if(result->match_score_index < 0 || result->match_score_index > 2) {
continue;
}
- result->scores[result->match_score_index] = (1.0 / (i + 1)) * INT64_MAX * 0.7;
+ // (1 / rank_of_document) * WEIGHT)
+ result->scores[result->match_score_index] = float_to_int64_t((1.0 / (i + 1)) * TEXT_MATCH_WEIGHT);
}
for(int i = 0; i < vec_results.size(); i++) {
@@ -3066,10 +3071,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
if(result_it != topster->kv_map.end()&& result_it->second->match_score_index >= 0 && result_it->second->match_score_index <= 2) {
auto result = result_it->second;
- result->scores[result->match_score_index] += (1.0 / (i + 1)) * INT64_MAX * 0.3;
+ // old_score + (1 / rank_of_document) * WEIGHT)
+ result->scores[result->match_score_index] = float_to_int64_t((int64_t_to_float(result->scores[result->match_score_index])) + ((1.0 / (i + 1)) * VECTOR_SEARCH_WEIGHT));
} else {
int64_t scores[3] = {0};
- scores[0] = (1.0 / (i + 1)) * INT64_MAX * 0.3;
+ // (1 / rank_of_document) * WEIGHT)
+ scores[0] = float_to_int64_t((1.0 / (i + 1)) * VECTOR_SEARCH_WEIGHT);
int64_t match_score_index = 0;
KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores);
topster->add(&kv);
diff --git a/src/text_embedder.cpp b/src/text_embedder.cpp
index 285ebd71..05ffa59e 100644
--- a/src/text_embedder.cpp
+++ b/src/text_embedder.cpp
@@ -7,13 +7,12 @@
#include <filesystem>
TextEmbedder::TextEmbedder(const std::string& model_path) {
-
// create environment
Ort::SessionOptions session_options;
std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_path);
LOG(INFO) << "Loading model from: " << abs_path;
session_ = new Ort::Session(env_, abs_path.c_str(), session_options);
- std::ifstream stream("vocab.txt");
+ std::ifstream stream(TextEmbedderManager::get_absolute_vocab_path());
std::stringstream ss;
ss << stream.rdbuf();
auto vocab_ = ss.str();
@@ -85,7 +84,6 @@ std::vector<float> TextEmbedder::Embed(const std::string& text) {
// print output tensor shape
auto shape = output_tensor[0].GetTensorTypeAndShapeInfo().GetShape();
- //LOG(INFO) << "Output tensor size: " << shape[0] << " x " << shape[1] << " x " << shape[2];
for (int i = 0; i < shape[1]; i++) {
std::vector<float> temp;
for (int j = 0; j < shape[2]; j++) {
@@ -93,11 +91,8 @@ std::vector<float> TextEmbedder::Embed(const std::string& text) {
}
output.push_back(temp);
}
- //LOG(INFO) << "Mean pooling";
auto pooled_output = mean_pooling(output);
-
-
return pooled_output;
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 98ebe805..f7743614 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -4700,22 +4700,18 @@ TEST_F(CollectionTest, HybridSearch) {
HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::get_absolute_vocab_path());
auto op = collectionManager.create_collection(schema);
- LOG(INFO) << collectionManager.get_collection("objects")->get_summary_json();
ASSERT_TRUE(op.ok());
Collection* coll = op.get();
nlohmann::json object;
object["name"] = "apple";
auto add_op = coll->add(object.dump());
- LOG(INFO) << "hybrid search";
ASSERT_TRUE(add_op.ok());
ASSERT_EQ("apple", add_op.get()["name"]);
ASSERT_EQ(384, add_op.get()["embedding"].size());
spp::sparse_hash_set<std::string> dummy_include_exclude;
- LOG(INFO) << "hybrid search 2";
auto search_res_op = coll->search("apple", {"name","embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");
- LOG(INFO) << "hybrid search 3";
ASSERT_TRUE(search_res_op.ok());
auto search_res = search_res_op.get();
ASSERT_EQ(1, search_res["found"].get<size_t>());
@@ -4755,6 +4751,80 @@ TEST_F(CollectionTest, EmbedFielsTest) {
ASSERT_EQ("apple", object["name"]);
ASSERT_EQ(384, object["embedding"].get<std::vector<float>>().size());
+ // delete models folder
+ system("rm -rf ./models");
+}
+
+TEST_F(CollectionTest, HybridSearchRankFusionTest) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "embedding", "type":"float[]", "create_from": ["name"]}
+ ]
+ })"_json;
+
+ TextEmbedderManager::model_dir = "./models/";
+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));
+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::get_absolute_vocab_path());
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll = op.get();
+ nlohmann::json object;
+ object["name"] = "butter";
+ auto add_op = coll->add(object.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ object["name"] = "butterball";
+ add_op = coll->add(object.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ object["name"] = "butterfly";
+ add_op = coll->add(object.dump());
+ ASSERT_TRUE(add_op.ok());
+
+ spp::sparse_hash_set<std::string> dummy_include_exclude;
+ auto search_res_op = coll->search("butter", {"embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");
+ ASSERT_TRUE(search_res_op.ok());
+ auto search_res = search_res_op.get();
+ ASSERT_EQ(3, search_res["found"].get<size_t>());
+ ASSERT_EQ(3, search_res["hits"].size());
+ // Vector search order:
+ // 1. butter
+ // 2. butterball
+ // 3. butterfly
+ ASSERT_EQ("butter", search_res["hits"][0]["document"]["name"].get<std::string>());
+ ASSERT_EQ("butterball", search_res["hits"][1]["document"]["name"].get<std::string>());
+ ASSERT_EQ("butterfly", search_res["hits"][2]["document"]["name"].get<std::string>());
+
+
+ search_res_op = coll->search("butter", {"name"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");
+ ASSERT_TRUE(search_res_op.ok());
+ search_res = search_res_op.get();
+ ASSERT_EQ(3, search_res["found"].get<size_t>());
+ ASSERT_EQ(3, search_res["hits"].size());
+ // Keyword search order:
+ // 1. butter
+ // 2. butterfly
+ // 3. butterball
+ ASSERT_EQ("butter", search_res["hits"][0]["document"]["name"].get<std::string>());
+ ASSERT_EQ("butterfly", search_res["hits"][1]["document"]["name"].get<std::string>());
+ ASSERT_EQ("butterball", search_res["hits"][2]["document"]["name"].get<std::string>());
+
+ search_res_op = coll->search("butter", {"name","embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");
+ ASSERT_TRUE(search_res_op.ok());
+ search_res = search_res_op.get();
+ ASSERT_EQ(3, search_res["found"].get<size_t>());
+ ASSERT_EQ(3, search_res["hits"].size());
+ // Hybrid search with rank fusion order:
+ // 1. butter (1/1 * 0.7) + (1/1 * 0.3) = 1
+ // 2. butterfly (1/2 * 0.7) + (1/3 * 0.3) = 0.567
+ // 3. butterball (1/3 * 0.7) + (1/2 * 0.3) = 0.533
+ ASSERT_EQ("butter", search_res["hits"][0]["document"]["name"].get<std::string>());
+ ASSERT_EQ("butterfly", search_res["hits"][1]["document"]["name"].get<std::string>());
+ ASSERT_EQ("butterball", search_res["hits"][2]["document"]["name"].get<std::string>());
+
// delete models folder
system("rm -rf ./models");
}
\ No newline at end of file