Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Review changes + unit test for rank fusion
ozanarmaganb10508a876c3
diff --git a/src/index.cpp b/src/index.cppindex e4feba90..faf0d973 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -3021,6 +3021,10 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}if(has_text_match) {+ // For hybrid search, we need to give weight to text match and vector search+ constexpr float TEXT_MATCH_WEIGHT = 0.7;+ constexpr float VECTOR_SEARCH_WEIGHT = 1.0 - TEXT_MATCH_WEIGHT;+VectorFilterFunctor filterFunctor(filter_result.docs, filter_result.count);auto& field_vector_index = vector_index.at(vector_query.field_name);std::vector<std::pair<float, size_t>> dist_labels;@@ -3040,22 +3044,23 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consauto vec_dist_score = (field_vector_index->distance_type == cosine) ? std::abs(dist_label.first) :dist_label.first;- auto score = (1.0 - vec_dist_score) * 100.0;- vec_results.emplace_back(seq_id, score);+ vec_results.emplace_back(seq_id, vec_dist_score);}std::sort(vec_results.begin(), vec_results.end(), [](const auto& a, const auto& b) {- return a.second > b.second;+ return a.second < b.second;});topster->sort();// Reciprocal rank fusion+ // Score is sum of (1 / rank_of_document) * WEIGHT from each list (text match and vector search)for(uint32_t i = 0; i < topster->size; i++) {auto result = topster->getKV(i);if(result->match_score_index < 0 || result->match_score_index > 2) {continue;}- result->scores[result->match_score_index] = (1.0 / (i + 1)) * INT64_MAX * 0.7;+ // (1 / rank_of_document) * WEIGHT)+ result->scores[result->match_score_index] = float_to_int64_t((1.0 / (i + 1)) * TEXT_MATCH_WEIGHT);}for(int i = 0; i < vec_results.size(); i++) {@@ -3066,10 +3071,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consif(result_it != topster->kv_map.end()&& result_it->second->match_score_index >= 0 && result_it->second->match_score_index <= 2) {auto result = result_it->second;- result->scores[result->match_score_index] += (1.0 / (i + 1)) * INT64_MAX * 0.3;+ // old_score + (1 / rank_of_document) * WEIGHT)+ result->scores[result->match_score_index] = float_to_int64_t((int64_t_to_float(result->scores[result->match_score_index])) + ((1.0 / (i + 1)) * VECTOR_SEARCH_WEIGHT));} else {int64_t scores[3] = {0};- scores[0] = (1.0 / (i + 1)) * INT64_MAX * 0.3;+ // (1 / rank_of_document) * WEIGHT)+ scores[0] = float_to_int64_t((1.0 / (i + 1)) * VECTOR_SEARCH_WEIGHT);int64_t match_score_index = 0;KV kv(searched_queries.size(), doc_id, doc_id, match_score_index, scores);topster->add(&kv);diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex 285ebd71..05ffa59e 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -7,13 +7,12 @@#include <filesystem>TextEmbedder::TextEmbedder(const std::string& model_path) {-// create environmentOrt::SessionOptions session_options;std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_path);LOG(INFO) << "Loading model from: " << abs_path;session_ = new Ort::Session(env_, abs_path.c_str(), session_options);- std::ifstream stream("vocab.txt");+ std::ifstream stream(TextEmbedderManager::get_absolute_vocab_path());std::stringstream ss;ss << stream.rdbuf();auto vocab_ = ss.str();@@ -85,7 +84,6 @@ std::vector<float> TextEmbedder::Embed(const std::string& text) {// print output tensor shapeauto shape = output_tensor[0].GetTensorTypeAndShapeInfo().GetShape();- //LOG(INFO) << "Output tensor size: " << shape[0] << " x " << shape[1] << " x " << shape[2];for (int i = 0; i < shape[1]; i++) {std::vector<float> temp;for (int j = 0; j < shape[2]; j++) {@@ -93,11 +91,8 @@ std::vector<float> TextEmbedder::Embed(const std::string& text) {}output.push_back(temp);}- //LOG(INFO) << "Mean pooling";auto pooled_output = mean_pooling(output);--return pooled_output;}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 98ebe805..f7743614 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -4700,22 +4700,18 @@ TEST_F(CollectionTest, HybridSearch) {HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::get_absolute_vocab_path());auto op = collectionManager.create_collection(schema);- LOG(INFO) << collectionManager.get_collection("objects")->get_summary_json();ASSERT_TRUE(op.ok());Collection* coll = op.get();nlohmann::json object;object["name"] = "apple";auto add_op = coll->add(object.dump());- LOG(INFO) << "hybrid search";ASSERT_TRUE(add_op.ok());ASSERT_EQ("apple", add_op.get()["name"]);ASSERT_EQ(384, add_op.get()["embedding"].size());spp::sparse_hash_set<std::string> dummy_include_exclude;- LOG(INFO) << "hybrid search 2";auto search_res_op = coll->search("apple", {"name","embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");- LOG(INFO) << "hybrid search 3";ASSERT_TRUE(search_res_op.ok());auto search_res = search_res_op.get();ASSERT_EQ(1, search_res["found"].get<size_t>());@@ -4755,6 +4751,80 @@ TEST_F(CollectionTest, EmbedFielsTest) {ASSERT_EQ("apple", object["name"]);ASSERT_EQ(384, object["embedding"].get<std::vector<float>>().size());+ // delete models folder+ system("rm -rf ./models");+}++TEST_F(CollectionTest, HybridSearchRankFusionTest) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "create_from": ["name"]}+ ]+ })"_json;++ TextEmbedderManager::model_dir = "./models/";+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_MODEL_URL, TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::DEFAULT_MODEL_NAME));+ HttpClient::get_instance().download_file(TextEmbedderManager::DEFAULT_VOCAB_URL, TextEmbedderManager::model_dir + TextEmbedderManager::get_absolute_vocab_path());++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();+ nlohmann::json object;+ object["name"] = "butter";+ auto add_op = coll->add(object.dump());+ ASSERT_TRUE(add_op.ok());++ object["name"] = "butterball";+ add_op = coll->add(object.dump());+ ASSERT_TRUE(add_op.ok());++ object["name"] = "butterfly";+ add_op = coll->add(object.dump());+ ASSERT_TRUE(add_op.ok());++ spp::sparse_hash_set<std::string> dummy_include_exclude;+ auto search_res_op = coll->search("butter", {"embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");+ ASSERT_TRUE(search_res_op.ok());+ auto search_res = search_res_op.get();+ ASSERT_EQ(3, search_res["found"].get<size_t>());+ ASSERT_EQ(3, search_res["hits"].size());+ // Vector search order:+ // 1. butter+ // 2. butterball+ // 3. butterfly+ ASSERT_EQ("butter", search_res["hits"][0]["document"]["name"].get<std::string>());+ ASSERT_EQ("butterball", search_res["hits"][1]["document"]["name"].get<std::string>());+ ASSERT_EQ("butterfly", search_res["hits"][2]["document"]["name"].get<std::string>());+++ search_res_op = coll->search("butter", {"name"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");+ ASSERT_TRUE(search_res_op.ok());+ search_res = search_res_op.get();+ ASSERT_EQ(3, search_res["found"].get<size_t>());+ ASSERT_EQ(3, search_res["hits"].size());+ // Keyword search order:+ // 1. butter+ // 2. butterfly+ // 3. butterball+ ASSERT_EQ("butter", search_res["hits"][0]["document"]["name"].get<std::string>());+ ASSERT_EQ("butterfly", search_res["hits"][1]["document"]["name"].get<std::string>());+ ASSERT_EQ("butterball", search_res["hits"][2]["document"]["name"].get<std::string>());++ search_res_op = coll->search("butter", {"name","embedding"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, Index::DROP_TOKENS_THRESHOLD, dummy_include_exclude, dummy_include_exclude, 10, "", 30, 4, "");+ ASSERT_TRUE(search_res_op.ok());+ search_res = search_res_op.get();+ ASSERT_EQ(3, search_res["found"].get<size_t>());+ ASSERT_EQ(3, search_res["hits"].size());+ // Hybrid search with rank fusion order:+ // 1. butter (1/1 * 0.7) + (1/1 * 0.3) = 1+ // 2. butterfly (1/2 * 0.7) + (1/3 * 0.3) = 0.567+ // 3. butterball (1/3 * 0.7) + (1/2 * 0.3) = 0.533+ ASSERT_EQ("butter", search_res["hits"][0]["document"]["name"].get<std::string>());+ ASSERT_EQ("butterfly", search_res["hits"][1]["document"]["name"].get<std::string>());+ ASSERT_EQ("butterball", search_res["hits"][2]["document"]["name"].get<std::string>());+// delete models foldersystem("rm -rf ./models");}\ No newline at end of file