Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Use weighted scoring of fields for matching.
kishorenc667eee1dadeb
diff --git a/include/match_score.h b/include/match_score.hindex 3b1ac48a..6243c3cb 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -49,7 +49,7 @@ struct Match {static inline uint64_t get_match_score(const uint32_t words_present, const uint32_t total_cost, const uint8_t distance,const uint8_t field_id) {uint64_t match_score = (- (int64_t(field_id) << 24) |+ //(int64_t(field_id) << 24) |(int64_t(words_present) << 16) |(int64_t(255 - total_cost) << 8) |(int64_t(distance))@@ -61,7 +61,7 @@ struct Match {// Construct a single match score from individual components (for multi-field sort)inline uint64_t get_match_score(const uint32_t total_cost, const uint8_t field_id) const {uint64_t match_score = (- (int64_t(field_id) << 24) |+ //(int64_t(field_id) << 24) |(int64_t(words_present) << 16) |(int64_t(255 - total_cost) << 8) |(int64_t(distance))diff --git a/src/index.cpp b/src/index.cppindex 8565ab68..47cd9f61 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -1437,7 +1437,7 @@ void Index::search(Option<uint32_t> & outcome,for(size_t i = 0; i < num_search_fields; i++) {// proceed to query search only when no filters are provided or when filtering produces resultsif(filters.empty() || filter_ids_length > 0) {- const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - (2*i)); // Order of `fields` are used to sort results+ const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);const std::string & field = search_fields[i];std::vector<std::string> query_tokens = q_include_tokens;@@ -1461,8 +1461,7 @@ void Index::search(Option<uint32_t> & outcome,num_tokens_dropped = 0;query_tokens = search_tokens = syn_tokens;- // for synonym we use a smaller field id than for original tokens- search_field(field_id-1, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,+ search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);@@ -1491,17 +1490,23 @@ void Index::search(Option<uint32_t> & outcome,existing_field_kvs.emplace(kv->field_id, kv);}+ int64_t aggregated_score = 0;+for(size_t i = 0; i < num_search_fields && num_search_fields > 1; i++) {- const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - (2*i)); // Order of `fields` used to sort results+ const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);++ // produces weights that are powers of 2+ size_t num_left_shifts = num_search_fields - i - 1;if(field_id == kvs[0]->field_id) {+ aggregated_score += (kvs[0]->scores[kvs[0]->match_score_index] << num_left_shifts);continue;}if(existing_field_kvs.count(field_id) != 0) {- // for existing field, we will simply sum field-wise match scores- kvs[0]->scores[kvs[0]->match_score_index] +=- existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index];+ // for existing field, we will simply sum field-wise weighted scores+ aggregated_score += (existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index]+ << num_left_shifts);continue;}@@ -1544,11 +1549,12 @@ void Index::search(Option<uint32_t> & outcome,if(words_present != 0) {uint64_t match_score = Match::get_match_score(words_present, 0, 100, field_id);- kvs[0]->scores[kvs[0]->match_score_index] += match_score;+ aggregated_score += (match_score << num_left_shifts);}}//LOG(INFO) << "kvs[0].key: " << kvs[0]->key;+ kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;topster->add(kvs[0]);}}diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 3ef5da56..74132378 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -301,14 +301,14 @@ TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {ASSERT_STREQ("1", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());ASSERT_STREQ("4", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());- ASSERT_STREQ("country", results["grouped_hits"][1]["group_key"][0].get<std::string>().c_str());- ASSERT_EQ(2, results["grouped_hits"][1]["hits"].size());- ASSERT_STREQ("8", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());- ASSERT_STREQ("3", results["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());-- ASSERT_STREQ("rock", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());- ASSERT_EQ(1, results["grouped_hits"][2]["hits"].size());- ASSERT_STREQ("5", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("rock", results["grouped_hits"][1]["group_key"][0].get<std::string>().c_str());+ ASSERT_EQ(1, results["grouped_hits"][1]["hits"].size());+ ASSERT_STREQ("5", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());++ ASSERT_STREQ("country", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());+ ASSERT_EQ(2, results["grouped_hits"][2]["hits"].size());+ ASSERT_STREQ("8", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());collectionManager.drop_collection("coll1");}diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex a126b2a5..73637c3f 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -359,8 +359,8 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {ASSERT_EQ(2, res["hits"].size());ASSERT_EQ(2, res["found"].get<uint32_t>());- ASSERT_STREQ("<mark>Samuel</mark> L. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());- ASSERT_STREQ("<mark>Samuel</mark> L. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());// for now we don't support synonyms on ANY prefix@@ -369,7 +369,7 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {ASSERT_EQ(0, res["found"].get<uint32_t>());}-TEST_F(CollectionSynonymsTest, ExactMatchRankedHigherThanSynonymMatch) {+TEST_F(CollectionSynonymsTest, ExactMatchRankedSameAsSynonymMatch) {Collection *coll1;std::vector<field> fields = {field("title", field_types::STRING, false),@@ -415,10 +415,10 @@ TEST_F(CollectionSynonymsTest, ExactMatchRankedHigherThanSynonymMatch) {ASSERT_EQ(4, res["hits"].size());ASSERT_EQ(4, res["found"].get<uint32_t>());- ASSERT_STREQ("1", res["hits"][0]["document"]["id"].get<std::string>().c_str());- ASSERT_STREQ("0", res["hits"][1]["document"]["id"].get<std::string>().c_str());- ASSERT_STREQ("3", res["hits"][2]["document"]["id"].get<std::string>().c_str());- ASSERT_STREQ("2", res["hits"][3]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("3", res["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("2", res["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", res["hits"][2]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", res["hits"][3]["document"]["id"].get<std::string>().c_str());collectionManager.drop_collection("coll1");}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex ba33d71d..94166cae 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2659,6 +2659,46 @@ TEST_F(CollectionTest, MultiFieldRelevance) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionTest, MultiFieldRelevance2) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1");+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"A Daikon Freestyle", "Ghosts on a Trampoline"},+ {"Leaving on a Jetplane", "Coby Grant"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("on a jetplane",+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();++ ASSERT_EQ(2, results["found"].get<size_t>());+ ASSERT_EQ(2, results["hits"].size());++ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionTest, MultiFieldMatchRanking) {Collection *coll1;