Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix scoring when tokens are spread out.
Kishore Nallan5413536b409c
diff --git a/include/match_score.h b/include/match_score.hindex 19ef660f..873aab80 100644--- a/include/match_score.h+++ b/include/match_score.h@@ -50,21 +50,10 @@ struct Match {}- // Explicit construction of match score- static inline uint64_t get_match_score(const uint32_t words_present, const uint32_t total_cost, const uint8_t distance) {- uint64_t match_score = (- (int64_t(words_present) << 24) |- (int64_t(255 - total_cost) << 16) |- (int64_t(100 - distance) << 8) |- (int64_t(0) << 0)- );-- return match_score;- }-// Construct a single match score from individual components (for multi-field sort)- inline uint64_t get_match_score(const uint32_t total_cost) const {+ inline uint64_t get_match_score(const uint32_t total_cost, const uint32_t unique_words) const {uint64_t match_score = (+ (int64_t(unique_words) << 32) |(int64_t(words_present) << 24) |(int64_t(255 - total_cost) << 16) |(int64_t(100 - distance) << 8) |diff --git a/src/collection.cpp b/src/collection.cppindex 59a12633..60e95521 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1839,7 +1839,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea}const Match & this_match = Match(field_order_kv->key, token_positions, true, true);- uint64_t this_match_score = this_match.get_match_score(1);+ uint64_t this_match_score = this_match.get_match_score(1, token_positions.size());match_indices.emplace_back(this_match, this_match_score, array_index);/*LOG(INFO) << "doc_id: " << document["id"] << ", words_present: " << size_t(this_match.words_present)diff --git a/src/index.cpp b/src/index.cppindex d11fcba4..e023bdf7 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2409,22 +2409,25 @@ void Index::aggregate_and_score_fields(const std::vector<query_tokens_t>& field_// we will reassemble match score to use unique tokens for cross-field matches+ uint32_t unique_tokens_found = ((match_score >> 32) & 0xFF);uint64_t tokens_found = ((match_score >> 24) & 0xFF);uint64_t typo_score = ((match_score >> 16) & 0xFF);uint64_t proximity_score = ((match_score >> 8) & 0xFF);int64_t exact_match_score = match_score & 0xFF;- uint32_t unique_tokens_found = __builtin_popcount(existing_field_kvs[field_id]->token_bits);// exclude dropped-token casesif(unique_tokens_found != field_query_tokens[0].q_include_tokens.size()) {exact_match_score = 0;}+ // NOTE: unique_tokens_found can be > than tokens_found as `tokens_found` is counted only within matching+ // window of X tokens.+int64_t multi_field_match_score = (int64_t(unique_tokens_found) << 32) |- (int64_t(typo_score) << 24) |- (int64_t(proximity_score) << 16) |- (int64_t(exact_match_score) << 8) |- (int64_t(0) << 0);+ (int64_t(tokens_found) << 24) |+ (int64_t(typo_score) << 16) |+ (int64_t(proximity_score) << 8) |+ (int64_t(exact_match_score) << 0);if(multi_field_match_score > max_field_match_score) {max_field_match_score = multi_field_match_score;@@ -2479,12 +2482,14 @@ void Index::aggregate_and_score_fields(const std::vector<query_tokens_t>& field_max_field_match_score = multi_field_match_score;max_field_match_index = i;}++ //LOG(INFO) << "!seq_id: " << seq_id << ", words_present: " << words_present;}}uint32_t num_tokens_found = __builtin_popcount(token_bits);- uint64_t aggregated_score = (int64_t(max_field_match_score) << 16) |- (int64_t(num_tokens_found) << 8) |+ uint64_t aggregated_score = (int64_t(num_tokens_found) << 48) |+ (int64_t(max_field_match_score) << 8) |(int64_t(the_fields[max_field_match_index].weight) << 0);//LOG(INFO) << "seq id: " << seq_id << ", aggregated_score: " << aggregated_score;@@ -3549,20 +3554,25 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16posting_list_t::is_single_token_verbatim_match(posting_lists[0], field_is_array));Match single_token_match = Match(1, 0, is_verbatim_match);- match_score = single_token_match.get_match_score(total_cost);+ match_score = single_token_match.get_match_score(total_cost, 1);} else {uint64_t total_tokens_found = 0, total_num_typos = 0, total_distance = 0, total_verbatim = 0;std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;posting_list_t::get_offsets(posting_lists, array_token_positions);+ // NOTE: tokens found returned by matcher is only within the best matched window, so we have to still consider+ // unique tokens found if they are spread across the text.+ uint32_t unique_tokens_found = __builtin_popcount(token_bits);+for (const auto& kv: array_token_positions) {const std::vector<token_positions_t>& token_positions = kv.second;if (token_positions.empty()) {continue;}+const Match &match = Match(seq_id, token_positions, false, prioritize_exact_match);- uint64_t this_match_score = match.get_match_score(total_cost);+ uint64_t this_match_score = match.get_match_score(total_cost, unique_tokens_found);total_tokens_found += ((this_match_score >> 24) & 0xFF);total_num_typos += 255 - ((this_match_score >> 16) & 0xFF);@@ -3579,6 +3589,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16}match_score = (+ (uint64_t(unique_tokens_found) << 32) |(uint64_t(total_tokens_found) << 24) |(uint64_t(255 - total_num_typos) << 16) |(uint64_t(100 - total_distance) << 8) |diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex 3256c960..2bf96c00 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -309,8 +309,8 @@ TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {ASSERT_STREQ("country", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());ASSERT_EQ(2, results["grouped_hits"][2]["hits"].size());- ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());- ASSERT_STREQ("8", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("8", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());collectionManager.drop_collection("coll1");}diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 9faa484d..501414ad 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1980,6 +1980,98 @@ TEST_F(CollectionSpecificTest, SearchShouldJoinToken) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionSpecificTest, TokenCountOfWordsFarApart) {+ // word proximity is calculated using a moving window of X tokens. If only 1 token is present in the best matched+ // window, proximity ends up being perfect. So we've to ensure that scoring uses total tokens found and not+ // window tokens found for scoring.+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("author", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ std::vector<std::vector<std::string>> records = {+ {"Central Arizona Project. - Hearing, Eighty-eighth Congress, Second Session, on H.R. 6796, H.R. 6797, "+ "H.R. 6798. November 9, 1964, Phoenix, Ariz", "JK"},+ {"Project Phoenix", "JK"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["author"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("Phoenix project)", {"title", "author"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {false},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 1, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSpecificTest, SingleFieldTokenCountOfWordsFarApart) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("author", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();++ std::vector<std::vector<std::string>> records = {+ {"Central Arizona Project. - Hearing, Eighty-eighth Congress, Second Session, on H.R. 6796, H.R. 6797, "+ "H.R. 6798. November 9, 1964, Phoenix, Ariz", "JK"},+ {"Project Aim Arizona", "JK"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["author"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("Phoenix project)", {"title"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {false},+ 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 10, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());++ // without drop tokens++ results = coll1->search("Phoenix project)", {"title"},+ "", {}, {}, {2}, 10,+ 1, FREQUENCY, {false},+ 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 1, {}, {}, {}, 0,+ "<mark>", "</mark>", {}, 1000, true).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionSpecificTest, SingleHyphenInQueryNotToBeTreatedAsExclusion) {std::vector<field> fields = {field("title", field_types::STRING, false),field("points", field_types::INT32, false),};