Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix scoring when tokens are spread out.

Kishore Nallan5413536b409c
diff --git a/include/match_score.h b/include/match_score.h
index 19ef660f..873aab80 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -50,21 +50,10 @@ struct Match {
}
- // Explicit construction of match score
- static inline uint64_t get_match_score(const uint32_t words_present, const uint32_t total_cost, const uint8_t distance) {
- uint64_t match_score = (
- (int64_t(words_present) << 24) |
- (int64_t(255 - total_cost) << 16) |
- (int64_t(100 - distance) << 8) |
- (int64_t(0) << 0)
- );
-
- return match_score;
- }
-
// Construct a single match score from individual components (for multi-field sort)
- inline uint64_t get_match_score(const uint32_t total_cost) const {
+ inline uint64_t get_match_score(const uint32_t total_cost, const uint32_t unique_words) const {
uint64_t match_score = (
+ (int64_t(unique_words) << 32) |
(int64_t(words_present) << 24) |
(int64_t(255 - total_cost) << 16) |
(int64_t(100 - distance) << 8) |
diff --git a/src/collection.cpp b/src/collection.cpp
index 59a12633..60e95521 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1839,7 +1839,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
}
const Match & this_match = Match(field_order_kv->key, token_positions, true, true);
- uint64_t this_match_score = this_match.get_match_score(1);
+ uint64_t this_match_score = this_match.get_match_score(1, token_positions.size());
match_indices.emplace_back(this_match, this_match_score, array_index);
/*LOG(INFO) << "doc_id: " << document["id"] << ", words_present: " << size_t(this_match.words_present)
diff --git a/src/index.cpp b/src/index.cpp
index d11fcba4..e023bdf7 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2409,22 +2409,25 @@ void Index::aggregate_and_score_fields(const std::vector<query_tokens_t>& field_
// we will reassemble match score to use unique tokens for cross-field matches
+ uint32_t unique_tokens_found = ((match_score >> 32) & 0xFF);
uint64_t tokens_found = ((match_score >> 24) & 0xFF);
uint64_t typo_score = ((match_score >> 16) & 0xFF);
uint64_t proximity_score = ((match_score >> 8) & 0xFF);
int64_t exact_match_score = match_score & 0xFF;
- uint32_t unique_tokens_found = __builtin_popcount(existing_field_kvs[field_id]->token_bits);
// exclude dropped-token cases
if(unique_tokens_found != field_query_tokens[0].q_include_tokens.size()) {
exact_match_score = 0;
}
+ // NOTE: unique_tokens_found can be > than tokens_found as `tokens_found` is counted only within matching
+ // window of X tokens.
+
int64_t multi_field_match_score = (int64_t(unique_tokens_found) << 32) |
- (int64_t(typo_score) << 24) |
- (int64_t(proximity_score) << 16) |
- (int64_t(exact_match_score) << 8) |
- (int64_t(0) << 0);
+ (int64_t(tokens_found) << 24) |
+ (int64_t(typo_score) << 16) |
+ (int64_t(proximity_score) << 8) |
+ (int64_t(exact_match_score) << 0);
if(multi_field_match_score > max_field_match_score) {
max_field_match_score = multi_field_match_score;
@@ -2479,12 +2482,14 @@ void Index::aggregate_and_score_fields(const std::vector<query_tokens_t>& field_
max_field_match_score = multi_field_match_score;
max_field_match_index = i;
}
+
+ //LOG(INFO) << "!seq_id: " << seq_id << ", words_present: " << words_present;
}
}
uint32_t num_tokens_found = __builtin_popcount(token_bits);
- uint64_t aggregated_score = (int64_t(max_field_match_score) << 16) |
- (int64_t(num_tokens_found) << 8) |
+ uint64_t aggregated_score = (int64_t(num_tokens_found) << 48) |
+ (int64_t(max_field_match_score) << 8) |
(int64_t(the_fields[max_field_match_index].weight) << 0);
//LOG(INFO) << "seq id: " << seq_id << ", aggregated_score: " << aggregated_score;
@@ -3549,20 +3554,25 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
posting_list_t::is_single_token_verbatim_match(posting_lists[0], field_is_array)
);
Match single_token_match = Match(1, 0, is_verbatim_match);
- match_score = single_token_match.get_match_score(total_cost);
+ match_score = single_token_match.get_match_score(total_cost, 1);
} else {
uint64_t total_tokens_found = 0, total_num_typos = 0, total_distance = 0, total_verbatim = 0;
std::unordered_map<size_t, std::vector<token_positions_t>> array_token_positions;
posting_list_t::get_offsets(posting_lists, array_token_positions);
+ // NOTE: tokens found returned by matcher is only within the best matched window, so we have to still consider
+ // unique tokens found if they are spread across the text.
+ uint32_t unique_tokens_found = __builtin_popcount(token_bits);
+
for (const auto& kv: array_token_positions) {
const std::vector<token_positions_t>& token_positions = kv.second;
if (token_positions.empty()) {
continue;
}
+
const Match &match = Match(seq_id, token_positions, false, prioritize_exact_match);
- uint64_t this_match_score = match.get_match_score(total_cost);
+ uint64_t this_match_score = match.get_match_score(total_cost, unique_tokens_found);
total_tokens_found += ((this_match_score >> 24) & 0xFF);
total_num_typos += 255 - ((this_match_score >> 16) & 0xFF);
@@ -3579,6 +3589,7 @@ void Index::score_results(const std::vector<sort_by> & sort_fields, const uint16
}
match_score = (
+ (uint64_t(unique_tokens_found) << 32) |
(uint64_t(total_tokens_found) << 24) |
(uint64_t(255 - total_num_typos) << 16) |
(uint64_t(100 - total_distance) << 8) |
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 3256c960..2bf96c00 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -309,8 +309,8 @@ TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {
ASSERT_STREQ("country", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());
ASSERT_EQ(2, results["grouped_hits"][2]["hits"].size());
- ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("8", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("8", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 9faa484d..501414ad 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -1980,6 +1980,98 @@ TEST_F(CollectionSpecificTest, SearchShouldJoinToken) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionSpecificTest, TokenCountOfWordsFarApart) {
+ // word proximity is calculated using a moving window of X tokens. If only 1 token is present in the best matched
+ // window, proximity ends up being perfect. So we've to ensure that scoring uses total tokens found and not
+ // window tokens found for scoring.
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("author", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ std::vector<std::vector<std::string>> records = {
+ {"Central Arizona Project. - Hearing, Eighty-eighth Congress, Second Session, on H.R. 6796, H.R. 6797, "
+ "H.R. 6798. November 9, 1964, Phoenix, Ariz", "JK"},
+ {"Project Phoenix", "JK"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["author"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("Phoenix project)", {"title", "author"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {false},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionSpecificTest, SingleFieldTokenCountOfWordsFarApart) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("author", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+
+ std::vector<std::vector<std::string>> records = {
+ {"Central Arizona Project. - Hearing, Eighty-eighth Congress, Second Session, on H.R. 6796, H.R. 6797, "
+ "H.R. 6798. November 9, 1964, Phoenix, Ariz", "JK"},
+ {"Project Aim Arizona", "JK"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["author"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("Phoenix project)", {"title"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {false},
+ 10, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 10, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ // without drop tokens
+
+ results = coll1->search("Phoenix project)", {"title"},
+ "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {false},
+ 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title", 1, {}, {}, {}, 0,
+ "<mark>", "</mark>", {}, 1000, true).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
+
TEST_F(CollectionSpecificTest, SingleHyphenInQueryNotToBeTreatedAsExclusion) {
std::vector<field> fields = {field("title", field_types::STRING, false),
field("points", field_types::INT32, false),};