Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Use weighted scoring of fields for matching.

kishorenc667eee1dadeb
diff --git a/include/match_score.h b/include/match_score.h
index 3b1ac48a..6243c3cb 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -49,7 +49,7 @@ struct Match {
static inline uint64_t get_match_score(const uint32_t words_present, const uint32_t total_cost, const uint8_t distance,
const uint8_t field_id) {
uint64_t match_score = (
- (int64_t(field_id) << 24) |
+ //(int64_t(field_id) << 24) |
(int64_t(words_present) << 16) |
(int64_t(255 - total_cost) << 8) |
(int64_t(distance))
@@ -61,7 +61,7 @@ struct Match {
// Construct a single match score from individual components (for multi-field sort)
inline uint64_t get_match_score(const uint32_t total_cost, const uint8_t field_id) const {
uint64_t match_score = (
- (int64_t(field_id) << 24) |
+ //(int64_t(field_id) << 24) |
(int64_t(words_present) << 16) |
(int64_t(255 - total_cost) << 8) |
(int64_t(distance))
diff --git a/src/index.cpp b/src/index.cpp
index 8565ab68..47cd9f61 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -1437,7 +1437,7 @@ void Index::search(Option<uint32_t> & outcome,
for(size_t i = 0; i < num_search_fields; i++) {
// proceed to query search only when no filters are provided or when filtering produces results
if(filters.empty() || filter_ids_length > 0) {
- const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - (2*i)); // Order of `fields` are used to sort results
+ const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
const std::string & field = search_fields[i];
std::vector<std::string> query_tokens = q_include_tokens;
@@ -1461,8 +1461,7 @@ void Index::search(Option<uint32_t> & outcome,
num_tokens_dropped = 0;
query_tokens = search_tokens = syn_tokens;
- // for synonym we use a smaller field id than for original tokens
- search_field(field_id-1, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
+ search_field(field_id, query_tokens, search_tokens, exclude_token_ids, exclude_token_ids_size, num_tokens_dropped,
field, filter_ids, filter_ids_length, curated_ids_sorted, facets, sort_fields_std,
num_typos, searched_queries, actual_topster, groups_processed, &all_result_ids, all_result_ids_len,
token_order, prefix, drop_tokens_threshold, typo_tokens_threshold);
@@ -1491,17 +1490,23 @@ void Index::search(Option<uint32_t> & outcome,
existing_field_kvs.emplace(kv->field_id, kv);
}
+ int64_t aggregated_score = 0;
+
for(size_t i = 0; i < num_search_fields && num_search_fields > 1; i++) {
- const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - (2*i)); // Order of `fields` used to sort results
+ const uint8_t field_id = (uint8_t)(FIELD_LIMIT_NUM - i);
+
+ // produces weights that are powers of 2
+ size_t num_left_shifts = num_search_fields - i - 1;
if(field_id == kvs[0]->field_id) {
+ aggregated_score += (kvs[0]->scores[kvs[0]->match_score_index] << num_left_shifts);
continue;
}
if(existing_field_kvs.count(field_id) != 0) {
- // for existing field, we will simply sum field-wise match scores
- kvs[0]->scores[kvs[0]->match_score_index] +=
- existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index];
+ // for existing field, we will simply sum field-wise weighted scores
+ aggregated_score += (existing_field_kvs[field_id]->scores[existing_field_kvs[field_id]->match_score_index]
+ << num_left_shifts);
continue;
}
@@ -1544,11 +1549,12 @@ void Index::search(Option<uint32_t> & outcome,
if(words_present != 0) {
uint64_t match_score = Match::get_match_score(words_present, 0, 100, field_id);
- kvs[0]->scores[kvs[0]->match_score_index] += match_score;
+ aggregated_score += (match_score << num_left_shifts);
}
}
//LOG(INFO) << "kvs[0].key: " << kvs[0]->key;
+ kvs[0]->scores[kvs[0]->match_score_index] = aggregated_score;
topster->add(kvs[0]);
}
}
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index 3ef5da56..74132378 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -301,14 +301,14 @@ TEST_F(CollectionGroupingTest, GroupingWithMultiFieldRelevance) {
ASSERT_STREQ("1", results["grouped_hits"][0]["hits"][0]["document"]["id"].get<std::string>().c_str());
ASSERT_STREQ("4", results["grouped_hits"][0]["hits"][1]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("country", results["grouped_hits"][1]["group_key"][0].get<std::string>().c_str());
- ASSERT_EQ(2, results["grouped_hits"][1]["hits"].size());
- ASSERT_STREQ("8", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("3", results["grouped_hits"][1]["hits"][1]["document"]["id"].get<std::string>().c_str());
-
- ASSERT_STREQ("rock", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());
- ASSERT_EQ(1, results["grouped_hits"][2]["hits"].size());
- ASSERT_STREQ("5", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("rock", results["grouped_hits"][1]["group_key"][0].get<std::string>().c_str());
+ ASSERT_EQ(1, results["grouped_hits"][1]["hits"].size());
+ ASSERT_STREQ("5", results["grouped_hits"][1]["hits"][0]["document"]["id"].get<std::string>().c_str());
+
+ ASSERT_STREQ("country", results["grouped_hits"][2]["group_key"][0].get<std::string>().c_str());
+ ASSERT_EQ(2, results["grouped_hits"][2]["hits"].size());
+ ASSERT_STREQ("8", results["grouped_hits"][2]["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("3", results["grouped_hits"][2]["hits"][1]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
index a126b2a5..73637c3f 100644
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@ -359,8 +359,8 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {
ASSERT_EQ(2, res["hits"].size());
ASSERT_EQ(2, res["found"].get<uint32_t>());
- ASSERT_STREQ("<mark>Samuel</mark> L. <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>Samuel</mark> L. <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>Samuel</mark> <mark>L.</mark> <mark>Jackson</mark>", res["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());
// for now we don't support synonyms on ANY prefix
@@ -369,7 +369,7 @@ TEST_F(CollectionSynonymsTest, MultiWaySynonym) {
ASSERT_EQ(0, res["found"].get<uint32_t>());
}
-TEST_F(CollectionSynonymsTest, ExactMatchRankedHigherThanSynonymMatch) {
+TEST_F(CollectionSynonymsTest, ExactMatchRankedSameAsSynonymMatch) {
Collection *coll1;
std::vector<field> fields = {field("title", field_types::STRING, false),
@@ -415,10 +415,10 @@ TEST_F(CollectionSynonymsTest, ExactMatchRankedHigherThanSynonymMatch) {
ASSERT_EQ(4, res["hits"].size());
ASSERT_EQ(4, res["found"].get<uint32_t>());
- ASSERT_STREQ("1", res["hits"][0]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("0", res["hits"][1]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("3", res["hits"][2]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("2", res["hits"][3]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("3", res["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("2", res["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", res["hits"][2]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("0", res["hits"][3]["document"]["id"].get<std::string>().c_str());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index ba33d71d..94166cae 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -2659,6 +2659,46 @@ TEST_F(CollectionTest, MultiFieldRelevance) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionTest, MultiFieldRelevance2) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1");
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"A Daikon Freestyle", "Ghosts on a Trampoline"},
+ {"Leaving on a Jetplane", "Coby Grant"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("on a jetplane",
+ {"title", "artist"}, "", {}, {}, 0, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(2, results["found"].get<size_t>());
+ ASSERT_EQ(2, results["hits"].size());
+
+ ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+
+ collectionManager.drop_collection("coll1");
+}
+
TEST_F(CollectionTest, MultiFieldMatchRanking) {
Collection *coll1;