Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Provide breakdown of text match score in response.

Kishore Nallan308b34eb9c87
diff --git a/include/collection.h b/include/collection.h
index 61b0acc9..30eed5f6 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -56,7 +56,7 @@ private:
uint64_t match_score;
std::vector<std::vector<std::string>> matched_tokens;
- highlight_t() {
+ highlight_t(): field_index(0), match_score(0) {
}
@@ -188,6 +188,8 @@ private:
std::vector<std::pair<uint32_t, uint32_t>>& included_ids,
std::vector<uint32_t>& excluded_ids) const;
+ void populate_text_match_info(nlohmann::json& info, uint64_t match_score) const;
+
public:
enum {MAX_ARRAY_MATCHES = 5};
diff --git a/include/match_score.h b/include/match_score.h
index f3db9b7f..42e6bf45 100644
--- a/include/match_score.h
+++ b/include/match_score.h
@@ -35,15 +35,15 @@ struct TokenOffset {
};
struct Match {
- uint8_t words_present;
- uint8_t distance;
- uint8_t max_offset;
- uint8_t exact_match;
- uint8_t phrase_match;
+ uint8_t words_present = 0;
+ uint8_t distance = 0;
+ uint8_t max_offset = 0;
+ uint8_t exact_match = 0;
+ uint8_t phrase_match = 0;
std::vector<TokenOffset> offsets;
- Match() : words_present(0), distance(0), exact_match(0), max_offset(0) {
+ Match() : words_present(0), distance(0), max_offset(0), exact_match(0) {
}
diff --git a/src/collection.cpp b/src/collection.cpp
index b964013a..7ca585e1 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1334,6 +1334,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
wrapper_doc["curated"] = true;
} else {
wrapper_doc["text_match"] = field_order_kv->scores[field_order_kv->match_score_index];
+
+ wrapper_doc["text_match_info"] = nlohmann::json::object();
+ populate_text_match_info(wrapper_doc["text_match_info"],
+ field_order_kv->scores[field_order_kv->match_score_index]);
}
nlohmann::json geo_distances;
@@ -1610,6 +1614,20 @@ void Collection::process_search_field_weights(const std::vector<std::string>& ra
}
}
+void Collection::populate_text_match_info(nlohmann::json& info, uint64_t match_score) const {
+ // [ sign | tokens_matched | best_field_score | best_field_weight | num_field_matches ]
+ // [ 1 | 4 | 48 | 8 | 3 ] (64 bits)
+
+ // 0 0001 000000000010000000111111111011001000000000100000 00000110 011
+
+ info["score"] = std::to_string(match_score);
+
+ info["tokens_matched"] = (match_score >> 59);
+ info["best_field_score"] = std::to_string((match_score << 5) >> (8 + 3 + 5));
+ info["best_field_weight"] = ((match_score << 53) >> (3 + 53));
+ info["fields_matched"] = ((match_score << 61) >> (61));
+}
+
void Collection::process_highlight_fields(const std::vector<std::string>& search_fields,
const spp::sparse_hash_set<std::string>& include_fields,
const spp::sparse_hash_set<std::string>& exclude_fields,
diff --git a/src/index.cpp b/src/index.cpp
index d0ff568f..0947b444 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -3181,22 +3181,23 @@ void Index::search_across_fields(const std::vector<token_t>& query_tokens,
// NOTE: `query_len` is total tokens matched across fields.
// Within a field, only a subset can match
- // [ query_len | max_field_score | max_field_weight | num_matching_fields]
- // [ 4 | 48 | 8 | 3 ] (63 bits)
+ // [ sign | tokens_matched | max_field_score | max_field_weight | num_matching_fields ]
+ // [ 1 | 4 | 48 | 8 | 3 ] (64 bits)
auto max_field_weight = std::min<size_t>(FIELD_MAX_WEIGHT, the_fields[max_field_match_index].weight);
num_matching_fields = std::min<size_t>(7, num_matching_fields);
uint64_t aggregated_score = (int64_t(query_len) << 59) |
- (int64_t(max_field_match_score) << 8) |
- (int64_t(max_field_weight) << 4) |
+ (int64_t(max_field_match_score) << 11) |
+ (int64_t(max_field_weight) << 3) |
(int64_t(num_matching_fields) << 0);
- /*LOG(INFO) << "seq_id: " << seq_id << ", query_tokens.size(): " << query_tokens.size()
+ /*LOG(INFO) << "seq_id: " << seq_id << ", query_len: " << query_len
<< ", syn_orig_num_tokens: " << syn_orig_num_tokens
<< ", max_field_match_score: " << max_field_match_score
<< ", max_field_match_index: " << max_field_match_index
<< ", field_weight: " << max_field_weight
+ << ", num_matching_fields: " << num_matching_fields
<< ", aggregated_score: " << aggregated_score;*/
KV kv(0, searched_queries.size(), 0, seq_id, distinct_id, match_score_index, scores);
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index e08f1b06..476c1afb 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -1729,10 +1729,10 @@ TEST_F(CollectionSortingTest, RepeatingTokenRanking) {
ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][3]["document"]["id"].get<std::string>());
- ASSERT_EQ(1153487750477971505, results["hits"][0]["text_match"].get<size_t>());
- ASSERT_EQ(1153487750477905969, results["hits"][1]["text_match"].get<size_t>());
- ASSERT_EQ(1153487750477905969, results["hits"][2]["text_match"].get<size_t>());
- ASSERT_EQ(1153487750477905969, results["hits"][3]["text_match"].get<size_t>());
+ ASSERT_EQ(1157451471575842841, results["hits"][0]["text_match"].get<size_t>());
+ ASSERT_EQ(1157451471575318553, results["hits"][1]["text_match"].get<size_t>());
+ ASSERT_EQ(1157451471575318553, results["hits"][2]["text_match"].get<size_t>());
+ ASSERT_EQ(1157451471575318553, results["hits"][3]["text_match"].get<size_t>());
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index d41e811c..2b7d067e 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -787,6 +787,24 @@ TEST_F(CollectionSpecificMoreTest, RelevanceConsiderAllFields) {
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
ASSERT_EQ("2", results["hits"][2]["document"]["id"].get<std::string>());
+ // verify match score component values
+ ASSERT_EQ("578730123365711899", results["hits"][0]["text_match_info"]["score"].get<std::string>());
+ ASSERT_EQ(3, results["hits"][0]["text_match_info"]["fields_matched"].get<size_t>());
+ ASSERT_EQ(2, results["hits"][1]["text_match_info"]["fields_matched"].get<size_t>());
+ ASSERT_EQ(1, results["hits"][2]["text_match_info"]["fields_matched"].get<size_t>());
+
+ ASSERT_EQ(1, results["hits"][0]["text_match_info"]["tokens_matched"].get<size_t>());
+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["tokens_matched"].get<size_t>());
+ ASSERT_EQ(1, results["hits"][2]["text_match_info"]["tokens_matched"].get<size_t>());
+
+ ASSERT_EQ("1108091339008", results["hits"][0]["text_match_info"]["best_field_score"].get<std::string>());
+ ASSERT_EQ("1108091339008", results["hits"][1]["text_match_info"]["best_field_score"].get<std::string>());
+ ASSERT_EQ("1108091339008", results["hits"][2]["text_match_info"]["best_field_score"].get<std::string>());
+
+ ASSERT_EQ(3, results["hits"][0]["text_match_info"]["best_field_weight"].get<size_t>());
+ ASSERT_EQ(3, results["hits"][1]["text_match_info"]["best_field_weight"].get<size_t>());
+ ASSERT_EQ(3, results["hits"][2]["text_match_info"]["best_field_weight"].get<size_t>());
+
collectionManager.drop_collection("coll1");
}