Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

add typo_prefix score and num_tokens_dropped in text_match_info (#1529)

Krunal Gandhi36fcdccddc12
diff --git a/include/collection.h b/include/collection.h
index 186c414c..522ff015 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -281,7 +281,8 @@ private:
nlohmann::json& override_metadata,
bool enable_typos_for_numerical_tokens=false) const;
- void populate_text_match_info(nlohmann::json& info, uint64_t match_score, const text_match_type_t match_type) const;
+ void populate_text_match_info(nlohmann::json& info, uint64_t match_score, const text_match_type_t match_type,
+ const size_t total_tokens) const;
bool handle_highlight_text(std::string& text, bool normalise, const field &search_field,
const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,
diff --git a/src/collection.cpp b/src/collection.cpp
index 9e1fa4cd..78c0ddcc 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2659,7 +2659,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
wrapper_doc["text_match"] = field_order_kv->text_match_score;
wrapper_doc["text_match_info"] = nlohmann::json::object();
populate_text_match_info(wrapper_doc["text_match_info"],
- field_order_kv->text_match_score, match_type);
+ field_order_kv->text_match_score, match_type,
+ field_query_tokens[0].q_include_tokens.size());
if(!vector_query.field_name.empty()) {
wrapper_doc["hybrid_search_info"] = nlohmann::json::object();
wrapper_doc["hybrid_search_info"]["rank_fusion_score"] = Index::int64_t_to_float(field_order_kv->scores[field_order_kv->match_score_index]);
@@ -3183,7 +3184,8 @@ uint64_t Collection::extract_bits(uint64_t value, unsigned lsb_offset, unsigned
}
void Collection::populate_text_match_info(nlohmann::json& info, uint64_t match_score,
- const text_match_type_t match_type) const {
+ const text_match_type_t match_type,
+ const size_t total_tokens) const {
// MAX_SCORE
// [ sign | tokens_matched | max_field_score | max_field_weight | num_matching_fields ]
@@ -3193,18 +3195,22 @@ void Collection::populate_text_match_info(nlohmann::json& info, uint64_t match_s
// [ sign | tokens_matched | max_field_weight | max_field_score | num_matching_fields ]
// [ 1 | 4 | 8 | 48 | 3 ] (64 bits)
+ auto tokens_matched = extract_bits(match_score, 59, 4);
+
info["score"] = std::to_string(match_score);
+ info["tokens_matched"] = tokens_matched;
+ info["fields_matched"] = extract_bits(match_score, 0, 3);
if(match_type == max_score) {
- info["tokens_matched"] = extract_bits(match_score, 59, 4);
info["best_field_score"] = std::to_string(extract_bits(match_score, 11, 48));
info["best_field_weight"] = extract_bits(match_score, 3, 8);
- info["fields_matched"] = extract_bits(match_score, 0, 3);
+ info["num_tokens_dropped"] = total_tokens - tokens_matched;
+ info["typo_prefix_score"] = 255 - extract_bits(match_score, 35, 8);
} else {
- info["tokens_matched"] = extract_bits(match_score, 59, 4);
info["best_field_weight"] = extract_bits(match_score, 51, 8);
info["best_field_score"] = std::to_string(extract_bits(match_score, 3, 48));
- info["fields_matched"] = extract_bits(match_score, 0, 3);
+ info["num_tokens_dropped"] = total_tokens - tokens_matched;
+ info["typo_prefix_score"] = 255 - extract_bits(match_score, 27, 8);
}
}
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 1e286e96..bc013023 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -2833,4 +2833,74 @@ TEST_F(CollectionSpecificMoreTest, TestStemming) {
auto no_stem_res = coll_no_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1);
ASSERT_TRUE(no_stem_res.ok());
ASSERT_EQ(0, no_stem_res.get()["hits"].size());
+}
+
+TEST_F(CollectionSpecificMoreTest, NumDroppedTokensTest) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection *coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "alpha beta";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["title"] = "beta gamma";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["title"] = "gamma delta";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["title"] = "delta epsilon";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["title"] = "epsilon alpha";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ bool exhaustive_search = false;
+ size_t drop_tokens_threshold = 5;
+
+ auto res = coll1->search("alpha zeta gamma", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true},
+ drop_tokens_threshold).get();
+
+ ASSERT_EQ(4, res["hits"].size());
+ ASSERT_EQ("4", res["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("epsilon alpha", res["hits"][0]["document"]["title"]);
+ ASSERT_EQ(2, res["hits"][0]["text_match_info"]["num_tokens_dropped"]);
+
+ ASSERT_EQ("2", res["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("gamma delta", res["hits"][1]["document"]["title"]);
+ ASSERT_EQ(2, res["hits"][1]["text_match_info"]["num_tokens_dropped"]);
+
+ ASSERT_EQ("1", res["hits"][2]["document"]["id"].get<std::string>());
+ ASSERT_EQ("beta gamma", res["hits"][2]["document"]["title"]);
+ ASSERT_EQ(2, res["hits"][2]["text_match_info"]["num_tokens_dropped"]);
+
+ ASSERT_EQ("0", res["hits"][3]["document"]["id"].get<std::string>());
+ ASSERT_EQ("alpha beta", res["hits"][3]["document"]["title"]);
+ ASSERT_EQ(2, res["hits"][3]["text_match_info"]["num_tokens_dropped"]);
+
+ res = coll1->search("zeta theta epsilon", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true},
+ drop_tokens_threshold).get();
+
+ ASSERT_EQ(2, res["hits"].size());
+ ASSERT_EQ("4", res["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("epsilon alpha", res["hits"][0]["document"]["title"]);
+ ASSERT_EQ(2, res["hits"][0]["text_match_info"]["num_tokens_dropped"]);
+
+ ASSERT_EQ("3", res["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ("delta epsilon", res["hits"][1]["document"]["title"]);
+ ASSERT_EQ(2, res["hits"][1]["text_match_info"]["num_tokens_dropped"]);
+
+ drop_tokens_threshold = 1;
+ res = coll1->search("alpha beta gamma", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true},
+ drop_tokens_threshold).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("alpha beta", res["hits"][0]["document"]["title"]);
+ ASSERT_EQ(1, res["hits"][0]["text_match_info"]["num_tokens_dropped"]);
}
\ No newline at end of file
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 388eb883..cb01319d 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -203,11 +203,12 @@ TEST_F(CollectionSpecificTest, ExactSingleFieldMatch) {
spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "", 30,
4, "title", 10).get();
- LOG(INFO) << results;
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["typo_prefix_score"]);
+ ASSERT_EQ(2, results["hits"][1]["text_match_info"]["typo_prefix_score"]);
// with typo_tokens_threshold = 1, only exact token match is fetched
results = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,
@@ -217,6 +218,7 @@ TEST_F(CollectionSpecificTest, ExactSingleFieldMatch) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["typo_prefix_score"]);
collectionManager.drop_collection("coll1");
}
@@ -253,6 +255,7 @@ TEST_F(CollectionSpecificTest, CheckProgressiveTypoSearching) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ(2, results["hits"][0]["text_match_info"]["typo_prefix_score"]);
// with typo_tokens_threshold = 10, both matches are fetched
results = coll1->search("convenient", {"title", "description"}, "", {}, {}, {2}, 10,
@@ -263,6 +266,8 @@ TEST_F(CollectionSpecificTest, CheckProgressiveTypoSearching) {
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ(2, results["hits"][0]["text_match_info"]["typo_prefix_score"]);
+ ASSERT_EQ(4, results["hits"][1]["text_match_info"]["typo_prefix_score"]);
collectionManager.drop_collection("coll1");
}
@@ -1078,7 +1083,9 @@ TEST_F(CollectionSpecificTest, CrossFieldMatchingExactMatchOnSingleField) {
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ(0, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);
results = coll1->search("john vegatable farmer", {"name", "description"},
"", {}, {}, {1, 1}, 10,
@@ -1089,7 +1096,9 @@ TEST_F(CollectionSpecificTest, CrossFieldMatchingExactMatchOnSingleField) {
ASSERT_EQ(2, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);
ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());
+ ASSERT_EQ(0, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);
collectionManager.drop_collection("coll1");
}
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index cbe23257..664dff23 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -3276,9 +3276,10 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {
ASSERT_EQ(2, results["found"].get<size_t>());
ASSERT_EQ(2, results["hits"].size());
-
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);
ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);
// changing weights to favor artist still favors title because it contains all tokens of the query
@@ -3289,7 +3290,9 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {
"<mark>", "</mark>", {1, 4}).get();
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);
ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);
// use same weights
@@ -3300,7 +3303,10 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {
"<mark>", "</mark>", {1, 1}).get();
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);
ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);
+
// add weights to favor artist without all tokens in a query being found in a field
@@ -3311,7 +3317,9 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {
"<mark>", "</mark>", {1, 4}).get();
ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);
ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);
collectionManager.drop_collection("coll1");
}
@@ -3400,7 +3408,9 @@ TEST_F(CollectionTest, MultiFieldRelevance3) {
ASSERT_EQ(2, results["hits"].size());
ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);
ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);
results = coll1->search("swift",
{"title", "artist"}, "", {}, {}, {0}, 10, 1, FREQUENCY,
@@ -3412,7 +3422,9 @@ TEST_F(CollectionTest, MultiFieldRelevance3) {
ASSERT_EQ(2, results["hits"].size());
ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);
ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());
+ ASSERT_EQ(0, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);
collectionManager.drop_collection("coll1");
}