Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
add typo_prefix score and num_tokens_dropped in text_match_info (#1529)
Krunal Gandhi36fcdccddc12
diff --git a/include/collection.h b/include/collection.hindex 186c414c..522ff015 100644--- a/include/collection.h+++ b/include/collection.h@@ -281,7 +281,8 @@ private:nlohmann::json& override_metadata,bool enable_typos_for_numerical_tokens=false) const;- void populate_text_match_info(nlohmann::json& info, uint64_t match_score, const text_match_type_t match_type) const;+ void populate_text_match_info(nlohmann::json& info, uint64_t match_score, const text_match_type_t match_type,+ const size_t total_tokens) const;bool handle_highlight_text(std::string& text, bool normalise, const field &search_field,const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,diff --git a/src/collection.cpp b/src/collection.cppindex 9e1fa4cd..78c0ddcc 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2659,7 +2659,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,wrapper_doc["text_match"] = field_order_kv->text_match_score;wrapper_doc["text_match_info"] = nlohmann::json::object();populate_text_match_info(wrapper_doc["text_match_info"],- field_order_kv->text_match_score, match_type);+ field_order_kv->text_match_score, match_type,+ field_query_tokens[0].q_include_tokens.size());if(!vector_query.field_name.empty()) {wrapper_doc["hybrid_search_info"] = nlohmann::json::object();wrapper_doc["hybrid_search_info"]["rank_fusion_score"] = Index::int64_t_to_float(field_order_kv->scores[field_order_kv->match_score_index]);@@ -3183,7 +3184,8 @@ uint64_t Collection::extract_bits(uint64_t value, unsigned lsb_offset, unsigned}void Collection::populate_text_match_info(nlohmann::json& info, uint64_t match_score,- const text_match_type_t match_type) const {+ const text_match_type_t match_type,+ const size_t total_tokens) const {// MAX_SCORE// [ sign | tokens_matched | max_field_score | max_field_weight | num_matching_fields ]@@ -3193,18 +3195,22 @@ void Collection::populate_text_match_info(nlohmann::json& info, uint64_t match_s// [ sign | tokens_matched | max_field_weight | max_field_score | num_matching_fields ]// [ 1 | 4 | 8 | 48 | 3 ] (64 bits)+ auto tokens_matched = extract_bits(match_score, 59, 4);+info["score"] = std::to_string(match_score);+ info["tokens_matched"] = tokens_matched;+ info["fields_matched"] = extract_bits(match_score, 0, 3);if(match_type == max_score) {- info["tokens_matched"] = extract_bits(match_score, 59, 4);info["best_field_score"] = std::to_string(extract_bits(match_score, 11, 48));info["best_field_weight"] = extract_bits(match_score, 3, 8);- info["fields_matched"] = extract_bits(match_score, 0, 3);+ info["num_tokens_dropped"] = total_tokens - tokens_matched;+ info["typo_prefix_score"] = 255 - extract_bits(match_score, 35, 8);} else {- info["tokens_matched"] = extract_bits(match_score, 59, 4);info["best_field_weight"] = extract_bits(match_score, 51, 8);info["best_field_score"] = std::to_string(extract_bits(match_score, 3, 48));- info["fields_matched"] = extract_bits(match_score, 0, 3);+ info["num_tokens_dropped"] = total_tokens - tokens_matched;+ info["typo_prefix_score"] = 255 - extract_bits(match_score, 27, 8);}}diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 1e286e96..bc013023 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -2833,4 +2833,74 @@ TEST_F(CollectionSpecificMoreTest, TestStemming) {auto no_stem_res = coll_no_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1);ASSERT_TRUE(no_stem_res.ok());ASSERT_EQ(0, no_stem_res.get()["hits"].size());+}++TEST_F(CollectionSpecificMoreTest, NumDroppedTokensTest) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection *coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["title"] = "alpha beta";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["title"] = "beta gamma";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["title"] = "gamma delta";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["title"] = "delta epsilon";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["title"] = "epsilon alpha";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ bool exhaustive_search = false;+ size_t drop_tokens_threshold = 5;++ auto res = coll1->search("alpha zeta gamma", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true},+ drop_tokens_threshold).get();++ ASSERT_EQ(4, res["hits"].size());+ ASSERT_EQ("4", res["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("epsilon alpha", res["hits"][0]["document"]["title"]);+ ASSERT_EQ(2, res["hits"][0]["text_match_info"]["num_tokens_dropped"]);++ ASSERT_EQ("2", res["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("gamma delta", res["hits"][1]["document"]["title"]);+ ASSERT_EQ(2, res["hits"][1]["text_match_info"]["num_tokens_dropped"]);++ ASSERT_EQ("1", res["hits"][2]["document"]["id"].get<std::string>());+ ASSERT_EQ("beta gamma", res["hits"][2]["document"]["title"]);+ ASSERT_EQ(2, res["hits"][2]["text_match_info"]["num_tokens_dropped"]);++ ASSERT_EQ("0", res["hits"][3]["document"]["id"].get<std::string>());+ ASSERT_EQ("alpha beta", res["hits"][3]["document"]["title"]);+ ASSERT_EQ(2, res["hits"][3]["text_match_info"]["num_tokens_dropped"]);++ res = coll1->search("zeta theta epsilon", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true},+ drop_tokens_threshold).get();++ ASSERT_EQ(2, res["hits"].size());+ ASSERT_EQ("4", res["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("epsilon alpha", res["hits"][0]["document"]["title"]);+ ASSERT_EQ(2, res["hits"][0]["text_match_info"]["num_tokens_dropped"]);++ ASSERT_EQ("3", res["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ("delta epsilon", res["hits"][1]["document"]["title"]);+ ASSERT_EQ(2, res["hits"][1]["text_match_info"]["num_tokens_dropped"]);++ drop_tokens_threshold = 1;+ res = coll1->search("alpha beta gamma", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true},+ drop_tokens_threshold).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("alpha beta", res["hits"][0]["document"]["title"]);+ ASSERT_EQ(1, res["hits"][0]["text_match_info"]["num_tokens_dropped"]);}\ No newline at end of filediff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 388eb883..cb01319d 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -203,11 +203,12 @@ TEST_F(CollectionSpecificTest, ExactSingleFieldMatch) {spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(), 10, "", 30,4, "title", 10).get();- LOG(INFO) << results;ASSERT_EQ(2, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["typo_prefix_score"]);+ ASSERT_EQ(2, results["hits"][1]["text_match_info"]["typo_prefix_score"]);// with typo_tokens_threshold = 1, only exact token match is fetchedresults = coll1->search("charger", {"title", "description"}, "", {}, {}, {2}, 10,@@ -217,6 +218,7 @@ TEST_F(CollectionSpecificTest, ExactSingleFieldMatch) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["typo_prefix_score"]);collectionManager.drop_collection("coll1");}@@ -253,6 +255,7 @@ TEST_F(CollectionSpecificTest, CheckProgressiveTypoSearching) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ(2, results["hits"][0]["text_match_info"]["typo_prefix_score"]);// with typo_tokens_threshold = 10, both matches are fetchedresults = coll1->search("convenient", {"title", "description"}, "", {}, {}, {2}, 10,@@ -263,6 +266,8 @@ TEST_F(CollectionSpecificTest, CheckProgressiveTypoSearching) {ASSERT_EQ(2, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ(2, results["hits"][0]["text_match_info"]["typo_prefix_score"]);+ ASSERT_EQ(4, results["hits"][1]["text_match_info"]["typo_prefix_score"]);collectionManager.drop_collection("coll1");}@@ -1078,7 +1083,9 @@ TEST_F(CollectionSpecificTest, CrossFieldMatchingExactMatchOnSingleField) {ASSERT_EQ(2, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ(0, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);results = coll1->search("john vegatable farmer", {"name", "description"},"", {}, {}, {1, 1}, 10,@@ -1089,7 +1096,9 @@ TEST_F(CollectionSpecificTest, CrossFieldMatchingExactMatchOnSingleField) {ASSERT_EQ(2, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);ASSERT_EQ("1", results["hits"][1]["document"]["id"].get<std::string>());+ ASSERT_EQ(0, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);collectionManager.drop_collection("coll1");}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex cbe23257..664dff23 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -3276,9 +3276,10 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {ASSERT_EQ(2, results["found"].get<size_t>());ASSERT_EQ(2, results["hits"].size());-ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);// changing weights to favor artist still favors title because it contains all tokens of the query@@ -3289,7 +3290,9 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {"<mark>", "</mark>", {1, 4}).get();ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);// use same weights@@ -3300,7 +3303,10 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {"<mark>", "</mark>", {1, 1}).get();ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);+// add weights to favor artist without all tokens in a query being found in a field@@ -3311,7 +3317,9 @@ TEST_F(CollectionTest, MultiFieldRelevance2) {"<mark>", "</mark>", {1, 4}).get();ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);collectionManager.drop_collection("coll1");}@@ -3400,7 +3408,9 @@ TEST_F(CollectionTest, MultiFieldRelevance3) {ASSERT_EQ(2, results["hits"].size());ASSERT_STREQ("1", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);ASSERT_STREQ("0", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);results = coll1->search("swift",{"title", "artist"}, "", {}, {}, {0}, 10, 1, FREQUENCY,@@ -3412,7 +3422,9 @@ TEST_F(CollectionTest, MultiFieldRelevance3) {ASSERT_EQ(2, results["hits"].size());ASSERT_STREQ("0", results["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(0, results["hits"][0]["text_match_info"]["num_tokens_dropped"]);ASSERT_STREQ("1", results["hits"][1]["document"]["id"].get<std::string>().c_str());+ ASSERT_EQ(0, results["hits"][1]["text_match_info"]["num_tokens_dropped"]);collectionManager.drop_collection("coll1");}