Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Return matched tokens in highlight response structure.
kishorenc6c1455bc2f4f
diff --git a/include/collection.h b/include/collection.hindex 065ba6f8..0a3f17ce 100644--- a/include/collection.h+++ b/include/collection.h@@ -106,6 +106,7 @@ private:std::vector<std::string> values;std::vector<size_t> indices;uint64_t match_score;+ std::vector<std::vector<std::string>> matched_tokens;highlight_t() {@@ -159,6 +160,8 @@ private:const size_t snippet_threshold,const size_t highlight_affix_num_tokens,bool highlighted_fully,+ const std::string& highlight_start_tag,+ const std::string& highlight_end_tag,highlight_t &highlight);void remove_document(const nlohmann::json & document, const uint32_t seq_id, bool remove_from_store);@@ -252,7 +255,9 @@ public:const std::map<size_t, std::vector<std::string>>& pinned_hits={},const std::vector<std::string>& hidden_hits={},const std::vector<std::string>& group_by_fields={},- const size_t group_limit = 0);+ const size_t group_limit = 0,+ const std::string& highlight_start_tag="<mark>",+ const std::string& highlight_end_tag="</mark>");Option<nlohmann::json> get(const std::string & id);diff --git a/include/index.h b/include/index.hindex 9e7570b9..854605d1 100644--- a/include/index.h+++ b/include/index.h@@ -139,8 +139,10 @@ private:StringUtils string_utils;- static inline std::vector<art_leaf *> next_suggestion(const std::vector<token_candidates> &token_candidates_vec,- long long int n);+ static inline void next_suggestion(const std::vector<token_candidates> &token_candidates_vec,+ long long int n,+ std::vector<art_leaf *>& actual_query_suggestion,+ std::vector<art_leaf *>& query_suggestion);void log_leaves(const int cost, const std::string &token, const std::vector<art_leaf *> &leaves) const;diff --git a/include/string_utils.h b/include/string_utils.hindex 46baf767..b7bfb207 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -290,4 +290,6 @@ struct StringUtils {static std::string randstring(size_t length);static std::string hmac(const std::string& key, const std::string& msg);++ static size_t unicode_length(const std::string& bytes);};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex 328eaaa3..c497271d 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -507,7 +507,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::const std::map<size_t, std::vector<std::string>>& pinned_hits,const std::vector<std::string>& hidden_hits,const std::vector<std::string>& group_by_fields,- const size_t group_limit) {+ const size_t group_limit,+ const std::string& highlight_start_tag,+ const std::string& highlight_end_tag) {if(query != "*" && search_fields.empty()) {return Option<nlohmann::json>(400, "No search fields specified for the query.");@@ -1080,7 +1082,7 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::highlight_t highlight;highlight_result(search_field, searched_queries, field_order_kv, document,string_utils, snippet_threshold, highlight_affix_num_tokens,- highlighted_fully, highlight);+ highlighted_fully, highlight_start_tag, highlight_end_tag, highlight);if(!highlight.snippets.empty()) {highlights.push_back(highlight);@@ -1096,13 +1098,14 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::bool highlight_fully = (fields_highlighted_fully.find(highlight.field) != fields_highlighted_fully.end());if(!highlight.indices.empty()) {+ h_json["matched_tokens"] = highlight.matched_tokens;h_json["indices"] = highlight.indices;h_json["snippets"] = highlight.snippets;if(highlight_fully) {h_json["values"] = highlight.values;}-} else {+ h_json["matched_tokens"] = highlight.matched_tokens[0];h_json["snippet"] = highlight.snippets[0];if(highlight_fully) {h_json["value"] = highlight.values[0];@@ -1205,7 +1208,9 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::// handle query token being larger than actual token (typo correction)query_token_len = std::min(query_token_len, tokens[i].size());const std::string & unmarked = tokens[i].substr(query_token_len, std::string::npos);- highlightedss << "<mark>" + tokens[i].substr(0, query_token_len) + "</mark>" + unmarked;+ highlightedss << highlight_start_tag ++ tokens[i].substr(0, query_token_len) ++ highlight_end_tag + unmarked;} else {highlightedss << tokens[i];}@@ -1354,10 +1359,12 @@ void Collection::highlight_result(const field &search_field,const size_t snippet_threshold,const size_t highlight_affix_num_tokens,bool highlighted_fully,+ const std::string& highlight_start_tag,+ const std::string& highlight_end_tag,highlight_t & highlight) {std::vector<uint32_t*> leaf_to_indices;- std::vector<art_leaf *> query_suggestion;+ std::vector<art_leaf*> query_suggestion;for (const art_leaf *token_leaf : searched_queries[field_order_kv->query_index]) {// Must search for the token string fresh on that field for the given document since `token_leaf`@@ -1418,9 +1425,9 @@ void Collection::highlight_result(const field &search_field,std::vector<std::string> tokens;if(search_field.type == field_types::STRING) {- StringUtils::split(document[search_field.name], tokens, " ");+ StringUtils::split(document[search_field.name], tokens, " ", true);} else {- StringUtils::split(document[search_field.name][match_index.index], tokens, " ");+ StringUtils::split(document[search_field.name][match_index.index], tokens, " ", true);}std::vector<size_t> token_indices;@@ -1445,7 +1452,7 @@ void Collection::highlight_result(const field &search_field,size_t prefix_length = highlight_affix_num_tokens;size_t suffix_length = highlight_affix_num_tokens + 1;- // For longer strings, pick surrounding tokens within 4 tokens of min_index and max_index for the snippet+ // For longer strings, pick surrounding tokens within `prefix_length` of min_index and max_index for snippetconst size_t start_index = (tokens.size() <= snippet_threshold) ? 0 :std::max(0, (int)(*(minmax.first) - prefix_length));@@ -1453,6 +1460,10 @@ void Collection::highlight_result(const field &search_field,std::min((int)tokens.size(), (int)(*(minmax.second) + suffix_length));std::stringstream snippet_stream;++ highlight.matched_tokens.emplace_back();+ std::vector<std::string>& matched_tokens = highlight.matched_tokens.back();+for(size_t snippet_index = start_index; snippet_index < end_index; snippet_index++) {if(snippet_index != start_index) {snippet_stream << " ";@@ -1462,7 +1473,8 @@ void Collection::highlight_result(const field &search_field,string_utils.unicode_normalize(token);if(token_hits.count(token) != 0) {- snippet_stream << "<mark>" + tokens[snippet_index] + "</mark>";+ snippet_stream << highlight_start_tag << tokens[snippet_index] << highlight_end_tag;+ matched_tokens.push_back(tokens[snippet_index]);} else {snippet_stream << tokens[snippet_index];}@@ -1484,7 +1496,7 @@ void Collection::highlight_result(const field &search_field,string_utils.unicode_normalize(token);if(token_hits.count(token) != 0) {- value_stream << "<mark>" + tokens[value_index] + "</mark>";+ value_stream << highlight_start_tag << tokens[value_index] << highlight_end_tag;} else {value_stream << tokens[value_index];}diff --git a/src/core_api.cpp b/src/core_api.cppindex a0e2ee21..d0fa76aa 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -272,6 +272,9 @@ bool get_search(http_req & req, http_res & res) {// list of fields which will be highlighted fully without snippetingconst char *HIGHLIGHT_FULL_FIELDS = "highlight_full_fields";+ const char *HIGHLIGHT_START_TAG = "highlight_start_tag";+ const char *HIGHLIGHT_END_TAG = "highlight_end_tag";+if(req.params.count(NUM_TYPOS) == 0) {req.params[NUM_TYPOS] = "2";}@@ -313,6 +316,14 @@ bool get_search(http_req & req, http_res & res) {req.params[HIGHLIGHT_FULL_FIELDS] = "";}+ if(req.params.count(HIGHLIGHT_START_TAG) == 0) {+ req.params[HIGHLIGHT_START_TAG] = "<mark>";+ }++ if(req.params.count(HIGHLIGHT_END_TAG) == 0) {+ req.params[HIGHLIGHT_END_TAG] = "</mark>";+ }+if(req.params.count(PER_PAGE) == 0) {if(req.params[FACET_QUERY].empty()) {req.params[PER_PAGE] = "10";diff --git a/src/index.cpp b/src/index.cppindex 4efa5575..adb81c6b 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -557,13 +557,17 @@ uint64_t Index::facet_token_hash(const field & a_field, const std::string &tokenvoid Index::index_string_field(const std::string & text, const int64_t score, art_tree *t,uint32_t seq_id, int facet_id, const field & a_field) {std::vector<std::string> tokens;- StringUtils::split(text, tokens, " ");+ StringUtils::split(text, tokens, " ", a_field.is_string());std::unordered_map<std::string, std::vector<uint32_t>> token_to_offsets;- for(uint32_t i=0; i<tokens.size(); i++) {+ for(size_t i=0; i<tokens.size(); i++) {auto & token = tokens[i];+ if(token.empty()) {+ continue;+ }+if(a_field.is_string()) {string_utils.unicode_normalize(token);}@@ -595,7 +599,7 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, cconst std::string & str = strings[array_index];std::vector<std::string> tokens;std::string delim = " ";- StringUtils::split(str, tokens, delim);+ StringUtils::split(str, tokens, delim, a_field.is_string());std::set<std::string> token_set; // required to deal with repeating tokens@@ -603,6 +607,10 @@ void Index::index_string_array_field(const std::vector<std::string> & strings, cfor(size_t i=0; i<tokens.size(); i++) {auto & token = tokens[i];+ if(token.empty()) {+ continue;+ }+if(a_field.is_string()) {string_utils.unicode_normalize(token);}@@ -903,7 +911,12 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, sifor(long long n=0; n<N && n<combination_limit; ++n) {// every element in `query_suggestion` contains a token and its associated hits- std::vector<art_leaf *> query_suggestion = next_suggestion(token_candidates_vec, n);+ std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());++ // actual query suggestion preserves original order of tokens in query+ std::vector<art_leaf*> actual_query_suggestion(token_candidates_vec.size());++ next_suggestion(token_candidates_vec, n, actual_query_suggestion, query_suggestion);/*LOG(INFO) << "n: " << n;for(size_t i=0; i < query_suggestion.size(); i++) {@@ -978,7 +991,7 @@ void Index::search_candidates(const uint8_t & field_id, uint32_t* filter_ids, sidelete[] result_ids;}- searched_queries.push_back(query_suggestion);+ searched_queries.push_back(actual_query_suggestion);//LOG(INFO) << "all_result_ids_len: " << all_result_ids_len << ", typo_tokens_threshold: " << typo_tokens_threshold;if(all_result_ids_len >= typo_tokens_threshold) {@@ -1854,14 +1867,15 @@ void Index::populate_token_positions(const std::vector<art_leaf *>& query_sugges}}-inline std::vector<art_leaf *> Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,- long long int n) {- std::vector<art_leaf*> query_suggestion(token_candidates_vec.size());-+inline void Index::next_suggestion(const std::vector<token_candidates> &token_candidates_vec,+ long long int n,+ std::vector<art_leaf *>& actual_query_suggestion,+ std::vector<art_leaf *>& query_suggestion) {// generate the next combination from `token_leaves` and store it in `query_suggestion`ldiv_t q { n, 0 };for(long long i = 0 ; i < (long long) token_candidates_vec.size(); i++) {q = ldiv(q.quot, token_candidates_vec[i].candidates.size());+ actual_query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];query_suggestion[i] = token_candidates_vec[i].candidates[q.rem];}@@ -1870,8 +1884,6 @@ inline std::vector<art_leaf *> Index::next_suggestion(const std::vector<token_casort(query_suggestion.begin(), query_suggestion.end(), [](const art_leaf* left, const art_leaf* right) {return left->values->ids.getLength() < right->values->ids.getLength();});-- return query_suggestion;}void Index::remove_and_shift_offset_index(sorted_array& offset_index, const uint32_t* indices_sorted,@@ -1989,11 +2001,11 @@ Option<uint32_t> Index::remove(const uint32_t seq_id, const nlohmann::json & docvoid Index::tokenize_doc_field(const nlohmann::json& document, const std::string& field_name, const field& search_field,std::vector<std::string>& tokens) {if(search_field.type == field_types::STRING) {- StringUtils::split(document[field_name], tokens, " ");+ StringUtils::split(document[field_name], tokens, " ", true);} else if(search_field.type == field_types::STRING_ARRAY) {const std::vector<std::string>& values = document[field_name].get<std::vector<std::string>>();for(const std::string & value: values) {- StringUtils::split(value, tokens, " ");+ StringUtils::split(value, tokens, " ", true);}} else if(search_field.type == field_types::INT32) {const int KEY_LEN = 8;diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 6c1e8835..39246d0f 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -3,6 +3,7 @@#include <openssl/evp.h>#include <openssl/hmac.h>#include <random>+#include <codecvt>std::string lower_and_no_special_chars(const std::string & str) {std::stringstream ss;@@ -29,6 +30,8 @@ void StringUtils::unicode_normalize(std::string & str) const {for (char *s = &str[0]; *s;) {char inbuf[5];char *p = inbuf;++ // group bytes to form a unicode representation*p++ = *s++;if ((*s & 0xC0) == 0x80) *p++ = *s++;if ((*s & 0xC0) == 0x80) *p++ = *s++;@@ -97,3 +100,8 @@ std::string StringUtils::str2hex(const std::string &str, bool capital) {return hexstr;}++size_t StringUtils::unicode_length(const std::string& bytes) {+ std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;+ return utf8conv.from_bytes(bytes).size();+}diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 92c4ab6d..91d3fd8c 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -726,31 +726,43 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {ASSERT_STREQ(id.c_str(), result_id.c_str());}- ASSERT_EQ(2, results["hits"][0]["highlights"][0].size());- ASSERT_STREQ(results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str(), "title");- ASSERT_STREQ(results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str(), "The <mark>Truth</mark> About Forever");+ ASSERT_EQ(3, results["hits"][0]["highlights"][0].size());+ ASSERT_STREQ("title", results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str());+ ASSERT_STREQ("The <mark>Truth</mark> About Forever", results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][0]["highlights"][0]["matched_tokens"].size());+ ASSERT_STREQ("Truth", results["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());- ASSERT_EQ(3, results["hits"][0]["highlights"][1].size());+ ASSERT_EQ(4, results["hits"][0]["highlights"][1].size());ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "tags");ASSERT_EQ(2, results["hits"][0]["highlights"][1]["snippets"].size());ASSERT_STREQ("the <mark>truth</mark>", results["hits"][0]["highlights"][1]["snippets"][0].get<std::string>().c_str());ASSERT_STREQ("<mark>truth</mark> about", results["hits"][0]["highlights"][1]["snippets"][1].get<std::string>().c_str());+ ASSERT_EQ(2, results["hits"][0]["highlights"][1]["matched_tokens"].size());+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][0][0].get<std::string>().c_str());+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][1][0].get<std::string>().c_str());+ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"].size());ASSERT_EQ(0, results["hits"][0]["highlights"][1]["indices"][0]);ASSERT_EQ(2, results["hits"][0]["highlights"][1]["indices"][1]);- ASSERT_EQ(2, results["hits"][1]["highlights"][0].size());- ASSERT_STREQ(results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str(), "title");- ASSERT_STREQ(results["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str(), "Plain <mark>Truth</mark>");+ ASSERT_EQ(3, results["hits"][1]["highlights"][0].size());+ ASSERT_STREQ("title", results["hits"][1]["highlights"][0]["field"].get<std::string>().c_str());+ ASSERT_STREQ("Plain <mark>Truth</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>().c_str());+ ASSERT_EQ(1, results["hits"][1]["highlights"][0]["matched_tokens"].size());+ ASSERT_STREQ("Truth", results["hits"][1]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());- ASSERT_EQ(3, results["hits"][1]["highlights"][1].size());+ ASSERT_EQ(4, results["hits"][1]["highlights"][1].size());ASSERT_STREQ(results["hits"][1]["highlights"][1]["field"].get<std::string>().c_str(), "tags");ASSERT_EQ(2, results["hits"][1]["highlights"][1]["snippets"].size());ASSERT_STREQ("<mark>truth</mark>", results["hits"][1]["highlights"][1]["snippets"][0].get<std::string>().c_str());ASSERT_STREQ("plain <mark>truth</mark>", results["hits"][1]["highlights"][1]["snippets"][1].get<std::string>().c_str());+ ASSERT_EQ(2, results["hits"][1]["highlights"][1]["matched_tokens"].size());+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][0][0].get<std::string>().c_str());+ ASSERT_STREQ("truth", results["hits"][0]["highlights"][1]["matched_tokens"][1][0].get<std::string>().c_str());+ASSERT_EQ(2, results["hits"][1]["highlights"][1]["indices"].size());ASSERT_EQ(1, results["hits"][1]["highlights"][1]["indices"][0]);ASSERT_EQ(2, results["hits"][1]["highlights"][1]["indices"][1]);@@ -761,16 +773,23 @@ TEST_F(CollectionTest, ArrayStringFieldHighlight) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ(2, results["hits"][0]["highlights"].size());- ASSERT_EQ(3, results["hits"][0]["highlights"][0].size());+ ASSERT_EQ(4, results["hits"][0]["highlights"][0].size());ASSERT_STREQ("tags", results["hits"][0]["highlights"][0]["field"].get<std::string>().c_str());ASSERT_STREQ("<mark>amazing</mark> <mark>movie</mark>", results["hits"][0]["highlights"][0]["snippets"][0].get<std::string>().c_str());ASSERT_EQ(1, results["hits"][0]["highlights"][0]["indices"].size());ASSERT_EQ(0, results["hits"][0]["highlights"][0]["indices"][0]);+ ASSERT_EQ(1, results["hits"][0]["highlights"][0]["matched_tokens"].size());+ ASSERT_STREQ("amazing", results["hits"][0]["highlights"][0]["matched_tokens"][0][0].get<std::string>().c_str());- ASSERT_EQ(2, results["hits"][0]["highlights"][1].size());+ ASSERT_EQ(3, results["hits"][0]["highlights"][1].size());ASSERT_STREQ(results["hits"][0]["highlights"][1]["field"].get<std::string>().c_str(), "title");ASSERT_STREQ(results["hits"][0]["highlights"][1]["snippet"].get<std::string>().c_str(),"<mark>Amazing</mark> Spiderman is <mark>amazing</mark>"); // should highlight duplicating tokens+ LOG(INFO) << results["hits"][0]["highlights"][1];++ ASSERT_EQ(2, results["hits"][0]["highlights"][1]["matched_tokens"].size());+ ASSERT_STREQ("Amazing", results["hits"][0]["highlights"][1]["matched_tokens"][0].get<std::string>().c_str());+ ASSERT_STREQ("amazing", results["hits"][0]["highlights"][1]["matched_tokens"][1].get<std::string>().c_str());// when query tokens are not found in an array field they should be ignoredresults = coll_array_text->search("winds", query_fields, "", facets, sort_fields, 0, 10, 1, FREQUENCY,@@ -1940,6 +1959,63 @@ TEST_F(CollectionTest, SearchHighlightShouldFollowThreshold) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionTest, SearchHighlightShouldUseHighlightTags) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, true),+ field("points", field_types::INT32, false)};++ std::vector<sort_by> sort_fields = {sort_by("points", "DESC")};++ coll1 = collectionManager.get_collection("coll1");+ if (coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 4, fields, "points").get();+ }++ nlohmann::json doc;+ doc["id"] = "100";+ doc["title"] = "The quick brown fox jumped over the lazy fox. "; // adding some extra spaces+ doc["points"] = 25;++ auto add_op = coll1->add(doc.dump());+ ASSERT_TRUE(add_op.ok());++ // use non-default highlighting tags++ auto res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "<em>", "</em>").get();++ ASSERT_STREQ("The quick brown fox jumped over the <em>lazy</em> fox. ",+ res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());++ // empty tags++ res = coll1->search("lazy", {"title"}, "", {}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0,+ "", "").get();++ ASSERT_STREQ("The quick brown fox jumped over the lazy fox. ",+ res["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());++ // check for matched tokens++ res = coll1->search("fox", {"title"}, "", {}, sort_fields, 0, 10, 1,+ token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "", 40, {}, {}, {}, 0).get();++ ASSERT_EQ(2, res["hits"][0]["highlights"][0]["matched_tokens"].size());+ ASSERT_STREQ("fox",+ res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());++ ASSERT_STREQ("fox.",+ res["hits"][0]["highlights"][0]["matched_tokens"][1].get<std::string>().c_str());++ collectionManager.drop_collection("coll1");+}+TEST_F(CollectionTest, UpdateDocument) {Collection *coll1;@@ -2201,7 +2277,7 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {token_ordering::FREQUENCY, true, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 5, 5, "").get();- ASSERT_EQ(2, res["hits"][0]["highlights"][0].size());+ ASSERT_EQ(3, res["hits"][0]["highlights"][0].size());// query multiple fieldsres = coll1->search("lazy", {"title", "tags"}, "", {}, sort_fields, 0, 10, 1,@@ -2212,6 +2288,9 @@ TEST_F(CollectionTest, SearchHighlightFieldFully) {ASSERT_STREQ("The quick brown fox jumped over the <mark>lazy</mark> dog and ran straight to the forest to sleep.",res["hits"][0]["highlights"][0]["value"].get<std::string>().c_str());+ ASSERT_EQ(1, res["hits"][0]["highlights"][0]["matched_tokens"].size());+ ASSERT_STREQ("lazy", res["hits"][0]["highlights"][0]["matched_tokens"][0].get<std::string>().c_str());+ASSERT_EQ(1, res["hits"][0]["highlights"][1]["values"][0].size());ASSERT_STREQ("<mark>LAZY</mark>", res["hits"][0]["highlights"][1]["values"][0].get<std::string>().c_str());