Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Simplify nested highlighting format.
Kishore Nallan23bd0af779a3
diff --git a/include/collection.h b/include/collection.hindex 298dfd12..6eda783a 100644--- a/include/collection.h+++ b/include/collection.h@@ -30,10 +30,11 @@ struct highlight_field_t {std::string name;bool fully_highlighted;bool infix;+ bool is_string;tsl::htrie_map<char, token_leaf> qtoken_leaves;- highlight_field_t(const std::string& name, bool fully_highlighted, bool infix):- name(name), fully_highlighted(fully_highlighted), infix(infix) {+ highlight_field_t(const std::string& name, bool fully_highlighted, bool infix, bool is_string):+ name(name), fully_highlighted(fully_highlighted), infix(infix), is_string(is_string) {}};@@ -134,8 +135,6 @@ private:const tsl::htrie_map<char, token_leaf>& qtoken_leaves,const KV* field_order_kv, const nlohmann::json &document,nlohmann::json& highlight_doc,- nlohmann::json& highlight_full_doc,- nlohmann::json& highlight_meta,StringUtils & string_utils,const size_t snippet_threshold,const size_t highlight_affix_num_tokens,@@ -245,10 +244,8 @@ private:template<class T>static bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,- const nlohmann::json& fdoc, nlohmann::json& fobj,- const nlohmann::json& mdoc, nlohmann::json& mobj,std::vector<std::string>& path_parts, size_t path_index,- int arr_index, nlohmann::json& parent_mobj, T func);+ T func);static Option<bool> resolve_field_type(field& new_field,nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,@@ -346,7 +343,7 @@ public:const index_operation_t op, const DIRTY_VALUES& dirty_values);static void prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,- const tsl::htrie_set<char>& exclude_names, std::string parent_name = "", size_t depth = 0);+ const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "", size_t depth = 0);const Index* _get_index() const;@@ -478,6 +475,7 @@ public:size_t snippet_start_offset) ;void process_highlight_fields(const std::vector<search_field_t>& search_fields,+ const std::vector<std::string>& raw_search_fields,const tsl::htrie_set<char>& exclude_fields,const tsl::htrie_set<char>& include_fields,const std::vector<std::string>& highlight_field_names,@@ -501,18 +499,16 @@ public:template<class T>bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,- const nlohmann::json& fdoc, nlohmann::json& fobj,- const nlohmann::json& mdoc, nlohmann::json& mobj,- std::vector<std::string>& path_parts, size_t path_index,- int arr_index, nlohmann::json& parent_mobj,- T func) {+ std::vector<std::string>& path_parts, size_t path_index,+ T func) {if(path_index == path_parts.size()) {// end of path: guaranteed to be a stringif(!hobj.is_string()) {- return false;+ //return false;}- func(hobj, fobj, mobj, arr_index, parent_mobj);+ func(hobj);+ return true;}const std::string& fragment = path_parts[path_index];@@ -521,24 +517,13 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::jsif(it != hobj.end()) {if(it.value().is_array()) {bool resolved = false;- auto& parent_mobj = mobj.empty() ? mobj : mobj[fragment];- nlohmann::json empty_obj;-for(size_t i = 0; i < it.value().size(); i++) {auto& h_ele = it.value().at(i);- auto& f_ele = fobj.empty() ? fobj : fobj[fragment][i];- // if value is a primitive, we don't want the array object getting destroyed- auto& m_ele = h_ele.is_object() && !mobj.empty() ? mobj[fragment][i] : empty_obj;- resolved |= highlight_nested_field(hdoc, h_ele, fdoc, f_ele, mdoc, m_ele,- path_parts, path_index + 1, i, parent_mobj, func);+ resolved |= highlight_nested_field(hdoc, h_ele, path_parts, path_index + 1, func);}return resolved;} else {- auto& f_ele = fobj.empty() ? fobj : fobj[fragment];- auto& m_ele = mobj.empty() ? mobj : mobj[fragment];- auto& parent_mobj = mobj.empty() ? mobj : mobj[fragment];- return highlight_nested_field(hdoc, it.value(), fdoc, f_ele, mdoc, m_ele, path_parts, path_index + 1,- -1, parent_mobj, func);+ return highlight_nested_field(hdoc, it.value(), path_parts, path_index + 1, func);}} {return false;diff --git a/src/collection.cpp b/src/collection.cppindex dec6f33e..49df2df8 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1437,7 +1437,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,StringUtils::split(highlight_full_fields, highlight_full_field_names, ",");if(query != "*") {- process_highlight_fields(weighted_search_fields, include_fields_full, exclude_fields_full,+ process_highlight_fields(weighted_search_fields, raw_search_fields, include_fields_full, exclude_fields_full,highlight_field_names, highlight_full_field_names, infixes, q_tokens,search_params->qtoken_set, highlight_items);@@ -1490,23 +1490,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,nlohmann::json highlight_res = nlohmann::json::object();if(!highlight_items.empty()) {- highlight_res["meta"] = nlohmann::json::object();- copy_highlight_doc(highlight_items, document, highlight_res["meta"]);- remove_flat_fields(highlight_res["meta"]);- highlight_res["meta"].erase("id");-- highlight_res["snippet"] = nlohmann::json::object();- copy_highlight_doc(highlight_items, document, highlight_res["snippet"]);- remove_flat_fields(highlight_res["snippet"]);- highlight_res["snippet"].erase("id");-- if(has_atleast_one_fully_highlighted_field) {- copy_highlight_doc(highlight_items, document, highlight_res["full"]);- remove_flat_fields(highlight_res["full"]);- highlight_res["full"].erase("id");- } else {- highlight_res["full"] = nlohmann::json::object();- }+ copy_highlight_doc(highlight_items, document, highlight_res);+ remove_flat_fields(highlight_res);+ highlight_res.erase("id");}nlohmann::json wrapper_doc;@@ -1526,9 +1512,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,field search_field = search_schema.at(field_name);- if(query != "*" && (search_field.type == field_types::STRING ||- search_field.type == field_types::STRING_ARRAY)) {-+ if(query != "*") {highlight_t highlight;highlight.field = search_field.name;@@ -1536,7 +1520,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,bool found_full_highlight = false;highlight_result(raw_query, search_field, i, highlight_item.qtoken_leaves, field_order_kv,- document, highlight_res["snippet"], highlight_res["full"], highlight_res["meta"],+ document, highlight_res,string_utils, snippet_threshold,highlight_affix_num_tokens, highlight_item.fully_highlighted, highlight_item.infix,highlight_start_tag, highlight_end_tag, index_symbols, highlight,@@ -1562,22 +1546,25 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,}}- for(auto& hfield_name: highlight_field_names) {- auto it = hfield_names.equal_prefix_range(hfield_name);- if(it.first != it.second) {- hfield_names.insert(hfield_name);+ if(highlight_field_names.empty()) {+ for(auto& raw_search_field: raw_search_fields) {+ auto it = hfield_names.equal_prefix_range(raw_search_field);+ if(it.first != it.second) {+ hfield_names.insert(raw_search_field);+ }+ }+ } else {+ for(auto& hfield_name: highlight_field_names) {+ auto it = hfield_names.equal_prefix_range(hfield_name);+ if(it.first != it.second) {+ hfield_names.insert(hfield_name);+ }}}// remove fields from highlight doc that were not highlightedif(!hfield_names.empty()) {- prune_doc(highlight_res["meta"], hfield_names, tsl::htrie_set<char>(), "");- prune_doc(highlight_res["snippet"], hfield_names, tsl::htrie_set<char>(), "");- prune_doc(highlight_res["full"], h_full_field_names, tsl::htrie_set<char>(), "");- } else {- highlight_res["meta"] = nlohmann::json::object();- highlight_res["snippet"] = nlohmann::json::object();- highlight_res["full"] = nlohmann::json::object();+ prune_doc(highlight_res, hfield_names, tsl::htrie_set<char>(), "");}std::sort(highlights.begin(), highlights.end());@@ -1951,6 +1938,7 @@ void Collection::populate_text_match_info(nlohmann::json& info, uint64_t match_s}void Collection::process_highlight_fields(const std::vector<search_field_t>& search_fields,+ const std::vector<std::string>& raw_search_fields,const tsl::htrie_set<char>& include_fields,const tsl::htrie_set<char>& exclude_fields,const std::vector<std::string>& highlight_field_names,@@ -1985,8 +1973,14 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& sea}if(highlight_field_names.empty()) {- for(size_t i = 0; i < search_fields.size(); i++) {- const auto& field_name = search_fields[i].name;+ std::vector<std::string> highlight_field_names_expanded;++ for(size_t i = 0; i < raw_search_fields.size(); i++) {+ extract_field_name(raw_search_fields[i], search_schema, highlight_field_names_expanded, false, enable_nested_fields);+ }++ for(size_t i = 0; i < highlight_field_names_expanded.size(); i++) {+ const auto& field_name = highlight_field_names_expanded[i];if(exclude_fields.count(field_name) != 0) {// should not pick excluded field for highlighting (only for implicit highlighting)continue;@@ -1999,23 +1993,28 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& seabool fully_highlighted = (fields_highlighted_fully_set.count(field_name) != 0);bool infixed = (fields_infixed_set.count(field_name) != 0);- highlight_items.emplace_back(field_name, fully_highlighted, infixed);+ auto schema_it = search_schema.find(field_name);+ bool is_string = (schema_it != search_schema.end()) && schema_it->is_string();++ highlight_items.emplace_back(field_name, fully_highlighted, infixed, is_string);}} else {std::vector<std::string> highlight_field_names_expanded;for(size_t i = 0; i < highlight_field_names.size(); i++) {- extract_field_name(highlight_field_names[i], search_schema, highlight_field_names_expanded, true, enable_nested_fields);+ extract_field_name(highlight_field_names[i], search_schema, highlight_field_names_expanded, false, enable_nested_fields);}for(size_t i = 0; i < highlight_field_names_expanded.size(); i++) {const auto& highlight_field_name = highlight_field_names_expanded[i];- if(search_schema.count(highlight_field_name) == 0) {+ auto schema_it = search_schema.find(highlight_field_name);+ if(schema_it == search_schema.end()) {// ignore fields not part of schemacontinue;}bool fully_highlighted = (fields_highlighted_fully_set.count(highlight_field_name) != 0);bool infixed = (fields_infixed_set.count(highlight_field_name) != 0);- highlight_items.emplace_back(highlight_field_name, fully_highlighted, infixed);+ bool is_string = schema_it->is_string();+ highlight_items.emplace_back(highlight_field_name, fully_highlighted, infixed, is_string);}}@@ -2024,6 +2023,10 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& seait.key(qtoken);for(auto& highlight_item: highlight_items) {+ if(!highlight_item.is_string) {+ continue;+ }+const auto& field_name = highlight_item.name;art_leaf* leaf = index->get_token_leaf(field_name, (const unsigned char*) qtoken.c_str(), qtoken.size()+1);if(leaf) {@@ -2039,6 +2042,9 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& seafor(auto& q_token: q_tokens) {if(qtoken_set.find(q_token) == qtoken_set.end()) {for(auto& highlight_item: highlight_items) {+ if(!highlight_item.is_string) {+ continue;+ }const auto& field_name = highlight_item.name;art_leaf* leaf = index->get_token_leaf(field_name, (const unsigned char*) q_token.c_str(), q_token.size()+1);if(leaf) {@@ -2324,8 +2330,6 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaconst tsl::htrie_map<char, token_leaf>& qtoken_leaves,const KV* field_order_kv, const nlohmann::json & document,nlohmann::json& highlight_doc,- nlohmann::json& highlight_full_doc,- nlohmann::json& highlight_meta,StringUtils & string_utils,const size_t snippet_threshold,const size_t highlight_affix_num_tokens,@@ -2358,12 +2362,6 @@ void Collection::highlight_result(const std::string& raw_query, const field &seasize_t prefix_token_num_chars = StringUtils::get_num_chars(last_raw_q_token);std::set<std::string> last_full_q_tokens;-- if(qtoken_leaves.empty() && !is_infix_search) {- // none of the tokens from the query were found on this field- return ;- }-std::vector<match_index_t> match_indices;if(is_infix_search) {@@ -2396,77 +2394,93 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaLOG(INFO) << "Token: " << qtok_buff << ", root_len: " << it.value().root_len;}*/- /* We have to handle the following differently:+ bool flat_field = highlight_doc.contains(search_field.name);- a) top-level field or a simple nested field- b) field inside an array of objects+ std::vector<std::string> path_parts;+ if(enable_nested_fields && !flat_field) {+ StringUtils::split(search_field.name, path_parts, ".");+ } else {+ path_parts = {search_field.name};+ }- For a) we can rely on matching index positions since field value on-disk will be in the same order- as the field value indexed. For b) we need to do a keyword based highlight since we cannot use- indexed offsets.- */+ highlight_nested_field(highlight_doc, highlight_doc, path_parts, 0, [&](nlohmann::json& h_obj) {+ Match match;+ match_index_t match_index(match, 0, 0);+ int last_valid_offset_index = -1;+ size_t last_valid_offset = 0;- int nested_array = search_field.nested_array;- std::vector<std::string> path_parts;+ highlight_t array_highlight = highlight;- if(nested_array == field::VAL_UNKNOWN) {- StringUtils::split(search_field.name, path_parts, ".");- nested_array = Collection::is_nested_array(document, path_parts, 0);- }-- if(nested_array) {- if(path_parts.empty()) {- StringUtils::split(search_field.name, path_parts, ".");- }-- highlight_nested_field(highlight_doc, highlight_doc, highlight_full_doc, highlight_full_doc,- highlight_meta, highlight_meta, path_parts, 0, -1, highlight_meta,- [&](nlohmann::json& h_obj, nlohmann::json& f_obj, nlohmann::json& m_obj,- int arr_index, nlohmann::json& parent_mobj) {- Match match;- match_index_t match_index(match, 0, 0);- int last_valid_offset_index = -1;- size_t last_valid_offset = 0;-- highlight_t array_highlight = highlight;-- std::string text = h_obj.get<std::string>();- handle_highlight_text(text, normalise, search_field, symbols_to_index,- token_separators, array_highlight, string_utils, use_word_tokenizer,- highlight_affix_num_tokens,- qtoken_leaves, last_valid_offset_index, match,- prefix_token_num_chars,- highlight_fully, snippet_threshold, is_infix_search,- raw_query_tokens,- last_valid_offset, highlight_start_tag, highlight_end_tag,- index_symbols, match_index);-- if(!array_highlight.snippets.empty()) {- h_obj = array_highlight.snippets[0];- found_highlight = found_highlight || true;-- nlohmann::json& ref_meta_obj = (arr_index >= 0) ? parent_mobj : m_obj;- if(!ref_meta_obj.is_object() || !ref_meta_obj.contains("matched_tokens")) {- ref_meta_obj = nlohmann::json::object();- }+ if(!h_obj.is_string()) {+ auto val_back = h_obj;+ h_obj = nlohmann::json::object();+ h_obj["snippet"] = to_string(val_back);+ h_obj["matched_tokens"] = nlohmann::json::array();+ if(highlight_fully) {+ h_obj["value"] = val_back;+ }+ return ;+ }- for(auto& token_vec: array_highlight.matched_tokens) {- for(auto& token: token_vec) {- ref_meta_obj["matched_tokens"].push_back(token);- }- }+ std::string text = h_obj.get<std::string>();+ h_obj = nlohmann::json::object();- if(arr_index >= 0) {- ref_meta_obj["matched_indices"].push_back(arr_index);- }+ if(qtoken_leaves.empty()) {+ h_obj["snippet"] = text;+ h_obj["matched_tokens"] = nlohmann::json::array();+ if(highlight_fully) {+ h_obj["value"] = text;}+ return ;+ }- if(highlight_fully && !array_highlight.values.empty()) {- f_obj = array_highlight.values[0];;- found_full_highlight = found_full_highlight || true;+ handle_highlight_text(text, normalise, search_field, symbols_to_index,+ token_separators, array_highlight, string_utils, use_word_tokenizer,+ highlight_affix_num_tokens,+ qtoken_leaves, last_valid_offset_index, match,+ prefix_token_num_chars,+ highlight_fully, snippet_threshold, is_infix_search,+ raw_query_tokens,+ last_valid_offset, highlight_start_tag, highlight_end_tag,+ index_symbols, match_index);+++ if(array_highlight.snippets.empty() && array_highlight.values.empty()) {+ h_obj["snippet"] = text;+ h_obj["matched_tokens"] = nlohmann::json::array();+ }++ if(!array_highlight.snippets.empty()) {+ found_highlight = found_highlight || true;++ h_obj["snippet"] = array_highlight.snippets[0];+ h_obj["matched_tokens"] = nlohmann::json::array();++ for(auto& token_vec: array_highlight.matched_tokens) {+ for(auto& token: token_vec) {+ h_obj["matched_tokens"].push_back(token);+ }}- });+ }+ if(!array_highlight.values.empty()) {+ h_obj["value"] = array_highlight.values[0];;+ found_full_highlight = found_full_highlight || true;+ } else if(highlight_fully) {+ h_obj["value"] = text;+ }+ });++ if(!flat_field) {+ return;+ }++ if(qtoken_leaves.empty()) {+ // none of the tokens from the query were found on this field+ return ;+ }++ if(!search_field.is_string()) {return ;}@@ -2571,86 +2585,6 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaif(!match_indices.empty()) {highlight.match_score = match_indices[0].match_score;}-- // in-place highlighting under the new highlight structure- std::vector<std::string> parts;- if(!enable_nested_fields || highlight_doc.contains(search_field.name)) {- parts = {search_field.name};- } else {- StringUtils::split(search_field.name, parts, ".");- }-- nlohmann::json* mval = highlight_meta.contains(parts[0]) ? &highlight_meta[parts[0]] : nullptr;- nlohmann::json* hval = highlight_doc.contains(parts[0]) ? &highlight_doc[parts[0]] : nullptr;- nlohmann::json* fval = highlight_full_doc.contains(parts[0]) ? &highlight_full_doc[parts[0]] : nullptr;-- for(size_t i = 1; mval != nullptr && i < parts.size(); i++) {- const auto& part = parts[i];- if(mval->contains(part)) {- mval = &mval->at(part);- } else {- mval = nullptr;- }- }-- for(size_t i = 1; hval != nullptr && i < parts.size(); i++) {- const auto& part = parts[i];- if(hval->contains(part)) {- hval = &hval->at(part);- } else {- hval = nullptr;- }- }-- for(size_t i = 1; fval != nullptr && i < parts.size(); i++) {- const auto& part = parts[i];- if(fval->contains(part)) {- fval = &fval->at(part);- } else {- fval = nullptr;- }- }-- if(hval) {- if(highlight.indices.empty()) {- *hval = highlight.snippets[0];- } else {- if(hval->is_array()) {- for(size_t hi = 0; hi < highlight.indices.size(); hi++) {- hval->at(highlight.indices[hi]) = highlight.snippets[hi];- }- }- }-- if(mval) {- *mval = nlohmann::json::object();- mval->emplace("matched_tokens", nlohmann::json::array());- for(auto it = matched_tokens.begin(); it != matched_tokens.end(); ++it) {- mval->at("matched_tokens").push_back(it.key());- }-- if(hval->is_array()) {- mval->emplace("matched_indices", nlohmann::json::array());- for(auto highlight_index: highlight.indices) {- mval->at("matched_indices").push_back(highlight_index);- }- }- }- }-- if(fval) {- if(!highlight.values.empty()) {- if(highlight.indices.empty()) {- *fval = highlight.values[0];- } else {- if(fval->is_array()) {- for(size_t hi = 0; hi < highlight.values.size(); hi++) {- fval->at(highlight.indices[hi]) = highlight.values[hi];- }- }- }- }- }}bool Collection::handle_highlight_text(std::string& text, bool normalise, const field &search_field,@@ -3483,7 +3417,7 @@ void Collection::remove_flat_fields(nlohmann::json& document) {void Collection::prune_doc(nlohmann::json& doc,const tsl::htrie_set<char>& include_names,const tsl::htrie_set<char>& exclude_names,- std::string parent_name, size_t depth) {+ const std::string& parent_name, size_t depth) {// doc can only be an objectauto it = doc.begin();while(it != doc.end()) {diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex ad30313f..dfecfb54 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -349,21 +349,85 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {ASSERT_EQ(doc, results["hits"][0]["document"]);auto highlight_doc = R"({- "employees":{- "tags":[- "senior plumber",- "<mark>electrician</mark>"+ "employees": {+ "num": {+ "matched_tokens": [],+ "snippet": "1200"+ },+ "tags": [+ {+ "matched_tokens": [],+ "snippet": "senior plumber"+ },+ {+ "matched_tokens": [+ "electrician"+ ],+ "snippet": "<mark>electrician</mark>"+ }]},- "locations":[+ "locations": [{- "address":{- "street":"One Bowerman Drive"+ "address": {+ "city": {+ "matched_tokens": [],+ "snippet": "Beaverton"+ },+ "products": [+ {+ "matched_tokens": [],+ "snippet": "shoes"+ },+ {+ "matched_tokens": [],+ "snippet": "tshirts"+ }+ ],+ "street": {+ "matched_tokens": [],+ "snippet": "One Bowerman Drive"+ }+ },+ "country": {+ "matched_tokens": [],+ "snippet": "USA"+ },+ "pincode": {+ "matched_tokens": [],+ "snippet": "100"}},{- "address":{- "street":"175 <mark>Commerce</mark> Valley"+ "address": {+ "city": {+ "matched_tokens": [],+ "snippet": "Thornhill"+ },+ "products": [+ {+ "matched_tokens": [],+ "snippet": "sneakers"+ },+ {+ "matched_tokens": [],+ "snippet": "shoes"+ }+ ],+ "street": {+ "matched_tokens": [+ "Commerce"+ ],+ "snippet": "175 <mark>Commerce</mark> Valley"+ }+ },+ "country": {+ "matched_tokens": [],+ "snippet": "Canada"+ },+ "pincode": {+ "matched_tokens": [],+ "snippet": "200"}}]@@ -380,7 +444,7 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {ASSERT_EQ(0, raw_doc.count("employees.tags"));ASSERT_EQ(4, raw_doc.size());- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());ASSERT_EQ(0, results["hits"][0]["highlights"].size());// after update also the flat fields or meta should not be present on disk@@ -404,18 +468,26 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {"locations":[{"address":{- "street":"<mark>One</mark> Bowerman Drive"+ "street":{+ "matched_tokens":[+ "One"+ ],+ "snippet":"<mark>One</mark> Bowerman Drive"+ }}},{"address":{- "street":"175 Commerce Valley"+ "street":{+ "matched_tokens":[],+ "snippet":"175 Commerce Valley"+ }}}]})"_json;- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());ASSERT_EQ(0, results["hits"][0]["highlights"].size());// try to search nested fields that don't exist@@ -510,7 +582,6 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {auto add_op = coll1->add(doc.dump(), CREATE);ASSERT_TRUE(add_op.ok());- // search both simply nested and deeply nested array-of-objectsauto results = coll1->search("One", {"locations.address"}, "", {}, sort_fields, {0}, 10, 1,token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address").get();@@ -521,44 +592,76 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {"locations":[{"address":{- "street":"<mark>One</mark> Bowerman Drive"+ "city":{+ "matched_tokens":[++ ],+ "snippet":"Beaverton",+ "value":"Beaverton"+ },+ "products":[+ {+ "matched_tokens":[++ ],+ "snippet":"shoes",+ "value":"shoes"+ },+ {+ "matched_tokens":[++ ],+ "snippet":"tshirts",+ "value":"tshirts"+ }+ ],+ "street":{+ "matched_tokens":[+ "One"+ ],+ "snippet":"<mark>One</mark> Bowerman Drive",+ "value":"<mark>One</mark> Bowerman Drive"+ }}},{"address":{- "street":"175 Commerce Drive"- }- }- ]- })"_json;+ "city":{+ "matched_tokens":[- auto highlight_full_doc = R"({- "locations":[- {- "address":{- "city":"Beaverton",- "products":[- "shoes",- "tshirts"],- "street":"<mark>One</mark> Bowerman Drive"- }- },- {- "address":{- "city":"Thornhill",- "products":[- "sneakers",- "shoes"+ "snippet":"Thornhill",+ "value":"Thornhill"+ },+ "products":[+ {+ "matched_tokens":[++ ],+ "snippet":"sneakers",+ "value":"sneakers"+ },+ {+ "matched_tokens":[++ ],+ "snippet":"shoes",+ "value":"shoes"+ }+ ],+ "street":{+ "matched_tokens":[+],- "street":"175 Commerce Drive"+ "snippet":"175 Commerce Drive",+ "value":"175 Commerce Drive"}}- ]+ }+ ]})"_json;- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());- ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["full"].dump());+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());ASSERT_EQ(0, results["hits"][0]["highlights"].size());// repeating token@@ -573,18 +676,76 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {"locations":[{"address":{- "street":"One Bowerman <mark>Drive</mark>"+ "city":{+ "matched_tokens":[++ ],+ "snippet":"Beaverton",+ "value":"Beaverton"+ },+ "products":[+ {+ "matched_tokens":[++ ],+ "snippet":"shoes",+ "value":"shoes"+ },+ {+ "matched_tokens":[++ ],+ "snippet":"tshirts",+ "value":"tshirts"+ }+ ],+ "street":{+ "matched_tokens":[+ "Drive"+ ],+ "snippet":"One Bowerman <mark>Drive</mark>",+ "value":"One Bowerman <mark>Drive</mark>"+ }}},{"address":{- "street":"175 Commerce <mark>Drive</mark>"+ "city":{+ "matched_tokens":[++ ],+ "snippet":"Thornhill",+ "value":"Thornhill"+ },+ "products":[+ {+ "matched_tokens":[++ ],+ "snippet":"sneakers",+ "value":"sneakers"+ },+ {+ "matched_tokens":[++ ],+ "snippet":"shoes",+ "value":"shoes"+ }+ ],+ "street":{+ "matched_tokens":[+ "Drive"+ ],+ "snippet":"175 Commerce <mark>Drive</mark>",+ "value":"175 Commerce <mark>Drive</mark>"+ }}}]})"_json;- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());ASSERT_EQ(0, results["hits"][0]["highlights"].size());// nested array of array, highlighting parent of searched nested field@@ -595,67 +756,81 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {"locations.address").get();ASSERT_EQ(1, results["hits"].size());- highlight_full_doc = R"({++ highlight_doc = R"({"locations":[{"address":{- "city":"Beaverton",+ "city":{+ "matched_tokens":[++ ],+ "snippet":"Beaverton",+ "value":"Beaverton"+ },"products":[- "<mark>shoes</mark>",- "tshirts"+ {+ "matched_tokens":[+ "shoes"+ ],+ "snippet":"<mark>shoes</mark>",+ "value":"<mark>shoes</mark>"+ },+ {+ "matched_tokens":[++ ],+ "snippet":"tshirts",+ "value":"tshirts"+ }],- "street":"One Bowerman Drive"+ "street":{+ "matched_tokens":[++ ],+ "snippet":"One Bowerman Drive",+ "value":"One Bowerman Drive"+ }}},{"address":{- "city":"Thornhill",+ "city":{+ "matched_tokens":[++ ],+ "snippet":"Thornhill",+ "value":"Thornhill"+ },"products":[- "sneakers",- "<mark>shoes</mark>"+ {+ "matched_tokens":[++ ],+ "snippet":"sneakers",+ "value":"sneakers"+ },+ {+ "matched_tokens":[+ "shoes"+ ],+ "snippet":"<mark>shoes</mark>",+ "value":"<mark>shoes</mark>"+ }],- "street":"175 Commerce Drive"+ "street":{+ "matched_tokens":[++ ],+ "snippet":"175 Commerce Drive",+ "value":"175 Commerce Drive"+ }}}]})"_json;- auto highlight_meta_doc = R"({- "locations":[- {- "address":{- "city":"Beaverton",- "products":{- "matched_indices":[- 0- ],- "matched_tokens":[- "shoes"- ]- },- "street":"One Bowerman Drive"- }- },- {- "address":{- "city":"Thornhill",- "products":{- "matched_indices":[- 1- ],- "matched_tokens":[- "shoes"- ]- },- "street":"175 Commerce Drive"- }- }- ]- })"_json;-- ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["full"].dump());- ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());- ASSERT_EQ(highlight_meta_doc.dump(), results["hits"][0]["highlight"]["meta"].dump());+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());// full highlighting only one of the 3 highlight fieldsresults = coll1->search("drive", {"company.names", "company_names", "locations.address"}, "", {}, sort_fields, {0}, 10, 1,@@ -664,62 +839,94 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false, true,"company.names,company_names,locations.address").get();- highlight_full_doc = R"({- "locations":[+ highlight_doc = R"({+ "company": {+ "names": [{- "address":{- "city":"Beaverton",- "products":[- "shoes",- "tshirts"- ],- "street":"One Bowerman <mark>Drive</mark>"- }+ "matched_tokens": [],+ "snippet": "Space Corp. LLC"},{- "address":{- "city":"Thornhill",- "products":[- "sneakers",- "shoes"- ],- "street":"175 Commerce <mark>Drive</mark>"- }+ "matched_tokens": [+ "Drive"+ ],+ "snippet": "<mark>Drive</mark> One Inc."}]- })"_json;-- highlight_doc = R"({- "company":{- "names": ["Space Corp. LLC", "<mark>Drive</mark> One Inc."]+ },+ "company_names": [+ {+ "matched_tokens": [],+ "snippet": "Space Corp. LLC"},- "company_names": ["Space Corp. LLC", "<mark>Drive</mark> One Inc."],- "locations":[- {- "address":{- "city":"Beaverton",- "products":[- "shoes",- "tshirts"+ {+ "matched_tokens": [+ "Drive"+ ],+ "snippet": "<mark>Drive</mark> One Inc."+ }+ ],+ "locations": [+ {+ "address": {+ "city": {+ "matched_tokens": [],+ "snippet": "Beaverton",+ "value": "Beaverton"+ },+ "products": [+ {+ "matched_tokens": [],+ "snippet": "shoes",+ "value": "shoes"+ },+ {+ "matched_tokens": [],+ "snippet": "tshirts",+ "value": "tshirts"+ }+ ],+ "street": {+ "matched_tokens": [+ "Drive"],- "street":"One Bowerman <mark>Drive</mark>"+ "snippet": "One Bowerman <mark>Drive</mark>",+ "value": "One Bowerman <mark>Drive</mark>"}- },- {- "address":{- "city":"Thornhill",- "products":[- "sneakers",- "shoes"+ }+ },+ {+ "address": {+ "city": {+ "matched_tokens": [],+ "snippet": "Thornhill",+ "value": "Thornhill"+ },+ "products": [+ {+ "matched_tokens": [],+ "snippet": "sneakers",+ "value": "sneakers"+ },+ {+ "matched_tokens": [],+ "snippet": "shoes",+ "value": "shoes"+ }+ ],+ "street": {+ "matched_tokens": [+ "Drive"],- "street":"175 Commerce <mark>Drive</mark>"+ "snippet": "175 Commerce <mark>Drive</mark>",+ "value": "175 Commerce <mark>Drive</mark>"}}- ]+ }+ ]})"_json;- ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["full"].dump());- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());// if highlight fields not provided, only matching sub-fields should appear in highlight@@ -728,14 +935,35 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();highlight_doc = R"({- "company":{- "names": ["<mark>Space</mark> Corp. LLC", "Drive One Inc."]+ "company":{+ "names":[+ {+ "matched_tokens":[+ "Space"+ ],+ "snippet":"<mark>Space</mark> Corp. LLC"+ },+ {+ "matched_tokens":[],+ "snippet":"Drive One Inc."+ }+ ]+ },+ "company_names":[+ {+ "matched_tokens":[+ "Space"+ ],+ "snippet":"<mark>Space</mark> Corp. LLC"},- "company_names": ["<mark>Space</mark> Corp. LLC", "Drive One Inc."]+ {+ "matched_tokens":[],+ "snippet":"Drive One Inc."+ }+ ]})"_json;- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());- ASSERT_EQ(0, results["hits"][0]["highlight"]["full"].size());+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());// only a single highlight full field provided@@ -743,30 +971,38 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "company.names").get();- highlight_full_doc = R"({- "company":{- "names":[- "<mark>Space</mark> Corp. LLC",- "Drive One Inc."- ]- }- })"_json;-highlight_doc = R"({- "company":{- "names":[- "<mark>Space</mark> Corp. LLC",- "Drive One Inc."+ "company": {+ "names": [+ {+ "matched_tokens": [+ "Space"+ ],+ "snippet": "<mark>Space</mark> Corp. LLC",+ "value": "<mark>Space</mark> Corp. LLC"+ },+ {+ "matched_tokens": [],+ "snippet": "Drive One Inc.",+ "value": "Drive One Inc."+ }]},- "company_names":[- "<mark>Space</mark> Corp. LLC",- "Drive One Inc."+ "company_names": [+ {+ "matched_tokens": [+ "Space"+ ],+ "snippet": "<mark>Space</mark> Corp. LLC"+ },+ {+ "matched_tokens": [],+ "snippet": "Drive One Inc."+ }]})"_json;- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());- ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["full"].dump());+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());// try to highlight `id` fieldresults = coll1->search("shoes", {"locations.address.products"}, "", {}, sort_fields, {0}, 10, 1,@@ -775,92 +1011,7 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false, true,"id").get();- ASSERT_TRUE(results["hits"][0]["highlight"]["snippet"].empty());- ASSERT_TRUE(results["hits"][0]["highlight"]["full"].empty());-}--TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {- std::vector<field> fields = {field(".*", field_types::AUTO, false, true)};-- auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO, {}, {}, true);- ASSERT_TRUE(op.ok());- Collection* coll1 = op.get();-- auto doc = R"({- "company_names": ["Quick brown fox jumped.", "The red fox was not fast."],- "details": {- "description": "Quick set, go.",- "names": ["Quick brown fox jumped.", "The red fox was not fast."]- },- "locations": [- {- "address": { "street": "Brown Shade Avenue" }- },- {- "address": { "street": "Graywolf Lane" }- }- ]- })"_json;-- auto add_op = coll1->add(doc.dump(), CREATE);- ASSERT_TRUE(add_op.ok());-- // search both simply nested and deeply nested array-of-objects- auto results = coll1->search("brown fox", {"company_names", "details", "locations"},- "", {}, sort_fields, {0}, 10, 1,- token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address").get();-- ASSERT_EQ(3, results["hits"][0]["highlight"]["meta"].size());- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"].size());-- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].size());- std::vector<std::string> matched_tokens = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].get<std::vector<std::string>>();- std::sort(matched_tokens.begin(), matched_tokens.end());- ASSERT_EQ("brown", matched_tokens[0]);- ASSERT_EQ("fox", matched_tokens[1]);-- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].size());- std::vector<size_t> matched_indices = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].get<std::vector<size_t>>();- ASSERT_EQ(0, matched_indices[0]);- ASSERT_EQ(1, matched_indices[1]);-- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].size());- matched_tokens = results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].get<std::vector<std::string>>();- std::sort(matched_tokens.begin(), matched_tokens.end());- ASSERT_EQ("brown", matched_tokens[0]);- ASSERT_EQ("fox", matched_tokens[1]);-- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["locations"][0]["address"]["street"]["matched_tokens"].size());- matched_tokens = results["hits"][0]["highlight"]["meta"]["locations"][0]["address"]["street"]["matched_tokens"].get<std::vector<std::string>>();- std::sort(matched_tokens.begin(), matched_tokens.end());- ASSERT_EQ("Brown", matched_tokens[0]);-- // partial matched index match-- results = coll1->search("fast", {"company_names"},- "", {}, sort_fields, {0}, 10, 1,- token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address").get();-- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"].size());- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"].size());-- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].size());- matched_indices = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].get<std::vector<size_t>>();- ASSERT_EQ(1, matched_indices[0]);-- // when no highlighting is enabled by setting unknown field for highlighting- results = coll1->search("brown fox", {"company_names", "details", "locations"}, "", {}, sort_fields, {0}, 10, 1,- token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "x",- 20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false, true,- "x").get();-- ASSERT_EQ(3, results["hits"][0]["highlight"].size());- ASSERT_EQ(0, results["hits"][0]["highlight"]["snippet"].size());- ASSERT_EQ(0, results["hits"][0]["highlight"]["full"].size());- ASSERT_EQ(0, results["hits"][0]["highlight"]["meta"].size());+ ASSERT_TRUE(results["hits"][0]["highlight"].empty());}TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {@@ -911,28 +1062,51 @@ TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();auto snippet_doc = R"({- "details":{- "names":[- "Quick <mark>brown</mark> <mark>fox</mark> jumped.",- "The red <mark>fox</mark> was not fast."- ]+ "details": {+ "description": {+ "matched_tokens": [],+ "snippet": "Quick set, go."+ },+ "names": [+ {+ "matched_tokens": [+ "brown",+ "fox"+ ],+ "snippet": "Quick <mark>brown</mark> <mark>fox</mark> jumped."},- "locations":[- {- "address":{- "street":"<mark>Brown</mark> Shade Avenue"- }- },- {- "address":{- "street":"Graywolf Lane"- }+ {+ "matched_tokens": [+ "fox"+ ],+ "snippet": "The red <mark>fox</mark> was not fast."+ }+ ]+ },+ "locations": [+ {+ "address": {+ "street": {+ "matched_tokens": [+ "Brown"+ ],+ "snippet": "<mark>Brown</mark> Shade Avenue"}- ]+ }+ },+ {+ "address": {+ "street": {+ "matched_tokens": [],+ "snippet": "Graywolf Lane"+ }+ }+ }+ ]})"_json;ASSERT_EQ(1, results["hits"].size());- ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());+ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"].dump());results = coll1->search("fix", {"company.name"},"", {}, sort_fields, {0}, 10, 1,@@ -967,21 +1141,29 @@ TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {ASSERT_EQ(1, results["hits"].size());snippet_doc = R"({- "locations":[+ "locations": [{- "address":{- "street":"<mark>Brown</mark> Shade Avenue"+ "address": {+ "street": {+ "matched_tokens": [+ "Brown"+ ],+ "snippet": "<mark>Brown</mark> Shade Avenue"+ }}},{- "address":{- "street":"Graywolf Lane"+ "address": {+ "street": {+ "matched_tokens": [],+ "snippet": "Graywolf Lane"+ }}}]})"_json;- ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());+ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"].dump());// explicit partial array object field in the schemaschema = R"({@@ -1009,21 +1191,29 @@ TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {ASSERT_EQ(1, results["hits"].size());snippet_doc = R"({- "locations":[+ "locations": [{- "address":{- "street":"<mark>Brown</mark> Shade Avenue"+ "address": {+ "street": {+ "matched_tokens": [+ "Brown"+ ],+ "snippet": "<mark>Brown</mark> Shade Avenue"+ }}},{- "address":{- "street":"Graywolf Lane"+ "address": {+ "street": {+ "matched_tokens": [],+ "snippet": "Graywolf Lane"+ }}}]})"_json;- ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());+ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"].dump());// non-optional object field validation (details)auto doc2 = R"({@@ -1690,25 +1880,43 @@ TEST_F(CollectionNestedFieldsTest, HighlightArrayInsideArrayOfObj) {ASSERT_EQ(1, results["hits"].size());nlohmann::json highlight_meta_doc = R"({- "studies":[+ "studies": [{- "tags":[- "foo",- "bar"+ "name": {+ "matched_tokens": [],+ "snippet": "College 1"+ },+ "tags": [+ {+ "matched_tokens": [],+ "snippet": "foo"+ },+ {+ "matched_tokens": [],+ "snippet": "bar"+ }]},{- "tags":{- "matched_indices":[- 1- ],- "matched_tokens":[- "beta"- ]- }+ "name": {+ "matched_tokens": [],+ "snippet": "College 1"+ },+ "tags": [+ {+ "matched_tokens": [],+ "snippet": "alpha"+ },+ {+ "matched_tokens": [+ "beta"+ ],+ "snippet": "<mark>beta</mark>"+ }+ ]}]})"_json;- ASSERT_EQ(highlight_meta_doc.dump(), results["hits"][0]["highlight"]["meta"].dump());+ ASSERT_EQ(highlight_meta_doc.dump(), results["hits"][0]["highlight"].dump());}diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 8fd8cd24..f80b73f6 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -913,26 +913,25 @@ TEST_F(CollectionSpecificMoreTest, HighlightWithAccentedChars) {ASSERT_EQ("<mark>Rāp</mark>eti Early Learning Centre", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());auto highlight_doc = R"({- "companies":[+ "companies": [{- "title":"<mark>Rāp</mark>eti Early Learning Centre"+ "title": {+ "matched_tokens": [+ "Rāp"+ ],+ "snippet": "<mark>Rāp</mark>eti Early Learning Centre"+ }}],- "title":"<mark>Rāp</mark>eti Early Learning Centre"+ "title": {+ "matched_tokens": [+ "Rāp"+ ],+ "snippet": "<mark>Rāp</mark>eti Early Learning Centre"+ }})"_json;- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());-- ASSERT_EQ(0, results["hits"][0]["highlight"]["full"].size());- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"].size());-- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["title"].size());- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["title"]["matched_tokens"].size());- ASSERT_EQ("Rāp", results["hits"][0]["highlight"]["meta"]["title"]["matched_tokens"][0]);-- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies"][0]["title"].size());- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies"][0]["title"]["matched_tokens"].size());- ASSERT_EQ("Rāp", results["hits"][0]["highlight"]["meta"]["companies"][0]["title"]["matched_tokens"][0]);+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());}TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {@@ -1272,9 +1271,9 @@ TEST_F(CollectionSpecificMoreTest, HandleStringFieldWithObjectValueEarlier) {TEST_F(CollectionSpecificMoreTest, CopyDocHelper) {std::vector<highlight_field_t> hightlight_items = {- highlight_field_t("foo.bar", false, false),- highlight_field_t("baz", false, false),- highlight_field_t("not-found", false, false),+ highlight_field_t("foo.bar", false, false, true),+ highlight_field_t("baz", false, false, true),+ highlight_field_t("not-found", false, false, true),};nlohmann::json src = R"({@@ -1641,8 +1640,8 @@ TEST_F(CollectionSpecificMoreTest, HighlightOnFieldNameWithDot) {ASSERT_EQ(1, res["hits"][0]["highlights"].size());ASSERT_EQ("<mark>Infinity</mark> Inc.", res["hits"][0]["highlights"][0]["snippet"].get<std::string>());- nlohmann::json snippet = R"({"org.title":"<mark>Infinity</mark> Inc."})"_json;- ASSERT_EQ(snippet.dump(), res["hits"][0]["highlight"]["snippet"].dump());+ nlohmann::json highlight = R"({"org.title":{"matched_tokens":["Infinity"],"snippet":"<mark>Infinity</mark> Inc."}})"_json;+ ASSERT_EQ(highlight.dump(), res["hits"][0]["highlight"].dump());// even if nested fields enabled, plain field names with dots should work fine@@ -1660,6 +1659,7 @@ TEST_F(CollectionSpecificMoreTest, HighlightOnFieldNameWithDot) {res = coll2->search("infinity", {"org.title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();ASSERT_EQ(1, res["hits"].size());ASSERT_EQ(0, res["hits"][0]["highlights"].size());- snippet = R"({"org.title":"<mark>Infinity</mark> Inc."})"_json;- ASSERT_EQ(snippet.dump(), res["hits"][0]["highlight"]["snippet"].dump());++ highlight = R"({"org.title":{"matched_tokens":["Infinity"],"snippet":"<mark>Infinity</mark> Inc."}})"_json;+ ASSERT_EQ(highlight.dump(), res["hits"][0]["highlight"].dump());}