Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Simplify nested highlighting format.

Kishore Nallan23bd0af779a3
diff --git a/include/collection.h b/include/collection.h
index 298dfd12..6eda783a 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -30,10 +30,11 @@ struct highlight_field_t {
std::string name;
bool fully_highlighted;
bool infix;
+ bool is_string;
tsl::htrie_map<char, token_leaf> qtoken_leaves;
- highlight_field_t(const std::string& name, bool fully_highlighted, bool infix):
- name(name), fully_highlighted(fully_highlighted), infix(infix) {
+ highlight_field_t(const std::string& name, bool fully_highlighted, bool infix, bool is_string):
+ name(name), fully_highlighted(fully_highlighted), infix(infix), is_string(is_string) {
}
};
@@ -134,8 +135,6 @@ private:
const tsl::htrie_map<char, token_leaf>& qtoken_leaves,
const KV* field_order_kv, const nlohmann::json &document,
nlohmann::json& highlight_doc,
- nlohmann::json& highlight_full_doc,
- nlohmann::json& highlight_meta,
StringUtils & string_utils,
const size_t snippet_threshold,
const size_t highlight_affix_num_tokens,
@@ -245,10 +244,8 @@ private:
template<class T>
static bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
- const nlohmann::json& fdoc, nlohmann::json& fobj,
- const nlohmann::json& mdoc, nlohmann::json& mobj,
std::vector<std::string>& path_parts, size_t path_index,
- int arr_index, nlohmann::json& parent_mobj, T func);
+ T func);
static Option<bool> resolve_field_type(field& new_field,
nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,
@@ -346,7 +343,7 @@ public:
const index_operation_t op, const DIRTY_VALUES& dirty_values);
static void prune_doc(nlohmann::json& doc, const tsl::htrie_set<char>& include_names,
- const tsl::htrie_set<char>& exclude_names, std::string parent_name = "", size_t depth = 0);
+ const tsl::htrie_set<char>& exclude_names, const std::string& parent_name = "", size_t depth = 0);
const Index* _get_index() const;
@@ -478,6 +475,7 @@ public:
size_t snippet_start_offset) ;
void process_highlight_fields(const std::vector<search_field_t>& search_fields,
+ const std::vector<std::string>& raw_search_fields,
const tsl::htrie_set<char>& exclude_fields,
const tsl::htrie_set<char>& include_fields,
const std::vector<std::string>& highlight_field_names,
@@ -501,18 +499,16 @@ public:
template<class T>
bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
- const nlohmann::json& fdoc, nlohmann::json& fobj,
- const nlohmann::json& mdoc, nlohmann::json& mobj,
- std::vector<std::string>& path_parts, size_t path_index,
- int arr_index, nlohmann::json& parent_mobj,
- T func) {
+ std::vector<std::string>& path_parts, size_t path_index,
+ T func) {
if(path_index == path_parts.size()) {
// end of path: guaranteed to be a string
if(!hobj.is_string()) {
- return false;
+ //return false;
}
- func(hobj, fobj, mobj, arr_index, parent_mobj);
+ func(hobj);
+ return true;
}
const std::string& fragment = path_parts[path_index];
@@ -521,24 +517,13 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::js
if(it != hobj.end()) {
if(it.value().is_array()) {
bool resolved = false;
- auto& parent_mobj = mobj.empty() ? mobj : mobj[fragment];
- nlohmann::json empty_obj;
-
for(size_t i = 0; i < it.value().size(); i++) {
auto& h_ele = it.value().at(i);
- auto& f_ele = fobj.empty() ? fobj : fobj[fragment][i];
- // if value is a primitive, we don't want the array object getting destroyed
- auto& m_ele = h_ele.is_object() && !mobj.empty() ? mobj[fragment][i] : empty_obj;
- resolved |= highlight_nested_field(hdoc, h_ele, fdoc, f_ele, mdoc, m_ele,
- path_parts, path_index + 1, i, parent_mobj, func);
+ resolved |= highlight_nested_field(hdoc, h_ele, path_parts, path_index + 1, func);
}
return resolved;
} else {
- auto& f_ele = fobj.empty() ? fobj : fobj[fragment];
- auto& m_ele = mobj.empty() ? mobj : mobj[fragment];
- auto& parent_mobj = mobj.empty() ? mobj : mobj[fragment];
- return highlight_nested_field(hdoc, it.value(), fdoc, f_ele, mdoc, m_ele, path_parts, path_index + 1,
- -1, parent_mobj, func);
+ return highlight_nested_field(hdoc, it.value(), path_parts, path_index + 1, func);
}
} {
return false;
diff --git a/src/collection.cpp b/src/collection.cpp
index dec6f33e..49df2df8 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1437,7 +1437,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
StringUtils::split(highlight_full_fields, highlight_full_field_names, ",");
if(query != "*") {
- process_highlight_fields(weighted_search_fields, include_fields_full, exclude_fields_full,
+ process_highlight_fields(weighted_search_fields, raw_search_fields, include_fields_full, exclude_fields_full,
highlight_field_names, highlight_full_field_names, infixes, q_tokens,
search_params->qtoken_set, highlight_items);
@@ -1490,23 +1490,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
nlohmann::json highlight_res = nlohmann::json::object();
if(!highlight_items.empty()) {
- highlight_res["meta"] = nlohmann::json::object();
- copy_highlight_doc(highlight_items, document, highlight_res["meta"]);
- remove_flat_fields(highlight_res["meta"]);
- highlight_res["meta"].erase("id");
-
- highlight_res["snippet"] = nlohmann::json::object();
- copy_highlight_doc(highlight_items, document, highlight_res["snippet"]);
- remove_flat_fields(highlight_res["snippet"]);
- highlight_res["snippet"].erase("id");
-
- if(has_atleast_one_fully_highlighted_field) {
- copy_highlight_doc(highlight_items, document, highlight_res["full"]);
- remove_flat_fields(highlight_res["full"]);
- highlight_res["full"].erase("id");
- } else {
- highlight_res["full"] = nlohmann::json::object();
- }
+ copy_highlight_doc(highlight_items, document, highlight_res);
+ remove_flat_fields(highlight_res);
+ highlight_res.erase("id");
}
nlohmann::json wrapper_doc;
@@ -1526,9 +1512,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
field search_field = search_schema.at(field_name);
- if(query != "*" && (search_field.type == field_types::STRING ||
- search_field.type == field_types::STRING_ARRAY)) {
-
+ if(query != "*") {
highlight_t highlight;
highlight.field = search_field.name;
@@ -1536,7 +1520,7 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
bool found_full_highlight = false;
highlight_result(raw_query, search_field, i, highlight_item.qtoken_leaves, field_order_kv,
- document, highlight_res["snippet"], highlight_res["full"], highlight_res["meta"],
+ document, highlight_res,
string_utils, snippet_threshold,
highlight_affix_num_tokens, highlight_item.fully_highlighted, highlight_item.infix,
highlight_start_tag, highlight_end_tag, index_symbols, highlight,
@@ -1562,22 +1546,25 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
}
}
- for(auto& hfield_name: highlight_field_names) {
- auto it = hfield_names.equal_prefix_range(hfield_name);
- if(it.first != it.second) {
- hfield_names.insert(hfield_name);
+ if(highlight_field_names.empty()) {
+ for(auto& raw_search_field: raw_search_fields) {
+ auto it = hfield_names.equal_prefix_range(raw_search_field);
+ if(it.first != it.second) {
+ hfield_names.insert(raw_search_field);
+ }
+ }
+ } else {
+ for(auto& hfield_name: highlight_field_names) {
+ auto it = hfield_names.equal_prefix_range(hfield_name);
+ if(it.first != it.second) {
+ hfield_names.insert(hfield_name);
+ }
}
}
// remove fields from highlight doc that were not highlighted
if(!hfield_names.empty()) {
- prune_doc(highlight_res["meta"], hfield_names, tsl::htrie_set<char>(), "");
- prune_doc(highlight_res["snippet"], hfield_names, tsl::htrie_set<char>(), "");
- prune_doc(highlight_res["full"], h_full_field_names, tsl::htrie_set<char>(), "");
- } else {
- highlight_res["meta"] = nlohmann::json::object();
- highlight_res["snippet"] = nlohmann::json::object();
- highlight_res["full"] = nlohmann::json::object();
+ prune_doc(highlight_res, hfield_names, tsl::htrie_set<char>(), "");
}
std::sort(highlights.begin(), highlights.end());
@@ -1951,6 +1938,7 @@ void Collection::populate_text_match_info(nlohmann::json& info, uint64_t match_s
}
void Collection::process_highlight_fields(const std::vector<search_field_t>& search_fields,
+ const std::vector<std::string>& raw_search_fields,
const tsl::htrie_set<char>& include_fields,
const tsl::htrie_set<char>& exclude_fields,
const std::vector<std::string>& highlight_field_names,
@@ -1985,8 +1973,14 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& sea
}
if(highlight_field_names.empty()) {
- for(size_t i = 0; i < search_fields.size(); i++) {
- const auto& field_name = search_fields[i].name;
+ std::vector<std::string> highlight_field_names_expanded;
+
+ for(size_t i = 0; i < raw_search_fields.size(); i++) {
+ extract_field_name(raw_search_fields[i], search_schema, highlight_field_names_expanded, false, enable_nested_fields);
+ }
+
+ for(size_t i = 0; i < highlight_field_names_expanded.size(); i++) {
+ const auto& field_name = highlight_field_names_expanded[i];
if(exclude_fields.count(field_name) != 0) {
// should not pick excluded field for highlighting (only for implicit highlighting)
continue;
@@ -1999,23 +1993,28 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& sea
bool fully_highlighted = (fields_highlighted_fully_set.count(field_name) != 0);
bool infixed = (fields_infixed_set.count(field_name) != 0);
- highlight_items.emplace_back(field_name, fully_highlighted, infixed);
+ auto schema_it = search_schema.find(field_name);
+ bool is_string = (schema_it != search_schema.end()) && schema_it->is_string();
+
+ highlight_items.emplace_back(field_name, fully_highlighted, infixed, is_string);
}
} else {
std::vector<std::string> highlight_field_names_expanded;
for(size_t i = 0; i < highlight_field_names.size(); i++) {
- extract_field_name(highlight_field_names[i], search_schema, highlight_field_names_expanded, true, enable_nested_fields);
+ extract_field_name(highlight_field_names[i], search_schema, highlight_field_names_expanded, false, enable_nested_fields);
}
for(size_t i = 0; i < highlight_field_names_expanded.size(); i++) {
const auto& highlight_field_name = highlight_field_names_expanded[i];
- if(search_schema.count(highlight_field_name) == 0) {
+ auto schema_it = search_schema.find(highlight_field_name);
+ if(schema_it == search_schema.end()) {
// ignore fields not part of schema
continue;
}
bool fully_highlighted = (fields_highlighted_fully_set.count(highlight_field_name) != 0);
bool infixed = (fields_infixed_set.count(highlight_field_name) != 0);
- highlight_items.emplace_back(highlight_field_name, fully_highlighted, infixed);
+ bool is_string = schema_it->is_string();
+ highlight_items.emplace_back(highlight_field_name, fully_highlighted, infixed, is_string);
}
}
@@ -2024,6 +2023,10 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& sea
it.key(qtoken);
for(auto& highlight_item: highlight_items) {
+ if(!highlight_item.is_string) {
+ continue;
+ }
+
const auto& field_name = highlight_item.name;
art_leaf* leaf = index->get_token_leaf(field_name, (const unsigned char*) qtoken.c_str(), qtoken.size()+1);
if(leaf) {
@@ -2039,6 +2042,9 @@ void Collection::process_highlight_fields(const std::vector<search_field_t>& sea
for(auto& q_token: q_tokens) {
if(qtoken_set.find(q_token) == qtoken_set.end()) {
for(auto& highlight_item: highlight_items) {
+ if(!highlight_item.is_string) {
+ continue;
+ }
const auto& field_name = highlight_item.name;
art_leaf* leaf = index->get_token_leaf(field_name, (const unsigned char*) q_token.c_str(), q_token.size()+1);
if(leaf) {
@@ -2324,8 +2330,6 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
const tsl::htrie_map<char, token_leaf>& qtoken_leaves,
const KV* field_order_kv, const nlohmann::json & document,
nlohmann::json& highlight_doc,
- nlohmann::json& highlight_full_doc,
- nlohmann::json& highlight_meta,
StringUtils & string_utils,
const size_t snippet_threshold,
const size_t highlight_affix_num_tokens,
@@ -2358,12 +2362,6 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
size_t prefix_token_num_chars = StringUtils::get_num_chars(last_raw_q_token);
std::set<std::string> last_full_q_tokens;
-
- if(qtoken_leaves.empty() && !is_infix_search) {
- // none of the tokens from the query were found on this field
- return ;
- }
-
std::vector<match_index_t> match_indices;
if(is_infix_search) {
@@ -2396,77 +2394,93 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
LOG(INFO) << "Token: " << qtok_buff << ", root_len: " << it.value().root_len;
}*/
- /* We have to handle the following differently:
+ bool flat_field = highlight_doc.contains(search_field.name);
- a) top-level field or a simple nested field
- b) field inside an array of objects
+ std::vector<std::string> path_parts;
+ if(enable_nested_fields && !flat_field) {
+ StringUtils::split(search_field.name, path_parts, ".");
+ } else {
+ path_parts = {search_field.name};
+ }
- For a) we can rely on matching index positions since field value on-disk will be in the same order
- as the field value indexed. For b) we need to do a keyword based highlight since we cannot use
- indexed offsets.
- */
+ highlight_nested_field(highlight_doc, highlight_doc, path_parts, 0, [&](nlohmann::json& h_obj) {
+ Match match;
+ match_index_t match_index(match, 0, 0);
+ int last_valid_offset_index = -1;
+ size_t last_valid_offset = 0;
- int nested_array = search_field.nested_array;
- std::vector<std::string> path_parts;
+ highlight_t array_highlight = highlight;
- if(nested_array == field::VAL_UNKNOWN) {
- StringUtils::split(search_field.name, path_parts, ".");
- nested_array = Collection::is_nested_array(document, path_parts, 0);
- }
-
- if(nested_array) {
- if(path_parts.empty()) {
- StringUtils::split(search_field.name, path_parts, ".");
- }
-
- highlight_nested_field(highlight_doc, highlight_doc, highlight_full_doc, highlight_full_doc,
- highlight_meta, highlight_meta, path_parts, 0, -1, highlight_meta,
- [&](nlohmann::json& h_obj, nlohmann::json& f_obj, nlohmann::json& m_obj,
- int arr_index, nlohmann::json& parent_mobj) {
- Match match;
- match_index_t match_index(match, 0, 0);
- int last_valid_offset_index = -1;
- size_t last_valid_offset = 0;
-
- highlight_t array_highlight = highlight;
-
- std::string text = h_obj.get<std::string>();
- handle_highlight_text(text, normalise, search_field, symbols_to_index,
- token_separators, array_highlight, string_utils, use_word_tokenizer,
- highlight_affix_num_tokens,
- qtoken_leaves, last_valid_offset_index, match,
- prefix_token_num_chars,
- highlight_fully, snippet_threshold, is_infix_search,
- raw_query_tokens,
- last_valid_offset, highlight_start_tag, highlight_end_tag,
- index_symbols, match_index);
-
- if(!array_highlight.snippets.empty()) {
- h_obj = array_highlight.snippets[0];
- found_highlight = found_highlight || true;
-
- nlohmann::json& ref_meta_obj = (arr_index >= 0) ? parent_mobj : m_obj;
- if(!ref_meta_obj.is_object() || !ref_meta_obj.contains("matched_tokens")) {
- ref_meta_obj = nlohmann::json::object();
- }
+ if(!h_obj.is_string()) {
+ auto val_back = h_obj;
+ h_obj = nlohmann::json::object();
+ h_obj["snippet"] = to_string(val_back);
+ h_obj["matched_tokens"] = nlohmann::json::array();
+ if(highlight_fully) {
+ h_obj["value"] = val_back;
+ }
+ return ;
+ }
- for(auto& token_vec: array_highlight.matched_tokens) {
- for(auto& token: token_vec) {
- ref_meta_obj["matched_tokens"].push_back(token);
- }
- }
+ std::string text = h_obj.get<std::string>();
+ h_obj = nlohmann::json::object();
- if(arr_index >= 0) {
- ref_meta_obj["matched_indices"].push_back(arr_index);
- }
+ if(qtoken_leaves.empty()) {
+ h_obj["snippet"] = text;
+ h_obj["matched_tokens"] = nlohmann::json::array();
+ if(highlight_fully) {
+ h_obj["value"] = text;
}
+ return ;
+ }
- if(highlight_fully && !array_highlight.values.empty()) {
- f_obj = array_highlight.values[0];;
- found_full_highlight = found_full_highlight || true;
+ handle_highlight_text(text, normalise, search_field, symbols_to_index,
+ token_separators, array_highlight, string_utils, use_word_tokenizer,
+ highlight_affix_num_tokens,
+ qtoken_leaves, last_valid_offset_index, match,
+ prefix_token_num_chars,
+ highlight_fully, snippet_threshold, is_infix_search,
+ raw_query_tokens,
+ last_valid_offset, highlight_start_tag, highlight_end_tag,
+ index_symbols, match_index);
+
+
+ if(array_highlight.snippets.empty() && array_highlight.values.empty()) {
+ h_obj["snippet"] = text;
+ h_obj["matched_tokens"] = nlohmann::json::array();
+ }
+
+ if(!array_highlight.snippets.empty()) {
+ found_highlight = found_highlight || true;
+
+ h_obj["snippet"] = array_highlight.snippets[0];
+ h_obj["matched_tokens"] = nlohmann::json::array();
+
+ for(auto& token_vec: array_highlight.matched_tokens) {
+ for(auto& token: token_vec) {
+ h_obj["matched_tokens"].push_back(token);
+ }
}
- });
+ }
+ if(!array_highlight.values.empty()) {
+ h_obj["value"] = array_highlight.values[0];;
+ found_full_highlight = found_full_highlight || true;
+ } else if(highlight_fully) {
+ h_obj["value"] = text;
+ }
+ });
+
+ if(!flat_field) {
+ return;
+ }
+
+ if(qtoken_leaves.empty()) {
+ // none of the tokens from the query were found on this field
+ return ;
+ }
+
+ if(!search_field.is_string()) {
return ;
}
@@ -2571,86 +2585,6 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
if(!match_indices.empty()) {
highlight.match_score = match_indices[0].match_score;
}
-
- // in-place highlighting under the new highlight structure
- std::vector<std::string> parts;
- if(!enable_nested_fields || highlight_doc.contains(search_field.name)) {
- parts = {search_field.name};
- } else {
- StringUtils::split(search_field.name, parts, ".");
- }
-
- nlohmann::json* mval = highlight_meta.contains(parts[0]) ? &highlight_meta[parts[0]] : nullptr;
- nlohmann::json* hval = highlight_doc.contains(parts[0]) ? &highlight_doc[parts[0]] : nullptr;
- nlohmann::json* fval = highlight_full_doc.contains(parts[0]) ? &highlight_full_doc[parts[0]] : nullptr;
-
- for(size_t i = 1; mval != nullptr && i < parts.size(); i++) {
- const auto& part = parts[i];
- if(mval->contains(part)) {
- mval = &mval->at(part);
- } else {
- mval = nullptr;
- }
- }
-
- for(size_t i = 1; hval != nullptr && i < parts.size(); i++) {
- const auto& part = parts[i];
- if(hval->contains(part)) {
- hval = &hval->at(part);
- } else {
- hval = nullptr;
- }
- }
-
- for(size_t i = 1; fval != nullptr && i < parts.size(); i++) {
- const auto& part = parts[i];
- if(fval->contains(part)) {
- fval = &fval->at(part);
- } else {
- fval = nullptr;
- }
- }
-
- if(hval) {
- if(highlight.indices.empty()) {
- *hval = highlight.snippets[0];
- } else {
- if(hval->is_array()) {
- for(size_t hi = 0; hi < highlight.indices.size(); hi++) {
- hval->at(highlight.indices[hi]) = highlight.snippets[hi];
- }
- }
- }
-
- if(mval) {
- *mval = nlohmann::json::object();
- mval->emplace("matched_tokens", nlohmann::json::array());
- for(auto it = matched_tokens.begin(); it != matched_tokens.end(); ++it) {
- mval->at("matched_tokens").push_back(it.key());
- }
-
- if(hval->is_array()) {
- mval->emplace("matched_indices", nlohmann::json::array());
- for(auto highlight_index: highlight.indices) {
- mval->at("matched_indices").push_back(highlight_index);
- }
- }
- }
- }
-
- if(fval) {
- if(!highlight.values.empty()) {
- if(highlight.indices.empty()) {
- *fval = highlight.values[0];
- } else {
- if(fval->is_array()) {
- for(size_t hi = 0; hi < highlight.values.size(); hi++) {
- fval->at(highlight.indices[hi]) = highlight.values[hi];
- }
- }
- }
- }
- }
}
bool Collection::handle_highlight_text(std::string& text, bool normalise, const field &search_field,
@@ -3483,7 +3417,7 @@ void Collection::remove_flat_fields(nlohmann::json& document) {
void Collection::prune_doc(nlohmann::json& doc,
const tsl::htrie_set<char>& include_names,
const tsl::htrie_set<char>& exclude_names,
- std::string parent_name, size_t depth) {
+ const std::string& parent_name, size_t depth) {
// doc can only be an object
auto it = doc.begin();
while(it != doc.end()) {
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index ad30313f..dfecfb54 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -349,21 +349,85 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {
ASSERT_EQ(doc, results["hits"][0]["document"]);
auto highlight_doc = R"({
- "employees":{
- "tags":[
- "senior plumber",
- "<mark>electrician</mark>"
+ "employees": {
+ "num": {
+ "matched_tokens": [],
+ "snippet": "1200"
+ },
+ "tags": [
+ {
+ "matched_tokens": [],
+ "snippet": "senior plumber"
+ },
+ {
+ "matched_tokens": [
+ "electrician"
+ ],
+ "snippet": "<mark>electrician</mark>"
+ }
]
},
- "locations":[
+ "locations": [
{
- "address":{
- "street":"One Bowerman Drive"
+ "address": {
+ "city": {
+ "matched_tokens": [],
+ "snippet": "Beaverton"
+ },
+ "products": [
+ {
+ "matched_tokens": [],
+ "snippet": "shoes"
+ },
+ {
+ "matched_tokens": [],
+ "snippet": "tshirts"
+ }
+ ],
+ "street": {
+ "matched_tokens": [],
+ "snippet": "One Bowerman Drive"
+ }
+ },
+ "country": {
+ "matched_tokens": [],
+ "snippet": "USA"
+ },
+ "pincode": {
+ "matched_tokens": [],
+ "snippet": "100"
}
},
{
- "address":{
- "street":"175 <mark>Commerce</mark> Valley"
+ "address": {
+ "city": {
+ "matched_tokens": [],
+ "snippet": "Thornhill"
+ },
+ "products": [
+ {
+ "matched_tokens": [],
+ "snippet": "sneakers"
+ },
+ {
+ "matched_tokens": [],
+ "snippet": "shoes"
+ }
+ ],
+ "street": {
+ "matched_tokens": [
+ "Commerce"
+ ],
+ "snippet": "175 <mark>Commerce</mark> Valley"
+ }
+ },
+ "country": {
+ "matched_tokens": [],
+ "snippet": "Canada"
+ },
+ "pincode": {
+ "matched_tokens": [],
+ "snippet": "200"
}
}
]
@@ -380,7 +444,7 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {
ASSERT_EQ(0, raw_doc.count("employees.tags"));
ASSERT_EQ(4, raw_doc.size());
- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
// after update also the flat fields or meta should not be present on disk
@@ -404,18 +468,26 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {
"locations":[
{
"address":{
- "street":"<mark>One</mark> Bowerman Drive"
+ "street":{
+ "matched_tokens":[
+ "One"
+ ],
+ "snippet":"<mark>One</mark> Bowerman Drive"
+ }
}
},
{
"address":{
- "street":"175 Commerce Valley"
+ "street":{
+ "matched_tokens":[],
+ "snippet":"175 Commerce Valley"
+ }
}
}
]
})"_json;
- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
// try to search nested fields that don't exist
@@ -510,7 +582,6 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {
auto add_op = coll1->add(doc.dump(), CREATE);
ASSERT_TRUE(add_op.ok());
- // search both simply nested and deeply nested array-of-objects
auto results = coll1->search("One", {"locations.address"}, "", {}, sort_fields, {0}, 10, 1,
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address").get();
@@ -521,44 +592,76 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {
"locations":[
{
"address":{
- "street":"<mark>One</mark> Bowerman Drive"
+ "city":{
+ "matched_tokens":[
+
+ ],
+ "snippet":"Beaverton",
+ "value":"Beaverton"
+ },
+ "products":[
+ {
+ "matched_tokens":[
+
+ ],
+ "snippet":"shoes",
+ "value":"shoes"
+ },
+ {
+ "matched_tokens":[
+
+ ],
+ "snippet":"tshirts",
+ "value":"tshirts"
+ }
+ ],
+ "street":{
+ "matched_tokens":[
+ "One"
+ ],
+ "snippet":"<mark>One</mark> Bowerman Drive",
+ "value":"<mark>One</mark> Bowerman Drive"
+ }
}
},
{
"address":{
- "street":"175 Commerce Drive"
- }
- }
- ]
- })"_json;
+ "city":{
+ "matched_tokens":[
- auto highlight_full_doc = R"({
- "locations":[
- {
- "address":{
- "city":"Beaverton",
- "products":[
- "shoes",
- "tshirts"
],
- "street":"<mark>One</mark> Bowerman Drive"
- }
- },
- {
- "address":{
- "city":"Thornhill",
- "products":[
- "sneakers",
- "shoes"
+ "snippet":"Thornhill",
+ "value":"Thornhill"
+ },
+ "products":[
+ {
+ "matched_tokens":[
+
+ ],
+ "snippet":"sneakers",
+ "value":"sneakers"
+ },
+ {
+ "matched_tokens":[
+
+ ],
+ "snippet":"shoes",
+ "value":"shoes"
+ }
+ ],
+ "street":{
+ "matched_tokens":[
+
],
- "street":"175 Commerce Drive"
+ "snippet":"175 Commerce Drive",
+ "value":"175 Commerce Drive"
}
}
- ]
+ }
+ ]
})"_json;
- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
- ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["full"].dump());
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
// repeating token
@@ -573,18 +676,76 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {
"locations":[
{
"address":{
- "street":"One Bowerman <mark>Drive</mark>"
+ "city":{
+ "matched_tokens":[
+
+ ],
+ "snippet":"Beaverton",
+ "value":"Beaverton"
+ },
+ "products":[
+ {
+ "matched_tokens":[
+
+ ],
+ "snippet":"shoes",
+ "value":"shoes"
+ },
+ {
+ "matched_tokens":[
+
+ ],
+ "snippet":"tshirts",
+ "value":"tshirts"
+ }
+ ],
+ "street":{
+ "matched_tokens":[
+ "Drive"
+ ],
+ "snippet":"One Bowerman <mark>Drive</mark>",
+ "value":"One Bowerman <mark>Drive</mark>"
+ }
}
},
{
"address":{
- "street":"175 Commerce <mark>Drive</mark>"
+ "city":{
+ "matched_tokens":[
+
+ ],
+ "snippet":"Thornhill",
+ "value":"Thornhill"
+ },
+ "products":[
+ {
+ "matched_tokens":[
+
+ ],
+ "snippet":"sneakers",
+ "value":"sneakers"
+ },
+ {
+ "matched_tokens":[
+
+ ],
+ "snippet":"shoes",
+ "value":"shoes"
+ }
+ ],
+ "street":{
+ "matched_tokens":[
+ "Drive"
+ ],
+ "snippet":"175 Commerce <mark>Drive</mark>",
+ "value":"175 Commerce <mark>Drive</mark>"
+ }
}
}
]
})"_json;
- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
ASSERT_EQ(0, results["hits"][0]["highlights"].size());
// nested array of array, highlighting parent of searched nested field
@@ -595,67 +756,81 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {
"locations.address").get();
ASSERT_EQ(1, results["hits"].size());
- highlight_full_doc = R"({
+
+ highlight_doc = R"({
"locations":[
{
"address":{
- "city":"Beaverton",
+ "city":{
+ "matched_tokens":[
+
+ ],
+ "snippet":"Beaverton",
+ "value":"Beaverton"
+ },
"products":[
- "<mark>shoes</mark>",
- "tshirts"
+ {
+ "matched_tokens":[
+ "shoes"
+ ],
+ "snippet":"<mark>shoes</mark>",
+ "value":"<mark>shoes</mark>"
+ },
+ {
+ "matched_tokens":[
+
+ ],
+ "snippet":"tshirts",
+ "value":"tshirts"
+ }
],
- "street":"One Bowerman Drive"
+ "street":{
+ "matched_tokens":[
+
+ ],
+ "snippet":"One Bowerman Drive",
+ "value":"One Bowerman Drive"
+ }
}
},
{
"address":{
- "city":"Thornhill",
+ "city":{
+ "matched_tokens":[
+
+ ],
+ "snippet":"Thornhill",
+ "value":"Thornhill"
+ },
"products":[
- "sneakers",
- "<mark>shoes</mark>"
+ {
+ "matched_tokens":[
+
+ ],
+ "snippet":"sneakers",
+ "value":"sneakers"
+ },
+ {
+ "matched_tokens":[
+ "shoes"
+ ],
+ "snippet":"<mark>shoes</mark>",
+ "value":"<mark>shoes</mark>"
+ }
],
- "street":"175 Commerce Drive"
+ "street":{
+ "matched_tokens":[
+
+ ],
+ "snippet":"175 Commerce Drive",
+ "value":"175 Commerce Drive"
+ }
}
}
]
})"_json;
- auto highlight_meta_doc = R"({
- "locations":[
- {
- "address":{
- "city":"Beaverton",
- "products":{
- "matched_indices":[
- 0
- ],
- "matched_tokens":[
- "shoes"
- ]
- },
- "street":"One Bowerman Drive"
- }
- },
- {
- "address":{
- "city":"Thornhill",
- "products":{
- "matched_indices":[
- 1
- ],
- "matched_tokens":[
- "shoes"
- ]
- },
- "street":"175 Commerce Drive"
- }
- }
- ]
- })"_json;
-
- ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["full"].dump());
- ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
- ASSERT_EQ(highlight_meta_doc.dump(), results["hits"][0]["highlight"]["meta"].dump());
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
// full highlighting only one of the 3 highlight fields
results = coll1->search("drive", {"company.names", "company_names", "locations.address"}, "", {}, sort_fields, {0}, 10, 1,
@@ -664,62 +839,94 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {
20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false, true,
"company.names,company_names,locations.address").get();
- highlight_full_doc = R"({
- "locations":[
+ highlight_doc = R"({
+ "company": {
+ "names": [
{
- "address":{
- "city":"Beaverton",
- "products":[
- "shoes",
- "tshirts"
- ],
- "street":"One Bowerman <mark>Drive</mark>"
- }
+ "matched_tokens": [],
+ "snippet": "Space Corp. LLC"
},
{
- "address":{
- "city":"Thornhill",
- "products":[
- "sneakers",
- "shoes"
- ],
- "street":"175 Commerce <mark>Drive</mark>"
- }
+ "matched_tokens": [
+ "Drive"
+ ],
+ "snippet": "<mark>Drive</mark> One Inc."
}
]
- })"_json;
-
- highlight_doc = R"({
- "company":{
- "names": ["Space Corp. LLC", "<mark>Drive</mark> One Inc."]
+ },
+ "company_names": [
+ {
+ "matched_tokens": [],
+ "snippet": "Space Corp. LLC"
},
- "company_names": ["Space Corp. LLC", "<mark>Drive</mark> One Inc."],
- "locations":[
- {
- "address":{
- "city":"Beaverton",
- "products":[
- "shoes",
- "tshirts"
+ {
+ "matched_tokens": [
+ "Drive"
+ ],
+ "snippet": "<mark>Drive</mark> One Inc."
+ }
+ ],
+ "locations": [
+ {
+ "address": {
+ "city": {
+ "matched_tokens": [],
+ "snippet": "Beaverton",
+ "value": "Beaverton"
+ },
+ "products": [
+ {
+ "matched_tokens": [],
+ "snippet": "shoes",
+ "value": "shoes"
+ },
+ {
+ "matched_tokens": [],
+ "snippet": "tshirts",
+ "value": "tshirts"
+ }
+ ],
+ "street": {
+ "matched_tokens": [
+ "Drive"
],
- "street":"One Bowerman <mark>Drive</mark>"
+ "snippet": "One Bowerman <mark>Drive</mark>",
+ "value": "One Bowerman <mark>Drive</mark>"
}
- },
- {
- "address":{
- "city":"Thornhill",
- "products":[
- "sneakers",
- "shoes"
+ }
+ },
+ {
+ "address": {
+ "city": {
+ "matched_tokens": [],
+ "snippet": "Thornhill",
+ "value": "Thornhill"
+ },
+ "products": [
+ {
+ "matched_tokens": [],
+ "snippet": "sneakers",
+ "value": "sneakers"
+ },
+ {
+ "matched_tokens": [],
+ "snippet": "shoes",
+ "value": "shoes"
+ }
+ ],
+ "street": {
+ "matched_tokens": [
+ "Drive"
],
- "street":"175 Commerce <mark>Drive</mark>"
+ "snippet": "175 Commerce <mark>Drive</mark>",
+ "value": "175 Commerce <mark>Drive</mark>"
}
}
- ]
+ }
+ ]
})"_json;
- ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["full"].dump());
- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
// if highlight fields not provided, only matching sub-fields should appear in highlight
@@ -728,14 +935,35 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();
highlight_doc = R"({
- "company":{
- "names": ["<mark>Space</mark> Corp. LLC", "Drive One Inc."]
+ "company":{
+ "names":[
+ {
+ "matched_tokens":[
+ "Space"
+ ],
+ "snippet":"<mark>Space</mark> Corp. LLC"
+ },
+ {
+ "matched_tokens":[],
+ "snippet":"Drive One Inc."
+ }
+ ]
+ },
+ "company_names":[
+ {
+ "matched_tokens":[
+ "Space"
+ ],
+ "snippet":"<mark>Space</mark> Corp. LLC"
},
- "company_names": ["<mark>Space</mark> Corp. LLC", "Drive One Inc."]
+ {
+ "matched_tokens":[],
+ "snippet":"Drive One Inc."
+ }
+ ]
})"_json;
- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
- ASSERT_EQ(0, results["hits"][0]["highlight"]["full"].size());
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
// only a single highlight full field provided
@@ -743,30 +971,38 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "company.names").get();
- highlight_full_doc = R"({
- "company":{
- "names":[
- "<mark>Space</mark> Corp. LLC",
- "Drive One Inc."
- ]
- }
- })"_json;
-
highlight_doc = R"({
- "company":{
- "names":[
- "<mark>Space</mark> Corp. LLC",
- "Drive One Inc."
+ "company": {
+ "names": [
+ {
+ "matched_tokens": [
+ "Space"
+ ],
+ "snippet": "<mark>Space</mark> Corp. LLC",
+ "value": "<mark>Space</mark> Corp. LLC"
+ },
+ {
+ "matched_tokens": [],
+ "snippet": "Drive One Inc.",
+ "value": "Drive One Inc."
+ }
]
},
- "company_names":[
- "<mark>Space</mark> Corp. LLC",
- "Drive One Inc."
+ "company_names": [
+ {
+ "matched_tokens": [
+ "Space"
+ ],
+ "snippet": "<mark>Space</mark> Corp. LLC"
+ },
+ {
+ "matched_tokens": [],
+ "snippet": "Drive One Inc."
+ }
]
})"_json;
- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
- ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["full"].dump());
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
// try to highlight `id` field
results = coll1->search("shoes", {"locations.address.products"}, "", {}, sort_fields, {0}, 10, 1,
@@ -775,92 +1011,7 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {
20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false, true,
"id").get();
- ASSERT_TRUE(results["hits"][0]["highlight"]["snippet"].empty());
- ASSERT_TRUE(results["hits"][0]["highlight"]["full"].empty());
-}
-
-TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {
- std::vector<field> fields = {field(".*", field_types::AUTO, false, true)};
-
- auto op = collectionManager.create_collection("coll1", 1, fields, "", 0, field_types::AUTO, {}, {}, true);
- ASSERT_TRUE(op.ok());
- Collection* coll1 = op.get();
-
- auto doc = R"({
- "company_names": ["Quick brown fox jumped.", "The red fox was not fast."],
- "details": {
- "description": "Quick set, go.",
- "names": ["Quick brown fox jumped.", "The red fox was not fast."]
- },
- "locations": [
- {
- "address": { "street": "Brown Shade Avenue" }
- },
- {
- "address": { "street": "Graywolf Lane" }
- }
- ]
- })"_json;
-
- auto add_op = coll1->add(doc.dump(), CREATE);
- ASSERT_TRUE(add_op.ok());
-
- // search both simply nested and deeply nested array-of-objects
- auto results = coll1->search("brown fox", {"company_names", "details", "locations"},
- "", {}, sort_fields, {0}, 10, 1,
- token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address").get();
-
- ASSERT_EQ(3, results["hits"][0]["highlight"]["meta"].size());
- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"].size());
-
- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].size());
- std::vector<std::string> matched_tokens = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].get<std::vector<std::string>>();
- std::sort(matched_tokens.begin(), matched_tokens.end());
- ASSERT_EQ("brown", matched_tokens[0]);
- ASSERT_EQ("fox", matched_tokens[1]);
-
- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].size());
- std::vector<size_t> matched_indices = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].get<std::vector<size_t>>();
- ASSERT_EQ(0, matched_indices[0]);
- ASSERT_EQ(1, matched_indices[1]);
-
- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].size());
- matched_tokens = results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].get<std::vector<std::string>>();
- std::sort(matched_tokens.begin(), matched_tokens.end());
- ASSERT_EQ("brown", matched_tokens[0]);
- ASSERT_EQ("fox", matched_tokens[1]);
-
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["locations"][0]["address"]["street"]["matched_tokens"].size());
- matched_tokens = results["hits"][0]["highlight"]["meta"]["locations"][0]["address"]["street"]["matched_tokens"].get<std::vector<std::string>>();
- std::sort(matched_tokens.begin(), matched_tokens.end());
- ASSERT_EQ("Brown", matched_tokens[0]);
-
- // partial matched index match
-
- results = coll1->search("fast", {"company_names"},
- "", {}, sort_fields, {0}, 10, 1,
- token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address").get();
-
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"].size());
- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"].size());
-
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].size());
- matched_indices = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].get<std::vector<size_t>>();
- ASSERT_EQ(1, matched_indices[0]);
-
- // when no highlighting is enabled by setting unknown field for highlighting
- results = coll1->search("brown fox", {"company_names", "details", "locations"}, "", {}, sort_fields, {0}, 10, 1,
- token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
- spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "x",
- 20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false, true,
- "x").get();
-
- ASSERT_EQ(3, results["hits"][0]["highlight"].size());
- ASSERT_EQ(0, results["hits"][0]["highlight"]["snippet"].size());
- ASSERT_EQ(0, results["hits"][0]["highlight"]["full"].size());
- ASSERT_EQ(0, results["hits"][0]["highlight"]["meta"].size());
+ ASSERT_TRUE(results["hits"][0]["highlight"].empty());
}
TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {
@@ -911,28 +1062,51 @@ TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {
spp::sparse_hash_set<std::string>(), 10, "", 30, 4).get();
auto snippet_doc = R"({
- "details":{
- "names":[
- "Quick <mark>brown</mark> <mark>fox</mark> jumped.",
- "The red <mark>fox</mark> was not fast."
- ]
+ "details": {
+ "description": {
+ "matched_tokens": [],
+ "snippet": "Quick set, go."
+ },
+ "names": [
+ {
+ "matched_tokens": [
+ "brown",
+ "fox"
+ ],
+ "snippet": "Quick <mark>brown</mark> <mark>fox</mark> jumped."
},
- "locations":[
- {
- "address":{
- "street":"<mark>Brown</mark> Shade Avenue"
- }
- },
- {
- "address":{
- "street":"Graywolf Lane"
- }
+ {
+ "matched_tokens": [
+ "fox"
+ ],
+ "snippet": "The red <mark>fox</mark> was not fast."
+ }
+ ]
+ },
+ "locations": [
+ {
+ "address": {
+ "street": {
+ "matched_tokens": [
+ "Brown"
+ ],
+ "snippet": "<mark>Brown</mark> Shade Avenue"
}
- ]
+ }
+ },
+ {
+ "address": {
+ "street": {
+ "matched_tokens": [],
+ "snippet": "Graywolf Lane"
+ }
+ }
+ }
+ ]
})"_json;
ASSERT_EQ(1, results["hits"].size());
- ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"].dump());
results = coll1->search("fix", {"company.name"},
"", {}, sort_fields, {0}, 10, 1,
@@ -967,21 +1141,29 @@ TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {
ASSERT_EQ(1, results["hits"].size());
snippet_doc = R"({
- "locations":[
+ "locations": [
{
- "address":{
- "street":"<mark>Brown</mark> Shade Avenue"
+ "address": {
+ "street": {
+ "matched_tokens": [
+ "Brown"
+ ],
+ "snippet": "<mark>Brown</mark> Shade Avenue"
+ }
}
},
{
- "address":{
- "street":"Graywolf Lane"
+ "address": {
+ "street": {
+ "matched_tokens": [],
+ "snippet": "Graywolf Lane"
+ }
}
}
]
})"_json;
- ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"].dump());
// explicit partial array object field in the schema
schema = R"({
@@ -1009,21 +1191,29 @@ TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {
ASSERT_EQ(1, results["hits"].size());
snippet_doc = R"({
- "locations":[
+ "locations": [
{
- "address":{
- "street":"<mark>Brown</mark> Shade Avenue"
+ "address": {
+ "street": {
+ "matched_tokens": [
+ "Brown"
+ ],
+ "snippet": "<mark>Brown</mark> Shade Avenue"
+ }
}
},
{
- "address":{
- "street":"Graywolf Lane"
+ "address": {
+ "street": {
+ "matched_tokens": [],
+ "snippet": "Graywolf Lane"
+ }
}
}
]
})"_json;
- ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
+ ASSERT_EQ(snippet_doc.dump(), results["hits"][0]["highlight"].dump());
// non-optional object field validation (details)
auto doc2 = R"({
@@ -1690,25 +1880,43 @@ TEST_F(CollectionNestedFieldsTest, HighlightArrayInsideArrayOfObj) {
ASSERT_EQ(1, results["hits"].size());
nlohmann::json highlight_meta_doc = R"({
- "studies":[
+ "studies": [
{
- "tags":[
- "foo",
- "bar"
+ "name": {
+ "matched_tokens": [],
+ "snippet": "College 1"
+ },
+ "tags": [
+ {
+ "matched_tokens": [],
+ "snippet": "foo"
+ },
+ {
+ "matched_tokens": [],
+ "snippet": "bar"
+ }
]
},
{
- "tags":{
- "matched_indices":[
- 1
- ],
- "matched_tokens":[
- "beta"
- ]
- }
+ "name": {
+ "matched_tokens": [],
+ "snippet": "College 1"
+ },
+ "tags": [
+ {
+ "matched_tokens": [],
+ "snippet": "alpha"
+ },
+ {
+ "matched_tokens": [
+ "beta"
+ ],
+ "snippet": "<mark>beta</mark>"
+ }
+ ]
}
]
})"_json;
- ASSERT_EQ(highlight_meta_doc.dump(), results["hits"][0]["highlight"]["meta"].dump());
+ ASSERT_EQ(highlight_meta_doc.dump(), results["hits"][0]["highlight"].dump());
}
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 8fd8cd24..f80b73f6 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -913,26 +913,25 @@ TEST_F(CollectionSpecificMoreTest, HighlightWithAccentedChars) {
ASSERT_EQ("<mark>Rāp</mark>eti Early Learning Centre", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
auto highlight_doc = R"({
- "companies":[
+ "companies": [
{
- "title":"<mark>Rāp</mark>eti Early Learning Centre"
+ "title": {
+ "matched_tokens": [
+ "Rāp"
+ ],
+ "snippet": "<mark>Rāp</mark>eti Early Learning Centre"
+ }
}
],
- "title":"<mark>Rāp</mark>eti Early Learning Centre"
+ "title": {
+ "matched_tokens": [
+ "Rāp"
+ ],
+ "snippet": "<mark>Rāp</mark>eti Early Learning Centre"
+ }
})"_json;
- ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());
-
- ASSERT_EQ(0, results["hits"][0]["highlight"]["full"].size());
- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"].size());
-
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["title"].size());
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["title"]["matched_tokens"].size());
- ASSERT_EQ("Rāp", results["hits"][0]["highlight"]["meta"]["title"]["matched_tokens"][0]);
-
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies"][0]["title"].size());
- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies"][0]["title"]["matched_tokens"].size());
- ASSERT_EQ("Rāp", results["hits"][0]["highlight"]["meta"]["companies"][0]["title"]["matched_tokens"][0]);
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
}
TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {
@@ -1272,9 +1271,9 @@ TEST_F(CollectionSpecificMoreTest, HandleStringFieldWithObjectValueEarlier) {
TEST_F(CollectionSpecificMoreTest, CopyDocHelper) {
std::vector<highlight_field_t> hightlight_items = {
- highlight_field_t("foo.bar", false, false),
- highlight_field_t("baz", false, false),
- highlight_field_t("not-found", false, false),
+ highlight_field_t("foo.bar", false, false, true),
+ highlight_field_t("baz", false, false, true),
+ highlight_field_t("not-found", false, false, true),
};
nlohmann::json src = R"({
@@ -1641,8 +1640,8 @@ TEST_F(CollectionSpecificMoreTest, HighlightOnFieldNameWithDot) {
ASSERT_EQ(1, res["hits"][0]["highlights"].size());
ASSERT_EQ("<mark>Infinity</mark> Inc.", res["hits"][0]["highlights"][0]["snippet"].get<std::string>());
- nlohmann::json snippet = R"({"org.title":"<mark>Infinity</mark> Inc."})"_json;
- ASSERT_EQ(snippet.dump(), res["hits"][0]["highlight"]["snippet"].dump());
+ nlohmann::json highlight = R"({"org.title":{"matched_tokens":["Infinity"],"snippet":"<mark>Infinity</mark> Inc."}})"_json;
+ ASSERT_EQ(highlight.dump(), res["hits"][0]["highlight"].dump());
// even if nested fields enabled, plain field names with dots should work fine
@@ -1660,6 +1659,7 @@ TEST_F(CollectionSpecificMoreTest, HighlightOnFieldNameWithDot) {
res = coll2->search("infinity", {"org.title"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(1, res["hits"].size());
ASSERT_EQ(0, res["hits"][0]["highlights"].size());
- snippet = R"({"org.title":"<mark>Infinity</mark> Inc."})"_json;
- ASSERT_EQ(snippet.dump(), res["hits"][0]["highlight"]["snippet"].dump());
+
+ highlight = R"({"org.title":{"matched_tokens":["Infinity"],"snippet":"<mark>Infinity</mark> Inc."}})"_json;
+ ASSERT_EQ(highlight.dump(), res["hits"][0]["highlight"].dump());
}