Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix new snippetting on flat fields.

Kishore Nallan856ad0efe5d2
diff --git a/include/collection.h b/include/collection.h
index 73d10449..75ad74da 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -221,8 +221,7 @@ private:
const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,
highlight_t& highlight, StringUtils & string_utils, bool use_word_tokenizer,
const size_t highlight_affix_num_tokens,
- const tsl::htrie_map<char, token_leaf>& qtoken_leaves,
- int last_valid_offset_index, const Match& match,
+ const tsl::htrie_map<char, token_leaf>& qtoken_leaves, int last_valid_offset_index,
const size_t prefix_token_num_chars, bool highlight_fully, const size_t snippet_threshold,
bool is_infix_search, std::vector<std::string>& raw_query_tokens, size_t last_valid_offset,
const std::string& highlight_start_tag, const std::string& highlight_end_tag,
@@ -239,7 +238,7 @@ private:
template<class T>
static bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
std::vector<std::string>& path_parts, size_t path_index,
- T func);
+ bool is_arr_obj_ele, int array_index, T func);
static Option<bool> resolve_field_type(field& new_field,
nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,
@@ -494,14 +493,9 @@ public:
template<class T>
bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,
std::vector<std::string>& path_parts, size_t path_index,
- T func) {
+ bool is_arr_obj_ele, int array_index, T func) {
if(path_index == path_parts.size()) {
- // end of path: guaranteed to be a string
- if(!hobj.is_string()) {
- //return false;
- }
-
- func(hobj);
+ func(hobj, is_arr_obj_ele, array_index);
return true;
}
@@ -513,11 +507,13 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::js
bool resolved = false;
for(size_t i = 0; i < it.value().size(); i++) {
auto& h_ele = it.value().at(i);
- resolved |= highlight_nested_field(hdoc, h_ele, path_parts, path_index + 1, func);
+ is_arr_obj_ele = is_arr_obj_ele || h_ele.is_object();
+ resolved = highlight_nested_field(hdoc, h_ele, path_parts, path_index + 1,
+ is_arr_obj_ele, i, func) || resolved;
}
return resolved;
} else {
- return highlight_nested_field(hdoc, it.value(), path_parts, path_index + 1, func);
+ return highlight_nested_field(hdoc, it.value(), path_parts, path_index + 1, is_arr_obj_ele, 0, func);
}
} {
return false;
diff --git a/src/collection.cpp b/src/collection.cpp
index 3d3de33a..27a41b45 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2363,6 +2363,14 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
return ;
}
+ bool flat_field = highlight_doc.contains(search_field.name);
+ std::vector<std::string> path_parts;
+ if(enable_nested_fields && !flat_field) {
+ StringUtils::split(search_field.name, path_parts, ".");
+ } else {
+ path_parts = {search_field.name};
+ }
+
const std::string& last_raw_q_token = raw_query_tokens.back();
size_t prefix_token_num_chars = StringUtils::get_num_chars(last_raw_q_token);
@@ -2399,38 +2407,69 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
LOG(INFO) << "Token: " << qtok_buff << ", root_len: " << it.value().root_len;
}*/
- bool flat_field = highlight_doc.contains(search_field.name);
-
- std::vector<std::string> path_parts;
- if(enable_nested_fields && !flat_field) {
- StringUtils::split(search_field.name, path_parts, ".");
- } else {
- path_parts = {search_field.name};
- }
+ if(!qtoken_leaves.empty()) {
+ std::vector<void*> posting_lists;
+ for(auto token_leaf: qtoken_leaves) {
+ posting_lists.push_back(token_leaf.leaf->values);
+ }
- highlight_nested_field(highlight_doc, highlight_doc, path_parts, 0, [&](nlohmann::json& h_obj) {
- Match match;
- match_index_t match_index(match, 0, 0);
- int last_valid_offset_index = -1;
- size_t last_valid_offset = 0;
+ std::map<size_t, std::vector<token_positions_t>> array_token_positions;
+ posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions);
- highlight_t array_highlight = highlight;
+ for(const auto& kv: array_token_positions) {
+ const std::vector<token_positions_t>& token_positions = kv.second;
+ size_t array_index = kv.first;
- if(!h_obj.is_string()) {
- auto val_back = h_obj;
- h_obj = nlohmann::json::object();
- h_obj["snippet"] = to_string(val_back);
- h_obj["matched_tokens"] = nlohmann::json::array();
- if(highlight_fully) {
- h_obj["value"] = val_back;
+ if(token_positions.empty()) {
+ continue;
}
- return ;
+
+ const Match & this_match = Match(field_order_kv->key, token_positions, true, true);
+ uint64_t this_match_score = this_match.get_match_score(1, token_positions.size());
+ match_indices.emplace_back(this_match, this_match_score, array_index);
+
+ /*LOG(INFO) << "doc_id: " << document["id"] << ", search_field: " << search_field.name
+ << ", words_present: " << size_t(this_match.words_present)
+ << ", match_score: " << this_match_score
+ << ", match.distance: " << size_t(this_match.distance);*/
}
+ }
+ }
+
+ const size_t max_array_matches = std::min((size_t)MAX_ARRAY_MATCHES, match_indices.size());
+ std::partial_sort(match_indices.begin(), match_indices.begin()+max_array_matches, match_indices.end());
- std::string text = h_obj.get<std::string>();
+ highlight_nested_field(highlight_doc, highlight_doc, path_parts, 0, false, -1,
+ [&](nlohmann::json& h_obj, bool is_arr_obj_ele, int array_i) {
+ if(!h_obj.is_string()) {
+ auto val_back = h_obj;
h_obj = nlohmann::json::object();
+ h_obj["snippet"] = to_string(val_back);
+ h_obj["matched_tokens"] = nlohmann::json::array();
+ if(highlight_fully) {
+ h_obj["value"] = val_back;
+ }
+ return ;
+ }
+
+ int matched_index = -1;
+
+ if(!is_arr_obj_ele) {
+ // Since we will iterate on both matching and non-matching array elements for highlighting,
+ // we need to check if `array_i`exists within match_indices vec.
+
+ for (size_t match_index = 0; match_index < match_indices.size(); match_index++) {
+ if (match_indices[match_index].index == array_i) {
+ matched_index = match_index;
+ break;
+ }
+ }
- if(qtoken_leaves.empty()) {
+ if(matched_index == -1) {
+ // If an element does not belong to an array of object field and also does not have a matching index
+ // we know that there cannot be any matching tokens for highlighting
+ std::string text = h_obj.get<std::string>();
+ h_obj = nlohmann::json::object();
h_obj["snippet"] = text;
h_obj["matched_tokens"] = nlohmann::json::array();
if(highlight_fully) {
@@ -2439,90 +2478,89 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
return ;
}
- handle_highlight_text(text, normalise, search_field, symbols_to_index,
- token_separators, array_highlight, string_utils, use_word_tokenizer,
- highlight_affix_num_tokens,
- qtoken_leaves, last_valid_offset_index, match,
- prefix_token_num_chars,
- highlight_fully, snippet_threshold, is_infix_search,
- raw_query_tokens,
- last_valid_offset, highlight_start_tag, highlight_end_tag,
- index_symbols, match_index);
+ std::sort(match_indices[matched_index].match.offsets.begin(),
+ match_indices[matched_index].match.offsets.end());
+ } else {
+ // array of object element indices will not match indexed offsets, so we will use dummy match
+ // the highlighting logic will ignore this and try to do exhaustive highlighting (look at all tokens)
+ match_indices.clear();
+ match_indices.push_back(match_index_t(Match(), 0, 0));
+ matched_index = 0;
+ }
+ const auto& match_index = match_indices[matched_index];
- if(array_highlight.snippets.empty() && array_highlight.values.empty()) {
- h_obj["snippet"] = text;
- h_obj["matched_tokens"] = nlohmann::json::array();
- }
+ size_t last_valid_offset = 0;
+ int last_valid_offset_index = -1;
- if(!array_highlight.snippets.empty()) {
- found_highlight = found_highlight || true;
+ for(size_t match_offset_i = 0; match_offset_i < match_index.match.offsets.size(); match_offset_i++) {
+ const auto& token_offset = match_index.match.offsets[match_offset_i];
+ if(token_offset.offset != MAX_DISPLACEMENT) {
+ last_valid_offset = token_offset.offset;
+ last_valid_offset_index = match_offset_i;
+ } else {
+ break;
+ }
+ }
- h_obj["snippet"] = array_highlight.snippets[0];
- h_obj["matched_tokens"] = nlohmann::json::array();
+ highlight_t array_highlight = highlight;
+ std::string text = h_obj.get<std::string>();
+ h_obj = nlohmann::json::object();
- for(auto& token_vec: array_highlight.matched_tokens) {
- for(auto& token: token_vec) {
- h_obj["matched_tokens"].push_back(token);
- }
- }
- }
+ handle_highlight_text(text, normalise, search_field, symbols_to_index,
+ token_separators, array_highlight, string_utils, use_word_tokenizer,
+ highlight_affix_num_tokens,
+ qtoken_leaves, last_valid_offset_index,
+ prefix_token_num_chars,
+ highlight_fully, snippet_threshold, is_infix_search,
+ raw_query_tokens,
+ last_valid_offset, highlight_start_tag, highlight_end_tag,
+ index_symbols, match_index);
- if(!array_highlight.values.empty()) {
- h_obj["value"] = array_highlight.values[0];;
- found_full_highlight = found_full_highlight || true;
- } else if(highlight_fully) {
- h_obj["value"] = text;
- }
- });
- if(!flat_field) {
- return;
+ if(array_highlight.snippets.empty() && array_highlight.values.empty()) {
+ h_obj["snippet"] = text;
+ h_obj["matched_tokens"] = nlohmann::json::array();
}
- if(qtoken_leaves.empty()) {
- // none of the tokens from the query were found on this field
- return ;
- }
+ if(!array_highlight.snippets.empty()) {
+ found_highlight = found_highlight || true;
- if(!search_field.is_string()) {
- return ;
- }
+ h_obj["snippet"] = array_highlight.snippets[0];
+ h_obj["matched_tokens"] = nlohmann::json::array();
- std::vector<void*> posting_lists;
- for(auto token_leaf: qtoken_leaves) {
- posting_lists.push_back(token_leaf.leaf->values);
+ for(auto& token_vec: array_highlight.matched_tokens) {
+ for(auto& token: token_vec) {
+ h_obj["matched_tokens"].push_back(token);
+ }
+ }
}
- std::map<size_t, std::vector<token_positions_t>> array_token_positions;
- posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions);
-
- for(const auto& kv: array_token_positions) {
- const std::vector<token_positions_t>& token_positions = kv.second;
- size_t array_index = kv.first;
+ if(!array_highlight.values.empty()) {
+ h_obj["value"] = array_highlight.values[0];;
+ found_full_highlight = found_full_highlight || true;
+ } else if(highlight_fully) {
+ h_obj["value"] = text;
+ }
+ });
- if(token_positions.empty()) {
- continue;
- }
+ if(!flat_field) {
+ return;
+ }
- const Match & this_match = Match(field_order_kv->key, token_positions, true, true);
- uint64_t this_match_score = this_match.get_match_score(1, token_positions.size());
- match_indices.emplace_back(this_match, this_match_score, array_index);
+ if(!search_field.is_string()) {
+ return ;
+ }
- /*LOG(INFO) << "doc_id: " << document["id"] << ", search_field: " << search_field.name
- << ", words_present: " << size_t(this_match.words_present)
- << ", match_score: " << this_match_score
- << ", match.distance: " << size_t(this_match.distance);*/
- }
+ if(!is_infix_search && qtoken_leaves.empty()) {
+ // none of the tokens from the query were found on this field
+ return ;
}
if(match_indices.empty()) {
return ;
}
- const size_t max_array_matches = std::min((size_t)MAX_ARRAY_MATCHES, match_indices.size());
- std::partial_sort(match_indices.begin(), match_indices.begin()+max_array_matches, match_indices.end());
-
for(size_t array_i = 0; array_i < max_array_matches; array_i++) {
std::sort(match_indices[array_i].match.offsets.begin(), match_indices[array_i].match.offsets.end());
const auto& match_index = match_indices[array_i];
@@ -2565,7 +2603,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
handle_highlight_text(text, normalise, search_field, symbols_to_index, token_separators,
highlight, string_utils, use_word_tokenizer, highlight_affix_num_tokens,
- qtoken_leaves, last_valid_offset_index, match, prefix_token_num_chars,
+ qtoken_leaves, last_valid_offset_index, prefix_token_num_chars,
highlight_fully, snippet_threshold, is_infix_search, raw_query_tokens,
last_valid_offset, highlight_start_tag, highlight_end_tag,
index_symbols, match_index);
@@ -2596,13 +2634,14 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, const
const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,
highlight_t& highlight, StringUtils & string_utils, bool use_word_tokenizer,
const size_t highlight_affix_num_tokens,
- const tsl::htrie_map<char, token_leaf>& qtoken_leaves,
- int last_valid_offset_index, const Match& match,
+ const tsl::htrie_map<char, token_leaf>& qtoken_leaves, int last_valid_offset_index,
const size_t prefix_token_num_chars, bool highlight_fully, const size_t snippet_threshold,
bool is_infix_search, std::vector<std::string>& raw_query_tokens, size_t last_valid_offset,
const std::string& highlight_start_tag, const std::string& highlight_end_tag,
const uint8_t* index_symbols, const match_index_t& match_index) const {
+ const Match& match = match_index.match;
+
Tokenizer tokenizer(text, normalise, false, search_field.locale, symbols_to_index, token_separators);
// word tokenizer is a secondary tokenizer used for specific languages that requires transliteration
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index bbaf42f5..29435745 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -401,13 +401,40 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {
nlohmann::json create_res = add_op.get();
ASSERT_EQ(doc.dump(), create_res.dump());
+ auto results = coll1->search("electrician", {"employees"}, "", {}, sort_fields,
+ {0}, 10, 1, FREQUENCY, {true}).get();
+
+ auto highlight_doc = R"({
+ "employees": {
+ "num": {
+ "matched_tokens": [],
+ "snippet": "1200"
+ },
+ "tags": [
+ {
+ "matched_tokens": [],
+ "snippet": "senior plumber"
+ },
+ {
+ "matched_tokens": [
+ "electrician"
+ ],
+ "snippet": "<mark>electrician</mark>"
+ }
+ ]
+ }
+ })"_json;
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
+
// search both simply nested and deeply nested array-of-objects
- auto results = coll1->search("electrician commerce", {"employees", "locations"}, "", {}, sort_fields,
+ results = coll1->search("electrician commerce", {"employees", "locations"}, "", {}, sort_fields,
{0}, 10, 1, FREQUENCY, {true}).get();
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ(doc, results["hits"][0]["document"]);
- auto highlight_doc = R"({
+ highlight_doc = R"({
"employees": {
"num": {
"matched_tokens": [],
@@ -2134,3 +2161,69 @@ TEST_F(CollectionNestedFieldsTest, UpdateNestedDocument) {
results = coll1->search("galt", {"contributors"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
ASSERT_EQ(0, results["found"].get<size_t>());
}
+
+TEST_F(CollectionNestedFieldsTest, HighlightOnFlatFieldWithSnippeting) {
+ std::vector<field> fields = {field("title", field_types::STRING, false),
+ field("body", field_types::STRING, false)};
+
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc1;
+ doc1["id"] = "0";
+ doc1["title"] = "pimples keep popping up on chin";
+ doc1["body"] = "on left side of chin under the corner of my mouth i keep getting huge pimples. they’ll go away for "
+ "a few days but come back every time and i don’t quit it. I have oily skin and acne prone. i "
+ "also just started using twice a week";
+
+ ASSERT_TRUE(coll1->add(doc1.dump()).ok());
+
+ auto results = coll1->search("pimples", {"title", "body"}, "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true}).get();
+
+ auto highlight_doc = R"({
+ "body": {
+ "matched_tokens": [
+ "pimples"
+ ],
+ "snippet": "i keep getting huge <mark>pimples</mark>. they’ll go away for"
+ },
+ "title": {
+ "matched_tokens": [
+ "pimples"
+ ],
+ "snippet": "<mark>pimples</mark> keep popping up on chin"
+ }
+ })"_json;
+
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
+
+ // with full highlighting
+
+ highlight_doc = R"({
+ "body": {
+ "matched_tokens": [
+ "pimples"
+ ],
+ "snippet": "i keep getting huge <mark>pimples</mark>. they’ll go away for",
+ "value": ""
+ },
+ "title": {
+ "matched_tokens": [
+ "pimples"
+ ],
+ "snippet": "<mark>pimples</mark> keep popping up on chin",
+ "value": "<mark>pimples</mark> keep popping up on chin"
+ }
+ })"_json;
+
+ highlight_doc["body"]["value"] = "on left side of chin under the corner of my mouth i keep getting huge "
+ "<mark>pimples</mark>. they’ll go away for a few days but come back every time "
+ "and i don’t quit it. I have oily skin and acne prone. i also just started "
+ "using twice a week";
+
+ results = coll1->search("pimples", {"title", "body"}, "", {}, {}, {2}, 10,
+ 1, FREQUENCY, {true}, 1, spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title,body").get();
+
+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());
+}