Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix new snippetting on flat fields.
Kishore Nallan856ad0efe5d2
diff --git a/include/collection.h b/include/collection.hindex 73d10449..75ad74da 100644--- a/include/collection.h+++ b/include/collection.h@@ -221,8 +221,7 @@ private:const std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,highlight_t& highlight, StringUtils & string_utils, bool use_word_tokenizer,const size_t highlight_affix_num_tokens,- const tsl::htrie_map<char, token_leaf>& qtoken_leaves,- int last_valid_offset_index, const Match& match,+ const tsl::htrie_map<char, token_leaf>& qtoken_leaves, int last_valid_offset_index,const size_t prefix_token_num_chars, bool highlight_fully, const size_t snippet_threshold,bool is_infix_search, std::vector<std::string>& raw_query_tokens, size_t last_valid_offset,const std::string& highlight_start_tag, const std::string& highlight_end_tag,@@ -239,7 +238,7 @@ private:template<class T>static bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,std::vector<std::string>& path_parts, size_t path_index,- T func);+ bool is_arr_obj_ele, int array_index, T func);static Option<bool> resolve_field_type(field& new_field,nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,@@ -494,14 +493,9 @@ public:template<class T>bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,std::vector<std::string>& path_parts, size_t path_index,- T func) {+ bool is_arr_obj_ele, int array_index, T func) {if(path_index == path_parts.size()) {- // end of path: guaranteed to be a string- if(!hobj.is_string()) {- //return false;- }-- func(hobj);+ func(hobj, is_arr_obj_ele, array_index);return true;}@@ -513,11 +507,13 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::jsbool resolved = false;for(size_t i = 0; i < it.value().size(); i++) {auto& h_ele = it.value().at(i);- resolved |= highlight_nested_field(hdoc, h_ele, path_parts, path_index + 1, func);+ is_arr_obj_ele = is_arr_obj_ele || h_ele.is_object();+ resolved = highlight_nested_field(hdoc, h_ele, path_parts, path_index + 1,+ is_arr_obj_ele, i, func) || resolved;}return resolved;} else {- return highlight_nested_field(hdoc, it.value(), path_parts, path_index + 1, func);+ return highlight_nested_field(hdoc, it.value(), path_parts, path_index + 1, is_arr_obj_ele, 0, func);}} {return false;diff --git a/src/collection.cpp b/src/collection.cppindex 3d3de33a..27a41b45 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2363,6 +2363,14 @@ void Collection::highlight_result(const std::string& raw_query, const field &seareturn ;}+ bool flat_field = highlight_doc.contains(search_field.name);+ std::vector<std::string> path_parts;+ if(enable_nested_fields && !flat_field) {+ StringUtils::split(search_field.name, path_parts, ".");+ } else {+ path_parts = {search_field.name};+ }+const std::string& last_raw_q_token = raw_query_tokens.back();size_t prefix_token_num_chars = StringUtils::get_num_chars(last_raw_q_token);@@ -2399,38 +2407,69 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaLOG(INFO) << "Token: " << qtok_buff << ", root_len: " << it.value().root_len;}*/- bool flat_field = highlight_doc.contains(search_field.name);-- std::vector<std::string> path_parts;- if(enable_nested_fields && !flat_field) {- StringUtils::split(search_field.name, path_parts, ".");- } else {- path_parts = {search_field.name};- }+ if(!qtoken_leaves.empty()) {+ std::vector<void*> posting_lists;+ for(auto token_leaf: qtoken_leaves) {+ posting_lists.push_back(token_leaf.leaf->values);+ }- highlight_nested_field(highlight_doc, highlight_doc, path_parts, 0, [&](nlohmann::json& h_obj) {- Match match;- match_index_t match_index(match, 0, 0);- int last_valid_offset_index = -1;- size_t last_valid_offset = 0;+ std::map<size_t, std::vector<token_positions_t>> array_token_positions;+ posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions);- highlight_t array_highlight = highlight;+ for(const auto& kv: array_token_positions) {+ const std::vector<token_positions_t>& token_positions = kv.second;+ size_t array_index = kv.first;- if(!h_obj.is_string()) {- auto val_back = h_obj;- h_obj = nlohmann::json::object();- h_obj["snippet"] = to_string(val_back);- h_obj["matched_tokens"] = nlohmann::json::array();- if(highlight_fully) {- h_obj["value"] = val_back;+ if(token_positions.empty()) {+ continue;}- return ;++ const Match & this_match = Match(field_order_kv->key, token_positions, true, true);+ uint64_t this_match_score = this_match.get_match_score(1, token_positions.size());+ match_indices.emplace_back(this_match, this_match_score, array_index);++ /*LOG(INFO) << "doc_id: " << document["id"] << ", search_field: " << search_field.name+ << ", words_present: " << size_t(this_match.words_present)+ << ", match_score: " << this_match_score+ << ", match.distance: " << size_t(this_match.distance);*/}+ }+ }++ const size_t max_array_matches = std::min((size_t)MAX_ARRAY_MATCHES, match_indices.size());+ std::partial_sort(match_indices.begin(), match_indices.begin()+max_array_matches, match_indices.end());- std::string text = h_obj.get<std::string>();+ highlight_nested_field(highlight_doc, highlight_doc, path_parts, 0, false, -1,+ [&](nlohmann::json& h_obj, bool is_arr_obj_ele, int array_i) {+ if(!h_obj.is_string()) {+ auto val_back = h_obj;h_obj = nlohmann::json::object();+ h_obj["snippet"] = to_string(val_back);+ h_obj["matched_tokens"] = nlohmann::json::array();+ if(highlight_fully) {+ h_obj["value"] = val_back;+ }+ return ;+ }++ int matched_index = -1;++ if(!is_arr_obj_ele) {+ // Since we will iterate on both matching and non-matching array elements for highlighting,+ // we need to check if `array_i`exists within match_indices vec.++ for (size_t match_index = 0; match_index < match_indices.size(); match_index++) {+ if (match_indices[match_index].index == array_i) {+ matched_index = match_index;+ break;+ }+ }- if(qtoken_leaves.empty()) {+ if(matched_index == -1) {+ // If an element does not belong to an array of object field and also does not have a matching index+ // we know that there cannot be any matching tokens for highlighting+ std::string text = h_obj.get<std::string>();+ h_obj = nlohmann::json::object();h_obj["snippet"] = text;h_obj["matched_tokens"] = nlohmann::json::array();if(highlight_fully) {@@ -2439,90 +2478,89 @@ void Collection::highlight_result(const std::string& raw_query, const field &seareturn ;}- handle_highlight_text(text, normalise, search_field, symbols_to_index,- token_separators, array_highlight, string_utils, use_word_tokenizer,- highlight_affix_num_tokens,- qtoken_leaves, last_valid_offset_index, match,- prefix_token_num_chars,- highlight_fully, snippet_threshold, is_infix_search,- raw_query_tokens,- last_valid_offset, highlight_start_tag, highlight_end_tag,- index_symbols, match_index);+ std::sort(match_indices[matched_index].match.offsets.begin(),+ match_indices[matched_index].match.offsets.end());+ } else {+ // array of object element indices will not match indexed offsets, so we will use dummy match+ // the highlighting logic will ignore this and try to do exhaustive highlighting (look at all tokens)+ match_indices.clear();+ match_indices.push_back(match_index_t(Match(), 0, 0));+ matched_index = 0;+ }+ const auto& match_index = match_indices[matched_index];- if(array_highlight.snippets.empty() && array_highlight.values.empty()) {- h_obj["snippet"] = text;- h_obj["matched_tokens"] = nlohmann::json::array();- }+ size_t last_valid_offset = 0;+ int last_valid_offset_index = -1;- if(!array_highlight.snippets.empty()) {- found_highlight = found_highlight || true;+ for(size_t match_offset_i = 0; match_offset_i < match_index.match.offsets.size(); match_offset_i++) {+ const auto& token_offset = match_index.match.offsets[match_offset_i];+ if(token_offset.offset != MAX_DISPLACEMENT) {+ last_valid_offset = token_offset.offset;+ last_valid_offset_index = match_offset_i;+ } else {+ break;+ }+ }- h_obj["snippet"] = array_highlight.snippets[0];- h_obj["matched_tokens"] = nlohmann::json::array();+ highlight_t array_highlight = highlight;+ std::string text = h_obj.get<std::string>();+ h_obj = nlohmann::json::object();- for(auto& token_vec: array_highlight.matched_tokens) {- for(auto& token: token_vec) {- h_obj["matched_tokens"].push_back(token);- }- }- }+ handle_highlight_text(text, normalise, search_field, symbols_to_index,+ token_separators, array_highlight, string_utils, use_word_tokenizer,+ highlight_affix_num_tokens,+ qtoken_leaves, last_valid_offset_index,+ prefix_token_num_chars,+ highlight_fully, snippet_threshold, is_infix_search,+ raw_query_tokens,+ last_valid_offset, highlight_start_tag, highlight_end_tag,+ index_symbols, match_index);- if(!array_highlight.values.empty()) {- h_obj["value"] = array_highlight.values[0];;- found_full_highlight = found_full_highlight || true;- } else if(highlight_fully) {- h_obj["value"] = text;- }- });- if(!flat_field) {- return;+ if(array_highlight.snippets.empty() && array_highlight.values.empty()) {+ h_obj["snippet"] = text;+ h_obj["matched_tokens"] = nlohmann::json::array();}- if(qtoken_leaves.empty()) {- // none of the tokens from the query were found on this field- return ;- }+ if(!array_highlight.snippets.empty()) {+ found_highlight = found_highlight || true;- if(!search_field.is_string()) {- return ;- }+ h_obj["snippet"] = array_highlight.snippets[0];+ h_obj["matched_tokens"] = nlohmann::json::array();- std::vector<void*> posting_lists;- for(auto token_leaf: qtoken_leaves) {- posting_lists.push_back(token_leaf.leaf->values);+ for(auto& token_vec: array_highlight.matched_tokens) {+ for(auto& token: token_vec) {+ h_obj["matched_tokens"].push_back(token);+ }+ }}- std::map<size_t, std::vector<token_positions_t>> array_token_positions;- posting_t::get_array_token_positions(field_order_kv->key, posting_lists, array_token_positions);-- for(const auto& kv: array_token_positions) {- const std::vector<token_positions_t>& token_positions = kv.second;- size_t array_index = kv.first;+ if(!array_highlight.values.empty()) {+ h_obj["value"] = array_highlight.values[0];;+ found_full_highlight = found_full_highlight || true;+ } else if(highlight_fully) {+ h_obj["value"] = text;+ }+ });- if(token_positions.empty()) {- continue;- }+ if(!flat_field) {+ return;+ }- const Match & this_match = Match(field_order_kv->key, token_positions, true, true);- uint64_t this_match_score = this_match.get_match_score(1, token_positions.size());- match_indices.emplace_back(this_match, this_match_score, array_index);+ if(!search_field.is_string()) {+ return ;+ }- /*LOG(INFO) << "doc_id: " << document["id"] << ", search_field: " << search_field.name- << ", words_present: " << size_t(this_match.words_present)- << ", match_score: " << this_match_score- << ", match.distance: " << size_t(this_match.distance);*/- }+ if(!is_infix_search && qtoken_leaves.empty()) {+ // none of the tokens from the query were found on this field+ return ;}if(match_indices.empty()) {return ;}- const size_t max_array_matches = std::min((size_t)MAX_ARRAY_MATCHES, match_indices.size());- std::partial_sort(match_indices.begin(), match_indices.begin()+max_array_matches, match_indices.end());-for(size_t array_i = 0; array_i < max_array_matches; array_i++) {std::sort(match_indices[array_i].match.offsets.begin(), match_indices[array_i].match.offsets.end());const auto& match_index = match_indices[array_i];@@ -2565,7 +2603,7 @@ void Collection::highlight_result(const std::string& raw_query, const field &seahandle_highlight_text(text, normalise, search_field, symbols_to_index, token_separators,highlight, string_utils, use_word_tokenizer, highlight_affix_num_tokens,- qtoken_leaves, last_valid_offset_index, match, prefix_token_num_chars,+ qtoken_leaves, last_valid_offset_index, prefix_token_num_chars,highlight_fully, snippet_threshold, is_infix_search, raw_query_tokens,last_valid_offset, highlight_start_tag, highlight_end_tag,index_symbols, match_index);@@ -2596,13 +2634,14 @@ bool Collection::handle_highlight_text(std::string& text, bool normalise, constconst std::vector<char>& symbols_to_index, const std::vector<char>& token_separators,highlight_t& highlight, StringUtils & string_utils, bool use_word_tokenizer,const size_t highlight_affix_num_tokens,- const tsl::htrie_map<char, token_leaf>& qtoken_leaves,- int last_valid_offset_index, const Match& match,+ const tsl::htrie_map<char, token_leaf>& qtoken_leaves, int last_valid_offset_index,const size_t prefix_token_num_chars, bool highlight_fully, const size_t snippet_threshold,bool is_infix_search, std::vector<std::string>& raw_query_tokens, size_t last_valid_offset,const std::string& highlight_start_tag, const std::string& highlight_end_tag,const uint8_t* index_symbols, const match_index_t& match_index) const {+ const Match& match = match_index.match;+Tokenizer tokenizer(text, normalise, false, search_field.locale, symbols_to_index, token_separators);// word tokenizer is a secondary tokenizer used for specific languages that requires transliterationdiff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex bbaf42f5..29435745 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -401,13 +401,40 @@ TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {nlohmann::json create_res = add_op.get();ASSERT_EQ(doc.dump(), create_res.dump());+ auto results = coll1->search("electrician", {"employees"}, "", {}, sort_fields,+ {0}, 10, 1, FREQUENCY, {true}).get();++ auto highlight_doc = R"({+ "employees": {+ "num": {+ "matched_tokens": [],+ "snippet": "1200"+ },+ "tags": [+ {+ "matched_tokens": [],+ "snippet": "senior plumber"+ },+ {+ "matched_tokens": [+ "electrician"+ ],+ "snippet": "<mark>electrician</mark>"+ }+ ]+ }+ })"_json;++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());+// search both simply nested and deeply nested array-of-objects- auto results = coll1->search("electrician commerce", {"employees", "locations"}, "", {}, sort_fields,+ results = coll1->search("electrician commerce", {"employees", "locations"}, "", {}, sort_fields,{0}, 10, 1, FREQUENCY, {true}).get();ASSERT_EQ(1, results["hits"].size());ASSERT_EQ(doc, results["hits"][0]["document"]);- auto highlight_doc = R"({+ highlight_doc = R"({"employees": {"num": {"matched_tokens": [],@@ -2134,3 +2161,69 @@ TEST_F(CollectionNestedFieldsTest, UpdateNestedDocument) {results = coll1->search("galt", {"contributors"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();ASSERT_EQ(0, results["found"].get<size_t>());}++TEST_F(CollectionNestedFieldsTest, HighlightOnFlatFieldWithSnippeting) {+ std::vector<field> fields = {field("title", field_types::STRING, false),+ field("body", field_types::STRING, false)};++ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc1;+ doc1["id"] = "0";+ doc1["title"] = "pimples keep popping up on chin";+ doc1["body"] = "on left side of chin under the corner of my mouth i keep getting huge pimples. they’ll go away for "+ "a few days but come back every time and i don’t quit it. I have oily skin and acne prone. i "+ "also just started using twice a week";++ ASSERT_TRUE(coll1->add(doc1.dump()).ok());++ auto results = coll1->search("pimples", {"title", "body"}, "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true}).get();++ auto highlight_doc = R"({+ "body": {+ "matched_tokens": [+ "pimples"+ ],+ "snippet": "i keep getting huge <mark>pimples</mark>. they’ll go away for"+ },+ "title": {+ "matched_tokens": [+ "pimples"+ ],+ "snippet": "<mark>pimples</mark> keep popping up on chin"+ }+ })"_json;++ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());++ // with full highlighting++ highlight_doc = R"({+ "body": {+ "matched_tokens": [+ "pimples"+ ],+ "snippet": "i keep getting huge <mark>pimples</mark>. they’ll go away for",+ "value": ""+ },+ "title": {+ "matched_tokens": [+ "pimples"+ ],+ "snippet": "<mark>pimples</mark> keep popping up on chin",+ "value": "<mark>pimples</mark> keep popping up on chin"+ }+ })"_json;++ highlight_doc["body"]["value"] = "on left side of chin under the corner of my mouth i keep getting huge "+ "<mark>pimples</mark>. they’ll go away for a few days but come back every time "+ "and i don’t quit it. I have oily skin and acne prone. i also just started "+ "using twice a week";++ results = coll1->search("pimples", {"title", "body"}, "", {}, {}, {2}, 10,+ 1, FREQUENCY, {true}, 1, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "title,body").get();++ ASSERT_EQ(highlight_doc.dump(), results["hits"][0]["highlight"].dump());+}