Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Highlight meta should follow nested doc structure.
Kishore Nallanc87e0ddf9ddb
diff --git a/include/collection.h b/include/collection.hindex 3f785124..297952ab 100644--- a/include/collection.h+++ b/include/collection.h@@ -246,6 +246,7 @@ private:template<class T>static bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,const nlohmann::json& fdoc, nlohmann::json& fobj,+ const nlohmann::json& mdoc, nlohmann::json& mobj,std::vector<std::string>& path_parts, size_t path_index, T func);static Option<bool> resolve_field_type(field& new_field,@@ -500,6 +501,7 @@ public:template<class T>bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,const nlohmann::json& fdoc, nlohmann::json& fobj,+ const nlohmann::json& mdoc, nlohmann::json& mobj,std::vector<std::string>& path_parts, size_t path_index, T func) {if(path_index == path_parts.size()) {// end of path: guaranteed to be a string@@ -507,7 +509,7 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::jsreturn false;}- func(hobj, fobj);+ func(hobj, fobj, mobj);}const std::string& fragment = path_parts[path_index];@@ -519,12 +521,15 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::jsfor(size_t i = 0; i < it.value().size(); i++) {auto& h_ele = it.value().at(i);auto& f_ele = fobj.empty() ? fobj : fobj[fragment][i];- resolved |= highlight_nested_field(hdoc, h_ele, fdoc, f_ele, path_parts, path_index + 1, func);+ auto& m_ele = mobj.empty() ? mobj : mobj[fragment][i];+ resolved |= highlight_nested_field(hdoc, h_ele, fdoc, f_ele, mdoc, m_ele,+ path_parts, path_index + 1, func);}return resolved;} else {auto& f_ele = fobj.empty() ? fobj : fobj[fragment];- return highlight_nested_field(hdoc, it.value(), fdoc, f_ele, path_parts, path_index + 1, func);+ auto& m_ele = mobj.empty() ? mobj : mobj[fragment];+ return highlight_nested_field(hdoc, it.value(), fdoc, f_ele, mdoc, m_ele, path_parts, path_index + 1, func);}} {return false;diff --git a/src/collection.cpp b/src/collection.cppindex 68884fe1..eb05e596 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1491,6 +1491,9 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,if(!highlight_items.empty()) {highlight_res["meta"] = nlohmann::json::object();+ copy_highlight_doc(highlight_items, document, highlight_res["meta"]);+ remove_flat_fields(highlight_res["meta"]);+ highlight_res["meta"].erase("id");highlight_res["snippet"] = nlohmann::json::object();copy_highlight_doc(highlight_items, document, highlight_res["snippet"]);@@ -1568,9 +1571,11 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,// remove fields from highlight doc that were not highlightedif(!hfield_names.empty()) {+ prune_doc(highlight_res["meta"], hfield_names, tsl::htrie_set<char>(), "");prune_doc(highlight_res["snippet"], hfield_names, tsl::htrie_set<char>(), "");prune_doc(highlight_res["full"], h_full_field_names, tsl::htrie_set<char>(), "");} else {+ highlight_res["meta"] = nlohmann::json::object();highlight_res["snippet"] = nlohmann::json::object();highlight_res["full"] = nlohmann::json::object();}@@ -2415,7 +2420,8 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea}highlight_nested_field(highlight_doc, highlight_doc, highlight_full_doc, highlight_full_doc,- path_parts, 0, [&](nlohmann::json& h_obj, nlohmann::json& f_obj) {+ highlight_meta, highlight_meta, path_parts, 0,+ [&](nlohmann::json& h_obj, nlohmann::json& f_obj, nlohmann::json& m_obj) {Match match;match_index_t match_index(match, 0, 0);int last_valid_offset_index = -1;@@ -2437,9 +2443,11 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaif(!array_highlight.snippets.empty()) {h_obj = array_highlight.snippets[0];found_highlight = found_highlight || true;++ m_obj = nlohmann::json::object();for(auto& token_vec: array_highlight.matched_tokens) {for(auto& token: token_vec) {- matched_tokens.insert(token);+ m_obj["matched_tokens"].push_back(token);}}}@@ -2450,13 +2458,6 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea}});- if(found_highlight) {- highlight_meta[search_field.name] = nlohmann::json::object();- for(auto it = matched_tokens.begin(); it != matched_tokens.end(); ++it) {- highlight_meta[search_field.name]["matched_tokens"].push_back(it.key());- }- }-return ;}@@ -2570,9 +2571,19 @@ void Collection::highlight_result(const std::string& raw_query, const field &seaStringUtils::split(search_field.name, parts, ".");}+ nlohmann::json* mval = highlight_meta.contains(parts[0]) ? &highlight_meta[parts[0]] : nullptr;nlohmann::json* hval = highlight_doc.contains(parts[0]) ? &highlight_doc[parts[0]] : nullptr;nlohmann::json* fval = highlight_full_doc.contains(parts[0]) ? &highlight_full_doc[parts[0]] : nullptr;+ for(size_t i = 1; mval != nullptr && i < parts.size(); i++) {+ const auto& part = parts[i];+ if(mval->contains(part)) {+ mval = &mval->at(part);+ } else {+ mval = nullptr;+ }+ }+for(size_t i = 1; hval != nullptr && i < parts.size(); i++) {const auto& part = parts[i];if(hval->contains(part)) {@@ -2602,9 +2613,12 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea}}- highlight_meta[search_field.name] = nlohmann::json::object();- for(auto it = matched_tokens.begin(); it != matched_tokens.end(); ++it) {- highlight_meta[search_field.name]["matched_tokens"].push_back(it.key());+ if(mval) {+ *mval = nlohmann::json::object();+ mval->emplace("matched_tokens", nlohmann::json::array());+ for(auto it = matched_tokens.begin(); it != matched_tokens.end(); ++it) {+ mval->at("matched_tokens").push_back(it.key());+ }}}diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex 58950b71..69e6ca78 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -786,14 +786,14 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {ASSERT_EQ("brown", matched_tokens[0]);ASSERT_EQ("fox", matched_tokens[1]);- ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"].size());- matched_tokens = results["hits"][0]["highlight"]["meta"]["details.names"]["matched_tokens"].get<std::vector<std::string>>();+ ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].size());+ matched_tokens = results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].get<std::vector<std::string>>();std::sort(matched_tokens.begin(), matched_tokens.end());ASSERT_EQ("brown", matched_tokens[0]);ASSERT_EQ("fox", matched_tokens[1]);- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["locations.address.street"]["matched_tokens"].size());- matched_tokens = results["hits"][0]["highlight"]["meta"]["locations.address.street"]["matched_tokens"].get<std::vector<std::string>>();+ ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["locations"][0]["address"]["street"]["matched_tokens"].size());+ matched_tokens = results["hits"][0]["highlight"]["meta"]["locations"][0]["address"]["street"]["matched_tokens"].get<std::vector<std::string>>();std::sort(matched_tokens.begin(), matched_tokens.end());ASSERT_EQ("Brown", matched_tokens[0]);@@ -804,9 +804,10 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {20, {}, {}, {}, 0, "<mark>", "</mark>", {}, 1000, true, false, true,"x").get();- ASSERT_EQ(2, results["hits"][0]["highlight"].size());+ ASSERT_EQ(3, results["hits"][0]["highlight"].size());ASSERT_EQ(0, results["hits"][0]["highlight"]["snippet"].size());ASSERT_EQ(0, results["hits"][0]["highlight"]["full"].size());+ ASSERT_EQ(0, results["hits"][0]["highlight"]["meta"].size());}TEST_F(CollectionNestedFieldsTest, FieldsWithExplicitSchema) {diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 6cc38afe..8fd8cd24 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -930,9 +930,9 @@ TEST_F(CollectionSpecificMoreTest, HighlightWithAccentedChars) {ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["title"]["matched_tokens"].size());ASSERT_EQ("Rāp", results["hits"][0]["highlight"]["meta"]["title"]["matched_tokens"][0]);- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies.title"].size());- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies.title"]["matched_tokens"].size());- ASSERT_EQ("Rāp", results["hits"][0]["highlight"]["meta"]["companies.title"]["matched_tokens"][0]);+ ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies"][0]["title"].size());+ ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["companies"][0]["title"]["matched_tokens"].size());+ ASSERT_EQ("Rāp", results["hits"][0]["highlight"]["meta"]["companies"][0]["title"]["matched_tokens"][0]);}TEST_F(CollectionSpecificMoreTest, FieldWeightNormalization) {