Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix highlight meta structure.
Kishore Nallan5292956718a1
diff --git a/include/collection.h b/include/collection.hindex 297952ab..298dfd12 100644--- a/include/collection.h+++ b/include/collection.h@@ -247,7 +247,8 @@ private:static bool highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,const nlohmann::json& fdoc, nlohmann::json& fobj,const nlohmann::json& mdoc, nlohmann::json& mobj,- std::vector<std::string>& path_parts, size_t path_index, T func);+ std::vector<std::string>& path_parts, size_t path_index,+ int arr_index, nlohmann::json& parent_mobj, T func);static Option<bool> resolve_field_type(field& new_field,nlohmann::detail::iter_impl<nlohmann::basic_json<>>& kv,@@ -502,14 +503,16 @@ template<class T>bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::json& hobj,const nlohmann::json& fdoc, nlohmann::json& fobj,const nlohmann::json& mdoc, nlohmann::json& mobj,- std::vector<std::string>& path_parts, size_t path_index, T func) {+ std::vector<std::string>& path_parts, size_t path_index,+ int arr_index, nlohmann::json& parent_mobj,+ T func) {if(path_index == path_parts.size()) {// end of path: guaranteed to be a stringif(!hobj.is_string()) {return false;}- func(hobj, fobj, mobj);+ func(hobj, fobj, mobj, arr_index, parent_mobj);}const std::string& fragment = path_parts[path_index];@@ -518,18 +521,24 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::jsif(it != hobj.end()) {if(it.value().is_array()) {bool resolved = false;+ auto& parent_mobj = mobj.empty() ? mobj : mobj[fragment];+ nlohmann::json empty_obj;+for(size_t i = 0; i < it.value().size(); i++) {auto& h_ele = it.value().at(i);auto& f_ele = fobj.empty() ? fobj : fobj[fragment][i];- auto& m_ele = mobj.empty() ? mobj : mobj[fragment][i];+ // if value is a primitive, we don't want the array object getting destroyed+ auto& m_ele = h_ele.is_object() && !mobj.empty() ? mobj[fragment][i] : empty_obj;resolved |= highlight_nested_field(hdoc, h_ele, fdoc, f_ele, mdoc, m_ele,- path_parts, path_index + 1, func);+ path_parts, path_index + 1, i, parent_mobj, func);}return resolved;} else {auto& f_ele = fobj.empty() ? fobj : fobj[fragment];auto& m_ele = mobj.empty() ? mobj : mobj[fragment];- return highlight_nested_field(hdoc, it.value(), fdoc, f_ele, mdoc, m_ele, path_parts, path_index + 1, func);+ auto& parent_mobj = mobj.empty() ? mobj : mobj[fragment];+ return highlight_nested_field(hdoc, it.value(), fdoc, f_ele, mdoc, m_ele, path_parts, path_index + 1,+ -1, parent_mobj, func);}} {return false;diff --git a/src/collection.cpp b/src/collection.cppindex eb05e596..dec6f33e 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2420,8 +2420,9 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea}highlight_nested_field(highlight_doc, highlight_doc, highlight_full_doc, highlight_full_doc,- highlight_meta, highlight_meta, path_parts, 0,- [&](nlohmann::json& h_obj, nlohmann::json& f_obj, nlohmann::json& m_obj) {+ highlight_meta, highlight_meta, path_parts, 0, -1, highlight_meta,+ [&](nlohmann::json& h_obj, nlohmann::json& f_obj, nlohmann::json& m_obj,+ int arr_index, nlohmann::json& parent_mobj) {Match match;match_index_t match_index(match, 0, 0);int last_valid_offset_index = -1;@@ -2444,12 +2445,20 @@ void Collection::highlight_result(const std::string& raw_query, const field &seah_obj = array_highlight.snippets[0];found_highlight = found_highlight || true;- m_obj = nlohmann::json::object();+ nlohmann::json& ref_meta_obj = (arr_index >= 0) ? parent_mobj : m_obj;+ if(!ref_meta_obj.is_object() || !ref_meta_obj.contains("matched_tokens")) {+ ref_meta_obj = nlohmann::json::object();+ }+for(auto& token_vec: array_highlight.matched_tokens) {for(auto& token: token_vec) {- m_obj["matched_tokens"].push_back(token);+ ref_meta_obj["matched_tokens"].push_back(token);}}++ if(arr_index >= 0) {+ ref_meta_obj["matched_indices"].push_back(arr_index);+ }}if(highlight_fully && !array_highlight.values.empty()) {@@ -2619,6 +2628,13 @@ void Collection::highlight_result(const std::string& raw_query, const field &seafor(auto it = matched_tokens.begin(); it != matched_tokens.end(); ++it) {mval->at("matched_tokens").push_back(it.key());}++ if(hval->is_array()) {+ mval->emplace("matched_indices", nlohmann::json::array());+ for(auto highlight_index: highlight.indices) {+ mval->at("matched_indices").push_back(highlight_index);+ }+ }}}diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex e1dc4c61..ad30313f 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -620,8 +620,42 @@ TEST_F(CollectionNestedFieldsTest, HighlightNestedFieldFully) {]})"_json;+ auto highlight_meta_doc = R"({+ "locations":[+ {+ "address":{+ "city":"Beaverton",+ "products":{+ "matched_indices":[+ 0+ ],+ "matched_tokens":[+ "shoes"+ ]+ },+ "street":"One Bowerman Drive"+ }+ },+ {+ "address":{+ "city":"Thornhill",+ "products":{+ "matched_indices":[+ 1+ ],+ "matched_tokens":[+ "shoes"+ ]+ },+ "street":"175 Commerce Drive"+ }+ }+ ]+ })"_json;+ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["full"].dump());ASSERT_EQ(highlight_full_doc.dump(), results["hits"][0]["highlight"]["snippet"].dump());+ ASSERT_EQ(highlight_meta_doc.dump(), results["hits"][0]["highlight"]["meta"].dump());// full highlighting only one of the 3 highlight fieldsresults = coll1->search("drive", {"company.names", "company_names", "locations.address"}, "", {}, sort_fields, {0}, 10, 1,@@ -778,7 +812,7 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address").get();ASSERT_EQ(3, results["hits"][0]["highlight"]["meta"].size());- ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["company_names"].size());+ ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"].size());ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].size());std::vector<std::string> matched_tokens = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_tokens"].get<std::vector<std::string>>();@@ -786,6 +820,11 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {ASSERT_EQ("brown", matched_tokens[0]);ASSERT_EQ("fox", matched_tokens[1]);+ ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].size());+ std::vector<size_t> matched_indices = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].get<std::vector<size_t>>();+ ASSERT_EQ(0, matched_indices[0]);+ ASSERT_EQ(1, matched_indices[1]);+ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].size());matched_tokens = results["hits"][0]["highlight"]["meta"]["details"]["names"]["matched_tokens"].get<std::vector<std::string>>();std::sort(matched_tokens.begin(), matched_tokens.end());@@ -797,6 +836,20 @@ TEST_F(CollectionNestedFieldsTest, HighlightShouldHaveMeta) {std::sort(matched_tokens.begin(), matched_tokens.end());ASSERT_EQ("Brown", matched_tokens[0]);+ // partial matched index match++ results = coll1->search("fast", {"company_names"},+ "", {}, sort_fields, {0}, 10, 1,+ token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "", 30, 4, "locations.address").get();++ ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"].size());+ ASSERT_EQ(2, results["hits"][0]["highlight"]["meta"]["company_names"].size());++ ASSERT_EQ(1, results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].size());+ matched_indices = results["hits"][0]["highlight"]["meta"]["company_names"]["matched_indices"].get<std::vector<size_t>>();+ ASSERT_EQ(1, matched_indices[0]);+// when no highlighting is enabled by setting unknown field for highlightingresults = coll1->search("brown fox", {"company_names", "details", "locations"}, "", {}, sort_fields, {0}, 10, 1,token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),@@ -1608,3 +1661,54 @@ TEST_F(CollectionNestedFieldsTest, NestedSchemaAutoAndFacet) {ASSERT_TRUE(coll1->get_schema()["schools.name"].optional);}++TEST_F(CollectionNestedFieldsTest, HighlightArrayInsideArrayOfObj) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "studies", "type": "auto"}+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc1 = R"({+ "id": "0",+ "studies": [+ {"name": "College 1", "tags": ["foo", "bar"]},+ {"name": "College 1", "tags": ["alpha", "beta"]}+ ]+ })"_json;++ ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());++ auto results = coll1->search("beta", {"studies"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());++ nlohmann::json highlight_meta_doc = R"({+ "studies":[+ {+ "tags":[+ "foo",+ "bar"+ ]+ },+ {+ "tags":{+ "matched_indices":[+ 1+ ],+ "matched_tokens":[+ "beta"+ ]+ }+ }+ ]+ })"_json;++ ASSERT_EQ(highlight_meta_doc.dump(), results["hits"][0]["highlight"]["meta"].dump());+}