Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix duplication of nested fields during flattening.
Kishore Nallancbf5985307b9
diff --git a/include/collection.h b/include/collection.hindex fd5c80b1..73d10449 100644--- a/include/collection.h+++ b/include/collection.h@@ -161,11 +161,10 @@ private:const DIRTY_VALUES& dirty_values,const tsl::htrie_map<char, field>& schema,const std::unordered_map<std::string, field>& dyn_fields,- const tsl::htrie_map<char, field>& nested_fields,+ tsl::htrie_map<char, field>& nested_fields,const std::string& fallback_field_type,bool is_update,std::vector<field>& new_fields,- std::vector<field>& nested_fields_found,bool enable_nested_fields);static bool facet_count_compare(const std::pair<uint64_t, facet_count_t>& a,@@ -199,13 +198,11 @@ private:Option<bool> batch_alter_data(const std::vector<field>& alter_fields,const std::vector<field>& del_fields,- const std::string& this_fallback_field_type,- const tsl::htrie_map<char, field>& alter_nested_fields);+ const std::string& this_fallback_field_type);Option<bool> validate_alter_payload(nlohmann::json& schema_changes,std::vector<field>& addition_fields,std::vector<field>& reindex_fields,- tsl::htrie_map<char, field>& new_nested_fields,std::vector<field>& del_fields,std::string& fallback_field_type);@@ -237,11 +234,6 @@ private:bool extract_only_string_fields,bool enable_nested_fields);- static Option<bool> flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields_found,- const tsl::htrie_map<char, field>& schema,- bool missing_is_ok,- std::vector<field>& new_fields);-bool is_nested_array(const nlohmann::json& obj, std::vector<std::string> path_parts, size_t part_i) const;template<class T>@@ -255,8 +247,8 @@ private:const DIRTY_VALUES& dirty_values,const bool found_dynamic_field,const std::string& fallback_field_type,- std::vector<field>& new_fields,- std::vector<field>& nested_fields_found);+ bool enable_nested_fields,+ std::vector<field>& new_fields);public:diff --git a/include/field.h b/include/field.hindex 49ef7336..b6057e1b 100644--- a/include/field.h+++ b/include/field.h@@ -424,10 +424,10 @@ struct field {std::vector<std::string>& path_parts, size_t path_index, bool has_array,bool has_obj_array, std::unordered_map<std::string, field>& flattened_fields);- static Option<bool> flatten_doc(nlohmann::json& document, const std::vector<field>& nested_fields,+ static Option<bool> flatten_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields,bool missing_is_ok, std::vector<field>& flattened_fields);- static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields);+ static void compact_nested_fields(tsl::htrie_map<char, field>& nested_fields);};struct filter_node_t;diff --git a/src/collection.cpp b/src/collection.cppindex 1f1cee5a..3d3de33a 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -257,7 +257,6 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma// if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexingif(!fallback_field_type.empty() || !dynamic_fields.empty() || !nested_fields.empty()) {std::vector<field> new_fields;- std::vector<field> nested_fields_found;std::unique_lock lock(mutex);Option<bool> new_fields_op = detect_new_fields(record.doc, dirty_values,@@ -265,27 +264,32 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmanested_fields,fallback_field_type,record.is_update,- new_fields, nested_fields_found,+ new_fields,enable_nested_fields);if(!new_fields_op.ok()) {record.index_failure(new_fields_op.code(), new_fields_op.error());}else if(!new_fields.empty()) {+ bool found_new_field = false;for(auto& new_field: new_fields) {- search_schema.emplace(new_field.name, new_field);- fields.emplace_back(new_field);- }-- for(auto& nested_field: nested_fields_found) {- nested_fields.emplace(nested_field.name, nested_field);+ if(search_schema.find(new_field.name) == search_schema.end()) {+ found_new_field = true;+ search_schema.emplace(new_field.name, new_field);+ fields.emplace_back(new_field);+ if(new_field.nested) {+ nested_fields.emplace(new_field.name, new_field);+ }+ }}- auto persist_op = persist_collection_meta();- if(!persist_op.ok()) {- record.index_failure(persist_op.code(), persist_op.error());- } else {- index->refresh_schemas(new_fields, {});+ if(found_new_field) {+ auto persist_op = persist_collection_meta();+ if(!persist_op.ok()) {+ record.index_failure(persist_op.code(), persist_op.error());+ } else {+ index->refresh_schemas(new_fields, {});+ }}}}@@ -347,7 +351,7 @@ void Collection::batch_index(std::vector<index_record>& index_records, std::vect}} else {- // remove flattened field values before storing on disk (.flat meta will be kept)+ // remove flattened field values before storing on diskremove_flat_fields(index_record.doc);const std::string& seq_id_str = std::to_string(index_record.seq_id);@@ -3110,8 +3114,10 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,}if(!raw_doc && enable_nested_fields) {- field::flatten_stored_doc(document, nested_fields);+ std::vector<field> flattened_fields;+ field::flatten_doc(document, nested_fields, true, flattened_fields);}+return Option<bool>(true);}@@ -3193,6 +3199,9 @@ SynonymIndex* Collection::get_synonym_index() {}Option<bool> Collection::persist_collection_meta() {+ // first compact nested fields (to keep only parents of expanded children)+ field::compact_nested_fields(nested_fields);+std::string coll_meta_json;StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);@@ -3229,8 +3238,7 @@ Option<bool> Collection::persist_collection_meta() {Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields,const std::vector<field>& del_fields,- const std::string& this_fallback_field_type,- const tsl::htrie_map<char, field>& alter_nested_fields) {+ const std::string& this_fallback_field_type) {// Update schema with additions (deletions can only be made later)std::vector<field> new_fields;tsl::htrie_map<char, field> schema_additions;@@ -3252,11 +3260,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields}if(f.nested) {- // We should add only root nested fields into `nested_fields` data structure- if(alter_nested_fields.count(f.name) != 0) {- nested_fields.emplace(f.name, f);- }-+ nested_fields.emplace(f.name, f);nested_field_names.push_back(f.name);}@@ -3265,6 +3269,8 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fieldsindex->refresh_schemas(new_fields, {});+ field::compact_nested_fields(nested_fields);+// Now, we can index existing data onto the updated schemaconst std::string seq_id_prefix = get_seq_id_collection_prefix();std::string upper_bound_key = get_seq_id_collection_prefix() + "`"; // cannot inline this@@ -3293,7 +3299,8 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields}if(enable_nested_fields) {- field::flatten_stored_doc(document, nested_fields);+ std::vector<field> flattened_fields;+ field::flatten_doc(document, nested_fields, true, flattened_fields);}index_record record(num_found_docs, seq_id, document, index_operation_t::CREATE, DIRTY_VALUES::REJECT);@@ -3344,6 +3351,10 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fieldsdynamic_fields.erase(del_field.name);}+ if(del_field.nested) {+ nested_fields.erase(del_field.name);+ }+if(del_field.name == ".*") {fallback_field_type = "";}@@ -3370,12 +3381,11 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {std::vector<field> del_fields;std::vector<field> addition_fields;std::vector<field> reindex_fields;- tsl::htrie_map<char, field> new_nested_fields;std::string this_fallback_field_type;auto validate_op = validate_alter_payload(alter_payload, addition_fields, reindex_fields,- new_nested_fields, del_fields, this_fallback_field_type);+ del_fields, this_fallback_field_type);if(!validate_op.ok()) {return validate_op;}@@ -3393,14 +3403,14 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {LOG(INFO) << "Processing field additions and deletions first...";}- auto batch_alter_op = batch_alter_data(addition_fields, del_fields, fallback_field_type, new_nested_fields);+ auto batch_alter_op = batch_alter_data(addition_fields, del_fields, fallback_field_type);if(!batch_alter_op.ok()) {return batch_alter_op;}if(!reindex_fields.empty()) {LOG(INFO) << "Processing field modifications now...";- batch_alter_op = batch_alter_data(reindex_fields, {}, fallback_field_type, new_nested_fields);+ batch_alter_op = batch_alter_data(reindex_fields, {}, fallback_field_type);if(!batch_alter_op.ok()) {return batch_alter_op;}@@ -3502,7 +3512,6 @@ void Collection::prune_doc(nlohmann::json& doc,Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,std::vector<field>& addition_fields,std::vector<field>& reindex_fields,- tsl::htrie_map<char, field>& new_nested_fields,std::vector<field>& del_fields,std::string& fallback_field_type) {if(!schema_changes.is_object()) {@@ -3585,19 +3594,15 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,updated_search_schema.erase(field_it.key());updated_nested_fields.erase(field_it.key());- // handle nested fields+ // should also remove children if the field being dropped is an objectif(field_it.value().nested && enable_nested_fields) {- std::vector<std::string> sub_fields;- auto ex_op = extract_field_name(field_name, search_schema, sub_fields, false, enable_nested_fields);- if(!ex_op.ok()) {- return ex_op;- }-- for(auto& sub_field: sub_fields) {- auto sub_field_it = search_schema.find(sub_field);- if(sub_field_it != search_schema.end()) {- del_fields.push_back(sub_field_it.value());- updated_search_schema.erase(sub_field);+ auto prefix_it = search_schema.equal_prefix_range(field_name);+ for(auto prefix_kv = prefix_it.first; prefix_kv != prefix_it.second; ++prefix_kv) {+ bool exact_key_match = (prefix_kv.key().size() == field_name.size());+ if(!exact_key_match) {+ del_fields.push_back(prefix_kv.value());+ updated_search_schema.erase(prefix_kv.key());+ updated_nested_fields.erase(prefix_kv.key());}}}@@ -3648,8 +3653,24 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,addition_fields.push_back(f);}- if(f.nested) {- new_nested_fields.emplace(f.name, f);+ if(f.nested && enable_nested_fields) {+ updated_nested_fields.emplace(f.name, f);++ // should also add children if the field is an object+ auto prefix_it = search_schema.equal_prefix_range(field_name);+ for(auto prefix_kv = prefix_it.first; prefix_kv != prefix_it.second; ++prefix_kv) {+ bool exact_key_match = (prefix_kv.key().size() == field_name.size());+ if(!exact_key_match) {+ updated_search_schema.emplace(prefix_kv.key(), prefix_kv.value());+ updated_nested_fields.emplace(prefix_kv.key(), prefix_kv.value());++ if(is_reindex) {+ reindex_fields.push_back(prefix_kv.value());+ } else {+ addition_fields.push_back(prefix_kv.value());+ }+ }+ }}} else {@@ -3687,28 +3708,26 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,nlohmann::detail::error_handler_t::ignore));}- if(enable_nested_fields) {- field::flatten_stored_doc(document, updated_nested_fields);- }-- if(!fallback_field_type.empty() || !new_dynamic_fields.empty() || !new_nested_fields.empty()) {+ if(!fallback_field_type.empty() || !new_dynamic_fields.empty() || !updated_nested_fields.empty()) {std::vector<field> new_fields;- std::vector<field> nested_fields_found;-Option<bool> new_fields_op = detect_new_fields(document, DIRTY_VALUES::DROP,updated_search_schema, new_dynamic_fields,- new_nested_fields,+ updated_nested_fields,fallback_field_type, false,- new_fields, nested_fields_found,+ new_fields,enable_nested_fields);if(!new_fields_op.ok()) {return new_fields_op;}for(auto& new_field: new_fields) {- updated_search_schema[new_field.name] = new_field;- // NOTE: cannot be pushed to `addition_fields` to support drop+re-addition use case- reindex_fields.push_back(new_field);+ if(updated_search_schema.find(new_field.name) == updated_search_schema.end()) {+ reindex_fields.push_back(new_field);+ updated_search_schema[new_field.name] = new_field;+ if(new_field.nested) {+ updated_nested_fields[new_field.name] = new_field;+ }+ }}}@@ -3774,8 +3793,8 @@ Option<bool> Collection::resolve_field_type(field& new_field,const DIRTY_VALUES& dirty_values,const bool found_dynamic_field,const std::string& fallback_field_type,- std::vector<field>& new_fields,- std::vector<field>& nested_fields_found) {+ const bool enable_nested_fields,+ std::vector<field>& new_fields) {if(!new_field.index) {return Option<bool>(true);}@@ -3845,9 +3864,8 @@ Option<bool> Collection::resolve_field_type(field& new_field,new_field.sort = true;}- if(new_field.nested) {- nested_fields_found.emplace_back(new_field);- } else {+ if(enable_nested_fields || !new_field.nested) {+ // only detect nested field if it is enabled explicitlynew_fields.emplace_back(new_field);}@@ -3858,19 +3876,12 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,const DIRTY_VALUES& dirty_values,const tsl::htrie_map<char, field>& schema,const std::unordered_map<std::string, field>& dyn_fields,- const tsl::htrie_map<char, field>& nested_fields,+ tsl::htrie_map<char, field>& nested_fields,const std::string& fallback_field_type,bool is_update,std::vector<field>& new_fields,- std::vector<field>& nested_fields_found,const bool enable_nested_fields) {- if(enable_nested_fields) {- for(auto& nested_field: nested_fields) {- nested_fields_found.push_back(nested_field);- }- }-auto kv = document.begin();while(kv != document.end()) {// we will not index the special "id" key@@ -3917,7 +3928,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,}auto add_op = resolve_field_type(new_field, kv, document, dirty_values, found_dynamic_field,- fallback_field_type, new_fields, nested_fields_found);+ fallback_field_type, enable_nested_fields, new_fields);if(!add_op.ok()) {return add_op;}@@ -3932,31 +3943,13 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,}if(enable_nested_fields) {- return flatten_and_identify_new_fields(document, nested_fields_found, schema, is_update, new_fields);- }-- return Option<bool>(true);-}--Option<bool> Collection::flatten_and_identify_new_fields(nlohmann::json& doc,- const std::vector<field>& nested_fields_found,- const tsl::htrie_map<char, field>& schema,- bool missing_is_ok,- std::vector<field>& new_fields) {- if(nested_fields_found.empty()) {- return Option<bool>(true);- }-- std::vector<field> flattened_fields;- auto flatten_op = field::flatten_doc(doc, nested_fields_found, missing_is_ok, flattened_fields);- if(!flatten_op.ok()) {- return flatten_op;- }-- for(const auto& flattened_field: flattened_fields) {- if(schema.count(flattened_field.name) == 0) {- new_fields.push_back(flattened_field);+ for(auto& new_field: new_fields) {+ if(new_field.nested) {+ nested_fields.emplace(new_field.name, new_field);+ }}++ return field::flatten_doc(document, nested_fields, is_update, new_fields);}return Option<bool>(true);@@ -3981,6 +3974,8 @@ Index* Collection::init_index() {}}+ field::compact_nested_fields(nested_fields);+synonym_index = new SynonymIndex(store);return new Index(name+std::to_string(0),diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 68c6a6c2..bb16f799 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1239,7 +1239,8 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection}if(collection->get_enable_nested_fields()) {- field::flatten_stored_doc(document, collection->get_nested_fields());+ std::vector<field> flattened_fields;+ field::flatten_doc(document, collection->get_nested_fields(), true, flattened_fields);}auto dirty_values = DIRTY_VALUES::DROP;diff --git a/src/field.cpp b/src/field.cppindex a5fbf1ad..fb334dc4 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -798,7 +798,7 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, cons}Option<bool> field::flatten_doc(nlohmann::json& document,- const std::vector<field>& nested_fields,+ const tsl::htrie_map<char, field>& nested_fields,bool missing_is_ok, std::vector<field>& flattened_fields) {std::unordered_map<std::string, field> flattened_fields_map;@@ -833,27 +833,13 @@ Option<bool> field::flatten_doc(nlohmann::json& document,return Option<bool>(true);}-Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields) {- std::unordered_map<std::string, field> flattened_fields_map;- for(const auto& nested_field: nested_fields) {- std::vector<std::string> field_parts;- StringUtils::split(nested_field.name, field_parts, ".");-- if(field_parts.size() > 1 && document.count(nested_field.name) != 0) {- // skip explicitly present nested fields- continue;- }-- flatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields_map);+void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {+ std::vector<std::string> nested_fields_vec;+ for(const auto& f: nested_fields) {+ nested_fields_vec.push_back(f.name);}- if(document.count(".flat") == 0) {- document[".flat"] = nlohmann::json::array();+ for(auto& field_name: nested_fields_vec) {+ nested_fields.erase_prefix(field_name + ".");}-- for(auto& kv: flattened_fields_map) {- document[".flat"].push_back(kv.second.name);- }-- return Option<bool>(true);}diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex ddce2f44..e3f7f961 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1109,11 +1109,12 @@ TEST_F(CollectionAllFieldsTest, WildcardFieldAndDictionaryField) {ASSERT_EQ(1, results["hits"].size());auto schema = coll1->get_fields();- ASSERT_EQ(4, schema.size());+ ASSERT_EQ(5, schema.size());ASSERT_EQ(".*", schema[0].name);- ASSERT_EQ("year", schema[1].name);- ASSERT_EQ("kinds.ZBXX", schema[2].name);- ASSERT_EQ("kinds.CGXX", schema[3].name);+ ASSERT_EQ("kinds", schema[1].name);+ ASSERT_EQ("year", schema[2].name);+ ASSERT_EQ("kinds.ZBXX", schema[3].name);+ ASSERT_EQ("kinds.CGXX", schema[4].name);// filter on object keyresults = coll1->search("*", {}, "kinds.CGXX: 13", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex 267957e1..bbaf42f5 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -35,6 +35,15 @@ protected:}};+tsl::htrie_map<char, field> get_nested_map(const std::vector<field>& nested_fields) {+ tsl::htrie_map<char, field> map;+ for(const auto& f: nested_fields) {+ map.emplace(f.name, f);+ }++ return map;+}+TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {auto json_str = R"({"company": {"name": "nike"},@@ -56,7 +65,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {// array of objectsstd::vector<field> flattened_fields;nlohmann::json doc = nlohmann::json::parse(json_str);- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());ASSERT_EQ(5, flattened_fields.size());for(const auto& f: flattened_fields) {@@ -99,7 +108,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {field("company", field_types::OBJECT, false)};- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());expected_json = R"({@@ -126,14 +135,14 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {field("locations.address", field_types::OBJECT, false)};- ASSERT_FALSE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok()); // must be of type object_array+ ASSERT_FALSE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok()); // must be of type object_arraynested_fields = {field("locations.address", field_types::OBJECT_ARRAY, false)};flattened_fields.clear();- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());expected_json = R"({@@ -165,7 +174,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {field("company.name", field_types::STRING, false)};- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());expected_json = R"({@@ -216,7 +225,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {// array of objectsstd::vector<field> flattened_fields;nlohmann::json doc = nlohmann::json::parse(json_str);- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());ASSERT_EQ(5, flattened_fields.size());for(const auto& f: flattened_fields) {@@ -232,7 +241,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {field("employees", field_types::OBJECT, false)};- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());ASSERT_EQ(5, flattened_fields.size());for(const auto& f: flattened_fields) {@@ -252,16 +261,20 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {field("employees.detail.tags", field_types::STRING_ARRAY, false),};- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());ASSERT_EQ(3, flattened_fields.size());+ std::sort(flattened_fields.begin(), flattened_fields.end(), [](field& a, field& b) {+ return a.name < b.name;+ });+ASSERT_EQ("employees.detail.tags",flattened_fields[0].name);ASSERT_FALSE(flattened_fields[0].nested_array);- ASSERT_EQ("employees.details.tags",flattened_fields[1].name);+ ASSERT_EQ("employees.details.num_tags",flattened_fields[1].name);ASSERT_TRUE(flattened_fields[1].nested_array);- ASSERT_EQ("employees.details.num_tags",flattened_fields[2].name);+ ASSERT_EQ("employees.details.tags",flattened_fields[2].name);ASSERT_TRUE(flattened_fields[2].nested_array);}@@ -277,7 +290,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {std::vector<field> flattened_fields;nlohmann::json doc = nlohmann::json::parse(json_str);- auto flatten_op = field::flatten_doc(doc, nested_fields, false, flattened_fields);+ auto flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields);ASSERT_FALSE(flatten_op.ok());ASSERT_EQ("Field `locations` not found.", flatten_op.error());@@ -286,7 +299,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {};flattened_fields.clear();- flatten_op = field::flatten_doc(doc, nested_fields, false, flattened_fields);+ flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields);ASSERT_FALSE(flatten_op.ok());ASSERT_EQ("Field `company` has an incorrect type.", flatten_op.error());}@@ -303,7 +316,8 @@ TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {schema.emplace("details.name", field("details.name", field_types::STRING_ARRAY, false));schema.emplace("details.year", field("details.year", field_types::INT32_ARRAY, false));- field::flatten_stored_doc(stored_doc, schema);+ std::vector<field> flattened_fields;+ field::flatten_doc(stored_doc, schema, true, flattened_fields);ASSERT_EQ(3, stored_doc[".flat"].size());ASSERT_EQ(7, stored_doc.size());@@ -313,6 +327,51 @@ TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {ASSERT_EQ(1, stored_doc.count("details.year"));}+TEST_F(CollectionNestedFieldsTest, CompactNestedFields) {+ auto stored_doc = R"({+ "company_name": "Acme Corp",+ "display_address": {+ "city": "LA",+ "street": "Lumbard St"+ },+ "id": "314",+ "location_addresses": [+ {+ "city": "Columbus",+ "street": "Yale St"+ },+ {+ "city": "Soda Springs",+ "street": "5th St"+ }+ ],+ "num_employees": 10,+ "primary_address": {+ "city": "Los Angeles",+ "street": "123 Lumbard St"+ }+ })"_json;++ tsl::htrie_map<char, field> schema;+ schema.emplace("location_addresses.city", field("location_addresses.city", field_types::STRING_ARRAY, true));+ schema.emplace("location_addresses", field("location_addresses", field_types::OBJECT_ARRAY, true));+ schema.emplace("primary_address", field("primary_address", field_types::OBJECT, true));+ schema.emplace("primary_address.city", field("primary_address.city", field_types::STRING, true));+ schema.emplace("location_addresses.street", field("location_addresses.street", field_types::STRING_ARRAY, true));+ schema.emplace("primary_address.street", field("primary_address.street", field_types::STRING, true));++ field::compact_nested_fields(schema);+ ASSERT_EQ(2, schema.size());+ ASSERT_EQ(1, schema.count("primary_address"));+ ASSERT_EQ(1, schema.count("location_addresses"));++ std::vector<field> flattened_fields;+ field::flatten_doc(stored_doc, schema, true, flattened_fields);++ ASSERT_EQ(2, stored_doc["location_addresses.city"].size());+ ASSERT_EQ(2, stored_doc["location_addresses.street"].size());+}+TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {std::vector<field> fields = {field(".*", field_types::AUTO, false, true)};@@ -1465,7 +1524,7 @@ TEST_F(CollectionNestedFieldsTest, ExplicitDotSeparatedFieldsShouldHavePrecendenASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());auto fs = coll1->get_fields();- ASSERT_EQ(4, coll1->get_fields().size());+ ASSERT_EQ(6, coll1->get_fields().size());// simple nested objectauto results = coll1->search("*", {}, "company.num_employees: 2000", {}, sort_fields, {0}, 10, 1,@@ -1852,6 +1911,68 @@ TEST_F(CollectionNestedFieldsTest, NestedSchemaAutoAndFacet) {ASSERT_TRUE(coll1->get_schema()["schools.name"].optional);}+TEST_F(CollectionNestedFieldsTest, ArrayOfObjectsFaceting) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "location_addresses", "type": "object[]", "facet": true}+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc1 = R"({+ "id": "0",+ "company_name": "Acme Corp",+ "display_address": {+ "city": "LA",+ "street": "Lumbard St"+ },+ "location_addresses": [+ {+ "city": "Columbus",+ "street": "Yale St"+ },+ {+ "city": "Soda Springs",+ "street": "5th St"+ }+ ],+ "num_employees": 10,+ "primary_address": {+ "city": "Los Angeles",+ "street": "123 Lumbard St"+ }+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ auto results = coll1->search("*", {}, "", {"location_addresses.city"}, {},+ {0}, 10, 1, FREQUENCY, {false}).get();++ // add same doc again+ doc1["id"] = "1";+ add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("*", {}, "", {"location_addresses.city"}, {},+ {0}, 10, 1, FREQUENCY, {false}).get();++ // facet count should be 2+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());++ ASSERT_EQ("Columbus", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());++ ASSERT_EQ("Soda Springs", results["facet_counts"][0]["counts"][1]["value"].get<std::string>());+ ASSERT_EQ(2, results["facet_counts"][0]["counts"][1]["count"].get<size_t>());+}+TEST_F(CollectionNestedFieldsTest, HighlightArrayInsideArrayOfObj) {nlohmann::json schema = R"({"name": "coll1",diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex de46e3df..91c379bd 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -1227,6 +1227,7 @@ TEST_F(CollectionSchemaChangeTest, NestedFieldExplicitSchemaDropping) {ASSERT_EQ(4, fields.size());ASSERT_EQ(4, schema_map.size());+ ASSERT_EQ(2, coll1->get_nested_fields().size());// drop object field@@ -1244,6 +1245,7 @@ TEST_F(CollectionSchemaChangeTest, NestedFieldExplicitSchemaDropping) {ASSERT_EQ(2, fields.size());ASSERT_EQ(2, schema_map.size());+ ASSERT_EQ(1, coll1->get_nested_fields().size());// drop primitive nested field@@ -1261,6 +1263,7 @@ TEST_F(CollectionSchemaChangeTest, NestedFieldExplicitSchemaDropping) {ASSERT_EQ(1, fields.size());ASSERT_EQ(1, schema_map.size());+ ASSERT_EQ(0, coll1->get_nested_fields().size());}TEST_F(CollectionSchemaChangeTest, NestedFieldSchemaAdditions) {@@ -1406,4 +1409,4 @@ TEST_F(CollectionSchemaChangeTest, DropAndReAddNestedObject) {ASSERT_EQ(4, fields.size());ASSERT_EQ(4, schema_map.size());-}\ No newline at end of file+}