Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix duplication of nested fields during flattening.

Kishore Nallancbf5985307b9
diff --git a/include/collection.h b/include/collection.h
index fd5c80b1..73d10449 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -161,11 +161,10 @@ private:
const DIRTY_VALUES& dirty_values,
const tsl::htrie_map<char, field>& schema,
const std::unordered_map<std::string, field>& dyn_fields,
- const tsl::htrie_map<char, field>& nested_fields,
+ tsl::htrie_map<char, field>& nested_fields,
const std::string& fallback_field_type,
bool is_update,
std::vector<field>& new_fields,
- std::vector<field>& nested_fields_found,
bool enable_nested_fields);
static bool facet_count_compare(const std::pair<uint64_t, facet_count_t>& a,
@@ -199,13 +198,11 @@ private:
Option<bool> batch_alter_data(const std::vector<field>& alter_fields,
const std::vector<field>& del_fields,
- const std::string& this_fallback_field_type,
- const tsl::htrie_map<char, field>& alter_nested_fields);
+ const std::string& this_fallback_field_type);
Option<bool> validate_alter_payload(nlohmann::json& schema_changes,
std::vector<field>& addition_fields,
std::vector<field>& reindex_fields,
- tsl::htrie_map<char, field>& new_nested_fields,
std::vector<field>& del_fields,
std::string& fallback_field_type);
@@ -237,11 +234,6 @@ private:
bool extract_only_string_fields,
bool enable_nested_fields);
- static Option<bool> flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields_found,
- const tsl::htrie_map<char, field>& schema,
- bool missing_is_ok,
- std::vector<field>& new_fields);
-
bool is_nested_array(const nlohmann::json& obj, std::vector<std::string> path_parts, size_t part_i) const;
template<class T>
@@ -255,8 +247,8 @@ private:
const DIRTY_VALUES& dirty_values,
const bool found_dynamic_field,
const std::string& fallback_field_type,
- std::vector<field>& new_fields,
- std::vector<field>& nested_fields_found);
+ bool enable_nested_fields,
+ std::vector<field>& new_fields);
public:
diff --git a/include/field.h b/include/field.h
index 49ef7336..b6057e1b 100644
--- a/include/field.h
+++ b/include/field.h
@@ -424,10 +424,10 @@ struct field {
std::vector<std::string>& path_parts, size_t path_index, bool has_array,
bool has_obj_array, std::unordered_map<std::string, field>& flattened_fields);
- static Option<bool> flatten_doc(nlohmann::json& document, const std::vector<field>& nested_fields,
+ static Option<bool> flatten_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields,
bool missing_is_ok, std::vector<field>& flattened_fields);
- static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields);
+ static void compact_nested_fields(tsl::htrie_map<char, field>& nested_fields);
};
struct filter_node_t;
diff --git a/src/collection.cpp b/src/collection.cpp
index 1f1cee5a..3d3de33a 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -257,7 +257,6 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
// if `fallback_field_type` or `dynamic_fields` is enabled, update schema first before indexing
if(!fallback_field_type.empty() || !dynamic_fields.empty() || !nested_fields.empty()) {
std::vector<field> new_fields;
- std::vector<field> nested_fields_found;
std::unique_lock lock(mutex);
Option<bool> new_fields_op = detect_new_fields(record.doc, dirty_values,
@@ -265,27 +264,32 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
nested_fields,
fallback_field_type,
record.is_update,
- new_fields, nested_fields_found,
+ new_fields,
enable_nested_fields);
if(!new_fields_op.ok()) {
record.index_failure(new_fields_op.code(), new_fields_op.error());
}
else if(!new_fields.empty()) {
+ bool found_new_field = false;
for(auto& new_field: new_fields) {
- search_schema.emplace(new_field.name, new_field);
- fields.emplace_back(new_field);
- }
-
- for(auto& nested_field: nested_fields_found) {
- nested_fields.emplace(nested_field.name, nested_field);
+ if(search_schema.find(new_field.name) == search_schema.end()) {
+ found_new_field = true;
+ search_schema.emplace(new_field.name, new_field);
+ fields.emplace_back(new_field);
+ if(new_field.nested) {
+ nested_fields.emplace(new_field.name, new_field);
+ }
+ }
}
- auto persist_op = persist_collection_meta();
- if(!persist_op.ok()) {
- record.index_failure(persist_op.code(), persist_op.error());
- } else {
- index->refresh_schemas(new_fields, {});
+ if(found_new_field) {
+ auto persist_op = persist_collection_meta();
+ if(!persist_op.ok()) {
+ record.index_failure(persist_op.code(), persist_op.error());
+ } else {
+ index->refresh_schemas(new_fields, {});
+ }
}
}
}
@@ -347,7 +351,7 @@ void Collection::batch_index(std::vector<index_record>& index_records, std::vect
}
} else {
- // remove flattened field values before storing on disk (.flat meta will be kept)
+ // remove flattened field values before storing on disk
remove_flat_fields(index_record.doc);
const std::string& seq_id_str = std::to_string(index_record.seq_id);
@@ -3110,8 +3114,10 @@ Option<bool> Collection::get_document_from_store(const std::string &seq_id_key,
}
if(!raw_doc && enable_nested_fields) {
- field::flatten_stored_doc(document, nested_fields);
+ std::vector<field> flattened_fields;
+ field::flatten_doc(document, nested_fields, true, flattened_fields);
}
+
return Option<bool>(true);
}
@@ -3193,6 +3199,9 @@ SynonymIndex* Collection::get_synonym_index() {
}
Option<bool> Collection::persist_collection_meta() {
+ // first compact nested fields (to keep only parents of expanded children)
+ field::compact_nested_fields(nested_fields);
+
std::string coll_meta_json;
StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);
@@ -3229,8 +3238,7 @@ Option<bool> Collection::persist_collection_meta() {
Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields,
const std::vector<field>& del_fields,
- const std::string& this_fallback_field_type,
- const tsl::htrie_map<char, field>& alter_nested_fields) {
+ const std::string& this_fallback_field_type) {
// Update schema with additions (deletions can only be made later)
std::vector<field> new_fields;
tsl::htrie_map<char, field> schema_additions;
@@ -3252,11 +3260,7 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
}
if(f.nested) {
- // We should add only root nested fields into `nested_fields` data structure
- if(alter_nested_fields.count(f.name) != 0) {
- nested_fields.emplace(f.name, f);
- }
-
+ nested_fields.emplace(f.name, f);
nested_field_names.push_back(f.name);
}
@@ -3265,6 +3269,8 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
index->refresh_schemas(new_fields, {});
+ field::compact_nested_fields(nested_fields);
+
// Now, we can index existing data onto the updated schema
const std::string seq_id_prefix = get_seq_id_collection_prefix();
std::string upper_bound_key = get_seq_id_collection_prefix() + "`"; // cannot inline this
@@ -3293,7 +3299,8 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
}
if(enable_nested_fields) {
- field::flatten_stored_doc(document, nested_fields);
+ std::vector<field> flattened_fields;
+ field::flatten_doc(document, nested_fields, true, flattened_fields);
}
index_record record(num_found_docs, seq_id, document, index_operation_t::CREATE, DIRTY_VALUES::REJECT);
@@ -3344,6 +3351,10 @@ Option<bool> Collection::batch_alter_data(const std::vector<field>& alter_fields
dynamic_fields.erase(del_field.name);
}
+ if(del_field.nested) {
+ nested_fields.erase(del_field.name);
+ }
+
if(del_field.name == ".*") {
fallback_field_type = "";
}
@@ -3370,12 +3381,11 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {
std::vector<field> del_fields;
std::vector<field> addition_fields;
std::vector<field> reindex_fields;
- tsl::htrie_map<char, field> new_nested_fields;
std::string this_fallback_field_type;
auto validate_op = validate_alter_payload(alter_payload, addition_fields, reindex_fields,
- new_nested_fields, del_fields, this_fallback_field_type);
+ del_fields, this_fallback_field_type);
if(!validate_op.ok()) {
return validate_op;
}
@@ -3393,14 +3403,14 @@ Option<bool> Collection::alter(nlohmann::json& alter_payload) {
LOG(INFO) << "Processing field additions and deletions first...";
}
- auto batch_alter_op = batch_alter_data(addition_fields, del_fields, fallback_field_type, new_nested_fields);
+ auto batch_alter_op = batch_alter_data(addition_fields, del_fields, fallback_field_type);
if(!batch_alter_op.ok()) {
return batch_alter_op;
}
if(!reindex_fields.empty()) {
LOG(INFO) << "Processing field modifications now...";
- batch_alter_op = batch_alter_data(reindex_fields, {}, fallback_field_type, new_nested_fields);
+ batch_alter_op = batch_alter_data(reindex_fields, {}, fallback_field_type);
if(!batch_alter_op.ok()) {
return batch_alter_op;
}
@@ -3502,7 +3512,6 @@ void Collection::prune_doc(nlohmann::json& doc,
Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
std::vector<field>& addition_fields,
std::vector<field>& reindex_fields,
- tsl::htrie_map<char, field>& new_nested_fields,
std::vector<field>& del_fields,
std::string& fallback_field_type) {
if(!schema_changes.is_object()) {
@@ -3585,19 +3594,15 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
updated_search_schema.erase(field_it.key());
updated_nested_fields.erase(field_it.key());
- // handle nested fields
+ // should also remove children if the field being dropped is an object
if(field_it.value().nested && enable_nested_fields) {
- std::vector<std::string> sub_fields;
- auto ex_op = extract_field_name(field_name, search_schema, sub_fields, false, enable_nested_fields);
- if(!ex_op.ok()) {
- return ex_op;
- }
-
- for(auto& sub_field: sub_fields) {
- auto sub_field_it = search_schema.find(sub_field);
- if(sub_field_it != search_schema.end()) {
- del_fields.push_back(sub_field_it.value());
- updated_search_schema.erase(sub_field);
+ auto prefix_it = search_schema.equal_prefix_range(field_name);
+ for(auto prefix_kv = prefix_it.first; prefix_kv != prefix_it.second; ++prefix_kv) {
+ bool exact_key_match = (prefix_kv.key().size() == field_name.size());
+ if(!exact_key_match) {
+ del_fields.push_back(prefix_kv.value());
+ updated_search_schema.erase(prefix_kv.key());
+ updated_nested_fields.erase(prefix_kv.key());
}
}
}
@@ -3648,8 +3653,24 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
addition_fields.push_back(f);
}
- if(f.nested) {
- new_nested_fields.emplace(f.name, f);
+ if(f.nested && enable_nested_fields) {
+ updated_nested_fields.emplace(f.name, f);
+
+ // should also add children if the field is an object
+ auto prefix_it = search_schema.equal_prefix_range(field_name);
+ for(auto prefix_kv = prefix_it.first; prefix_kv != prefix_it.second; ++prefix_kv) {
+ bool exact_key_match = (prefix_kv.key().size() == field_name.size());
+ if(!exact_key_match) {
+ updated_search_schema.emplace(prefix_kv.key(), prefix_kv.value());
+ updated_nested_fields.emplace(prefix_kv.key(), prefix_kv.value());
+
+ if(is_reindex) {
+ reindex_fields.push_back(prefix_kv.value());
+ } else {
+ addition_fields.push_back(prefix_kv.value());
+ }
+ }
+ }
}
} else {
@@ -3687,28 +3708,26 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
nlohmann::detail::error_handler_t::ignore));
}
- if(enable_nested_fields) {
- field::flatten_stored_doc(document, updated_nested_fields);
- }
-
- if(!fallback_field_type.empty() || !new_dynamic_fields.empty() || !new_nested_fields.empty()) {
+ if(!fallback_field_type.empty() || !new_dynamic_fields.empty() || !updated_nested_fields.empty()) {
std::vector<field> new_fields;
- std::vector<field> nested_fields_found;
-
Option<bool> new_fields_op = detect_new_fields(document, DIRTY_VALUES::DROP,
updated_search_schema, new_dynamic_fields,
- new_nested_fields,
+ updated_nested_fields,
fallback_field_type, false,
- new_fields, nested_fields_found,
+ new_fields,
enable_nested_fields);
if(!new_fields_op.ok()) {
return new_fields_op;
}
for(auto& new_field: new_fields) {
- updated_search_schema[new_field.name] = new_field;
- // NOTE: cannot be pushed to `addition_fields` to support drop+re-addition use case
- reindex_fields.push_back(new_field);
+ if(updated_search_schema.find(new_field.name) == updated_search_schema.end()) {
+ reindex_fields.push_back(new_field);
+ updated_search_schema[new_field.name] = new_field;
+ if(new_field.nested) {
+ updated_nested_fields[new_field.name] = new_field;
+ }
+ }
}
}
@@ -3774,8 +3793,8 @@ Option<bool> Collection::resolve_field_type(field& new_field,
const DIRTY_VALUES& dirty_values,
const bool found_dynamic_field,
const std::string& fallback_field_type,
- std::vector<field>& new_fields,
- std::vector<field>& nested_fields_found) {
+ const bool enable_nested_fields,
+ std::vector<field>& new_fields) {
if(!new_field.index) {
return Option<bool>(true);
}
@@ -3845,9 +3864,8 @@ Option<bool> Collection::resolve_field_type(field& new_field,
new_field.sort = true;
}
- if(new_field.nested) {
- nested_fields_found.emplace_back(new_field);
- } else {
+ if(enable_nested_fields || !new_field.nested) {
+ // only detect nested field if it is enabled explicitly
new_fields.emplace_back(new_field);
}
@@ -3858,19 +3876,12 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
const DIRTY_VALUES& dirty_values,
const tsl::htrie_map<char, field>& schema,
const std::unordered_map<std::string, field>& dyn_fields,
- const tsl::htrie_map<char, field>& nested_fields,
+ tsl::htrie_map<char, field>& nested_fields,
const std::string& fallback_field_type,
bool is_update,
std::vector<field>& new_fields,
- std::vector<field>& nested_fields_found,
const bool enable_nested_fields) {
- if(enable_nested_fields) {
- for(auto& nested_field: nested_fields) {
- nested_fields_found.push_back(nested_field);
- }
- }
-
auto kv = document.begin();
while(kv != document.end()) {
// we will not index the special "id" key
@@ -3917,7 +3928,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
}
auto add_op = resolve_field_type(new_field, kv, document, dirty_values, found_dynamic_field,
- fallback_field_type, new_fields, nested_fields_found);
+ fallback_field_type, enable_nested_fields, new_fields);
if(!add_op.ok()) {
return add_op;
}
@@ -3932,31 +3943,13 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
}
if(enable_nested_fields) {
- return flatten_and_identify_new_fields(document, nested_fields_found, schema, is_update, new_fields);
- }
-
- return Option<bool>(true);
-}
-
-Option<bool> Collection::flatten_and_identify_new_fields(nlohmann::json& doc,
- const std::vector<field>& nested_fields_found,
- const tsl::htrie_map<char, field>& schema,
- bool missing_is_ok,
- std::vector<field>& new_fields) {
- if(nested_fields_found.empty()) {
- return Option<bool>(true);
- }
-
- std::vector<field> flattened_fields;
- auto flatten_op = field::flatten_doc(doc, nested_fields_found, missing_is_ok, flattened_fields);
- if(!flatten_op.ok()) {
- return flatten_op;
- }
-
- for(const auto& flattened_field: flattened_fields) {
- if(schema.count(flattened_field.name) == 0) {
- new_fields.push_back(flattened_field);
+ for(auto& new_field: new_fields) {
+ if(new_field.nested) {
+ nested_fields.emplace(new_field.name, new_field);
+ }
}
+
+ return field::flatten_doc(document, nested_fields, is_update, new_fields);
}
return Option<bool>(true);
@@ -3981,6 +3974,8 @@ Index* Collection::init_index() {
}
}
+ field::compact_nested_fields(nested_fields);
+
synonym_index = new SynonymIndex(store);
return new Index(name+std::to_string(0),
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 68c6a6c2..bb16f799 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1239,7 +1239,8 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection
}
if(collection->get_enable_nested_fields()) {
- field::flatten_stored_doc(document, collection->get_nested_fields());
+ std::vector<field> flattened_fields;
+ field::flatten_doc(document, collection->get_nested_fields(), true, flattened_fields);
}
auto dirty_values = DIRTY_VALUES::DROP;
diff --git a/src/field.cpp b/src/field.cpp
index a5fbf1ad..fb334dc4 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -798,7 +798,7 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, cons
}
Option<bool> field::flatten_doc(nlohmann::json& document,
- const std::vector<field>& nested_fields,
+ const tsl::htrie_map<char, field>& nested_fields,
bool missing_is_ok, std::vector<field>& flattened_fields) {
std::unordered_map<std::string, field> flattened_fields_map;
@@ -833,27 +833,13 @@ Option<bool> field::flatten_doc(nlohmann::json& document,
return Option<bool>(true);
}
-Option<bool> field::flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields) {
- std::unordered_map<std::string, field> flattened_fields_map;
- for(const auto& nested_field: nested_fields) {
- std::vector<std::string> field_parts;
- StringUtils::split(nested_field.name, field_parts, ".");
-
- if(field_parts.size() > 1 && document.count(nested_field.name) != 0) {
- // skip explicitly present nested fields
- continue;
- }
-
- flatten_field(document, document, nested_field, field_parts, 0, false, false, flattened_fields_map);
+void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {
+ std::vector<std::string> nested_fields_vec;
+ for(const auto& f: nested_fields) {
+ nested_fields_vec.push_back(f.name);
}
- if(document.count(".flat") == 0) {
- document[".flat"] = nlohmann::json::array();
+ for(auto& field_name: nested_fields_vec) {
+ nested_fields.erase_prefix(field_name + ".");
}
-
- for(auto& kv: flattened_fields_map) {
- document[".flat"].push_back(kv.second.name);
- }
-
- return Option<bool>(true);
}
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index ddce2f44..e3f7f961 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -1109,11 +1109,12 @@ TEST_F(CollectionAllFieldsTest, WildcardFieldAndDictionaryField) {
ASSERT_EQ(1, results["hits"].size());
auto schema = coll1->get_fields();
- ASSERT_EQ(4, schema.size());
+ ASSERT_EQ(5, schema.size());
ASSERT_EQ(".*", schema[0].name);
- ASSERT_EQ("year", schema[1].name);
- ASSERT_EQ("kinds.ZBXX", schema[2].name);
- ASSERT_EQ("kinds.CGXX", schema[3].name);
+ ASSERT_EQ("kinds", schema[1].name);
+ ASSERT_EQ("year", schema[2].name);
+ ASSERT_EQ("kinds.ZBXX", schema[3].name);
+ ASSERT_EQ("kinds.CGXX", schema[4].name);
// filter on object key
results = coll1->search("*", {}, "kinds.CGXX: 13", {}, sort_fields, {0}, 10, 1, FREQUENCY, {false}).get();
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index 267957e1..bbaf42f5 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -35,6 +35,15 @@ protected:
}
};
+tsl::htrie_map<char, field> get_nested_map(const std::vector<field>& nested_fields) {
+ tsl::htrie_map<char, field> map;
+ for(const auto& f: nested_fields) {
+ map.emplace(f.name, f);
+ }
+
+ return map;
+}
+
TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
auto json_str = R"({
"company": {"name": "nike"},
@@ -56,7 +65,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
// array of objects
std::vector<field> flattened_fields;
nlohmann::json doc = nlohmann::json::parse(json_str);
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
ASSERT_EQ(5, flattened_fields.size());
for(const auto& f: flattened_fields) {
@@ -99,7 +108,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
field("company", field_types::OBJECT, false)
};
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
expected_json = R"(
{
@@ -126,14 +135,14 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
field("locations.address", field_types::OBJECT, false)
};
- ASSERT_FALSE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok()); // must be of type object_array
+ ASSERT_FALSE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok()); // must be of type object_array
nested_fields = {
field("locations.address", field_types::OBJECT_ARRAY, false)
};
flattened_fields.clear();
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
expected_json = R"(
{
@@ -165,7 +174,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
field("company.name", field_types::STRING, false)
};
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
expected_json = R"(
{
@@ -216,7 +225,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {
// array of objects
std::vector<field> flattened_fields;
nlohmann::json doc = nlohmann::json::parse(json_str);
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
ASSERT_EQ(5, flattened_fields.size());
for(const auto& f: flattened_fields) {
@@ -232,7 +241,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {
field("employees", field_types::OBJECT, false)
};
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
ASSERT_EQ(5, flattened_fields.size());
for(const auto& f: flattened_fields) {
@@ -252,16 +261,20 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {
field("employees.detail.tags", field_types::STRING_ARRAY, false),
};
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields).ok());
ASSERT_EQ(3, flattened_fields.size());
+ std::sort(flattened_fields.begin(), flattened_fields.end(), [](field& a, field& b) {
+ return a.name < b.name;
+ });
+
ASSERT_EQ("employees.detail.tags",flattened_fields[0].name);
ASSERT_FALSE(flattened_fields[0].nested_array);
- ASSERT_EQ("employees.details.tags",flattened_fields[1].name);
+ ASSERT_EQ("employees.details.num_tags",flattened_fields[1].name);
ASSERT_TRUE(flattened_fields[1].nested_array);
- ASSERT_EQ("employees.details.num_tags",flattened_fields[2].name);
+ ASSERT_EQ("employees.details.tags",flattened_fields[2].name);
ASSERT_TRUE(flattened_fields[2].nested_array);
}
@@ -277,7 +290,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {
std::vector<field> flattened_fields;
nlohmann::json doc = nlohmann::json::parse(json_str);
- auto flatten_op = field::flatten_doc(doc, nested_fields, false, flattened_fields);
+ auto flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields);
ASSERT_FALSE(flatten_op.ok());
ASSERT_EQ("Field `locations` not found.", flatten_op.error());
@@ -286,7 +299,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {
};
flattened_fields.clear();
- flatten_op = field::flatten_doc(doc, nested_fields, false, flattened_fields);
+ flatten_op = field::flatten_doc(doc, get_nested_map(nested_fields), false, flattened_fields);
ASSERT_FALSE(flatten_op.ok());
ASSERT_EQ("Field `company` has an incorrect type.", flatten_op.error());
}
@@ -303,7 +316,8 @@ TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {
schema.emplace("details.name", field("details.name", field_types::STRING_ARRAY, false));
schema.emplace("details.year", field("details.year", field_types::INT32_ARRAY, false));
- field::flatten_stored_doc(stored_doc, schema);
+ std::vector<field> flattened_fields;
+ field::flatten_doc(stored_doc, schema, true, flattened_fields);
ASSERT_EQ(3, stored_doc[".flat"].size());
ASSERT_EQ(7, stored_doc.size());
@@ -313,6 +327,51 @@ TEST_F(CollectionNestedFieldsTest, FlattenStoredDoc) {
ASSERT_EQ(1, stored_doc.count("details.year"));
}
+TEST_F(CollectionNestedFieldsTest, CompactNestedFields) {
+ auto stored_doc = R"({
+ "company_name": "Acme Corp",
+ "display_address": {
+ "city": "LA",
+ "street": "Lumbard St"
+ },
+ "id": "314",
+ "location_addresses": [
+ {
+ "city": "Columbus",
+ "street": "Yale St"
+ },
+ {
+ "city": "Soda Springs",
+ "street": "5th St"
+ }
+ ],
+ "num_employees": 10,
+ "primary_address": {
+ "city": "Los Angeles",
+ "street": "123 Lumbard St"
+ }
+ })"_json;
+
+ tsl::htrie_map<char, field> schema;
+ schema.emplace("location_addresses.city", field("location_addresses.city", field_types::STRING_ARRAY, true));
+ schema.emplace("location_addresses", field("location_addresses", field_types::OBJECT_ARRAY, true));
+ schema.emplace("primary_address", field("primary_address", field_types::OBJECT, true));
+ schema.emplace("primary_address.city", field("primary_address.city", field_types::STRING, true));
+ schema.emplace("location_addresses.street", field("location_addresses.street", field_types::STRING_ARRAY, true));
+ schema.emplace("primary_address.street", field("primary_address.street", field_types::STRING, true));
+
+ field::compact_nested_fields(schema);
+ ASSERT_EQ(2, schema.size());
+ ASSERT_EQ(1, schema.count("primary_address"));
+ ASSERT_EQ(1, schema.count("location_addresses"));
+
+ std::vector<field> flattened_fields;
+ field::flatten_doc(stored_doc, schema, true, flattened_fields);
+
+ ASSERT_EQ(2, stored_doc["location_addresses.city"].size());
+ ASSERT_EQ(2, stored_doc["location_addresses.street"].size());
+}
+
TEST_F(CollectionNestedFieldsTest, SearchOnFieldsOnWildcardSchema) {
std::vector<field> fields = {field(".*", field_types::AUTO, false, true)};
@@ -1465,7 +1524,7 @@ TEST_F(CollectionNestedFieldsTest, ExplicitDotSeparatedFieldsShouldHavePrecenden
ASSERT_TRUE(coll1->add(doc1.dump(), CREATE).ok());
auto fs = coll1->get_fields();
- ASSERT_EQ(4, coll1->get_fields().size());
+ ASSERT_EQ(6, coll1->get_fields().size());
// simple nested object
auto results = coll1->search("*", {}, "company.num_employees: 2000", {}, sort_fields, {0}, 10, 1,
@@ -1852,6 +1911,68 @@ TEST_F(CollectionNestedFieldsTest, NestedSchemaAutoAndFacet) {
ASSERT_TRUE(coll1->get_schema()["schools.name"].optional);
}
+TEST_F(CollectionNestedFieldsTest, ArrayOfObjectsFaceting) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "location_addresses", "type": "object[]", "facet": true}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ auto doc1 = R"({
+ "id": "0",
+ "company_name": "Acme Corp",
+ "display_address": {
+ "city": "LA",
+ "street": "Lumbard St"
+ },
+ "location_addresses": [
+ {
+ "city": "Columbus",
+ "street": "Yale St"
+ },
+ {
+ "city": "Soda Springs",
+ "street": "5th St"
+ }
+ ],
+ "num_employees": 10,
+ "primary_address": {
+ "city": "Los Angeles",
+ "street": "123 Lumbard St"
+ }
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto results = coll1->search("*", {}, "", {"location_addresses.city"}, {},
+ {0}, 10, 1, FREQUENCY, {false}).get();
+
+ // add same doc again
+ doc1["id"] = "1";
+ add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll1->search("*", {}, "", {"location_addresses.city"}, {},
+ {0}, 10, 1, FREQUENCY, {false}).get();
+
+ // facet count should be 2
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"].size());
+
+ ASSERT_EQ("Columbus", results["facet_counts"][0]["counts"][0]["value"].get<std::string>());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"][0]["count"].get<size_t>());
+
+ ASSERT_EQ("Soda Springs", results["facet_counts"][0]["counts"][1]["value"].get<std::string>());
+ ASSERT_EQ(2, results["facet_counts"][0]["counts"][1]["count"].get<size_t>());
+}
+
TEST_F(CollectionNestedFieldsTest, HighlightArrayInsideArrayOfObj) {
nlohmann::json schema = R"({
"name": "coll1",
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index de46e3df..91c379bd 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -1227,6 +1227,7 @@ TEST_F(CollectionSchemaChangeTest, NestedFieldExplicitSchemaDropping) {
ASSERT_EQ(4, fields.size());
ASSERT_EQ(4, schema_map.size());
+ ASSERT_EQ(2, coll1->get_nested_fields().size());
// drop object field
@@ -1244,6 +1245,7 @@ TEST_F(CollectionSchemaChangeTest, NestedFieldExplicitSchemaDropping) {
ASSERT_EQ(2, fields.size());
ASSERT_EQ(2, schema_map.size());
+ ASSERT_EQ(1, coll1->get_nested_fields().size());
// drop primitive nested field
@@ -1261,6 +1263,7 @@ TEST_F(CollectionSchemaChangeTest, NestedFieldExplicitSchemaDropping) {
ASSERT_EQ(1, fields.size());
ASSERT_EQ(1, schema_map.size());
+ ASSERT_EQ(0, coll1->get_nested_fields().size());
}
TEST_F(CollectionSchemaChangeTest, NestedFieldSchemaAdditions) {
@@ -1406,4 +1409,4 @@ TEST_F(CollectionSchemaChangeTest, DropAndReAddNestedObject) {
ASSERT_EQ(4, fields.size());
ASSERT_EQ(4, schema_map.size());
-}
\ No newline at end of file
+}