Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle missing optional object fields.
Kishore Nallan4e7c5560aa63
diff --git a/include/collection.h b/include/collection.hindex 6eda783a..fd5c80b1 100644--- a/include/collection.h+++ b/include/collection.h@@ -163,6 +163,7 @@ private:const std::unordered_map<std::string, field>& dyn_fields,const tsl::htrie_map<char, field>& nested_fields,const std::string& fallback_field_type,+ bool is_update,std::vector<field>& new_fields,std::vector<field>& nested_fields_found,bool enable_nested_fields);@@ -238,6 +239,7 @@ private:static Option<bool> flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields_found,const tsl::htrie_map<char, field>& schema,+ bool missing_is_ok,std::vector<field>& new_fields);bool is_nested_array(const nlohmann::json& obj, std::vector<std::string> path_parts, size_t part_i) const;diff --git a/include/field.h b/include/field.hindex 8dc8ac55..49ef7336 100644--- a/include/field.h+++ b/include/field.h@@ -425,7 +425,7 @@ struct field {bool has_obj_array, std::unordered_map<std::string, field>& flattened_fields);static Option<bool> flatten_doc(nlohmann::json& document, const std::vector<field>& nested_fields,- std::vector<field>& flattened_fields);+ bool missing_is_ok, std::vector<field>& flattened_fields);static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields);};diff --git a/src/collection.cpp b/src/collection.cppindex d39a8e0e..1f1cee5a 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -264,6 +264,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohmasearch_schema, dynamic_fields,nested_fields,fallback_field_type,+ record.is_update,new_fields, nested_fields_found,enable_nested_fields);if(!new_fields_op.ok()) {@@ -3697,7 +3698,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,Option<bool> new_fields_op = detect_new_fields(document, DIRTY_VALUES::DROP,updated_search_schema, new_dynamic_fields,new_nested_fields,- fallback_field_type,+ fallback_field_type, false,new_fields, nested_fields_found,enable_nested_fields);if(!new_fields_op.ok()) {@@ -3859,6 +3860,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,const std::unordered_map<std::string, field>& dyn_fields,const tsl::htrie_map<char, field>& nested_fields,const std::string& fallback_field_type,+ bool is_update,std::vector<field>& new_fields,std::vector<field>& nested_fields_found,const bool enable_nested_fields) {@@ -3930,7 +3932,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,}if(enable_nested_fields) {- return flatten_and_identify_new_fields(document, nested_fields_found, schema, new_fields);+ return flatten_and_identify_new_fields(document, nested_fields_found, schema, is_update, new_fields);}return Option<bool>(true);@@ -3939,13 +3941,14 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,Option<bool> Collection::flatten_and_identify_new_fields(nlohmann::json& doc,const std::vector<field>& nested_fields_found,const tsl::htrie_map<char, field>& schema,+ bool missing_is_ok,std::vector<field>& new_fields) {if(nested_fields_found.empty()) {return Option<bool>(true);}std::vector<field> flattened_fields;- auto flatten_op = field::flatten_doc(doc, nested_fields_found, flattened_fields);+ auto flatten_op = field::flatten_doc(doc, nested_fields_found, missing_is_ok, flattened_fields);if(!flatten_op.ok()) {return flatten_op;}diff --git a/src/field.cpp b/src/field.cppindex 1f42243b..a5fbf1ad 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -799,7 +799,7 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, consOption<bool> field::flatten_doc(nlohmann::json& document,const std::vector<field>& nested_fields,- std::vector<field>& flattened_fields) {+ bool missing_is_ok, std::vector<field>& flattened_fields) {std::unordered_map<std::string, field> flattened_fields_map;@@ -817,7 +817,7 @@ Option<bool> field::flatten_doc(nlohmann::json& document,continue;}- if(op.code() == 404 && nested_field.optional) {+ if(op.code() == 404 && (missing_is_ok || nested_field.optional)) {continue;} else {return op;diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cppindex c14b4670..267957e1 100644--- a/test/collection_nested_fields_test.cpp+++ b/test/collection_nested_fields_test.cpp@@ -56,7 +56,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {// array of objectsstd::vector<field> flattened_fields;nlohmann::json doc = nlohmann::json::parse(json_str);- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());ASSERT_EQ(5, flattened_fields.size());for(const auto& f: flattened_fields) {@@ -99,7 +99,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {field("company", field_types::OBJECT, false)};- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());expected_json = R"({@@ -126,14 +126,14 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {field("locations.address", field_types::OBJECT, false)};- ASSERT_FALSE(field::flatten_doc(doc, nested_fields, flattened_fields).ok()); // must be of type object_array+ ASSERT_FALSE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok()); // must be of type object_arraynested_fields = {field("locations.address", field_types::OBJECT_ARRAY, false)};flattened_fields.clear();- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());expected_json = R"({@@ -165,7 +165,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {field("company.name", field_types::STRING, false)};- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());expected_json = R"({@@ -216,7 +216,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {// array of objectsstd::vector<field> flattened_fields;nlohmann::json doc = nlohmann::json::parse(json_str);- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());ASSERT_EQ(5, flattened_fields.size());for(const auto& f: flattened_fields) {@@ -232,7 +232,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {field("employees", field_types::OBJECT, false)};- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());ASSERT_EQ(5, flattened_fields.size());for(const auto& f: flattened_fields) {@@ -252,7 +252,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {field("employees.detail.tags", field_types::STRING_ARRAY, false),};- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());ASSERT_EQ(3, flattened_fields.size());ASSERT_EQ("employees.detail.tags",flattened_fields[0].name);@@ -277,7 +277,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {std::vector<field> flattened_fields;nlohmann::json doc = nlohmann::json::parse(json_str);- auto flatten_op = field::flatten_doc(doc, nested_fields, flattened_fields);+ auto flatten_op = field::flatten_doc(doc, nested_fields, false, flattened_fields);ASSERT_FALSE(flatten_op.ok());ASSERT_EQ("Field `locations` not found.", flatten_op.error());@@ -286,7 +286,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {};flattened_fields.clear();- flatten_op = field::flatten_doc(doc, nested_fields, flattened_fields);+ flatten_op = field::flatten_doc(doc, nested_fields, false, flattened_fields);ASSERT_FALSE(flatten_op.ok());ASSERT_EQ("Field `company` has an incorrect type.", flatten_op.error());}@@ -1946,3 +1946,70 @@ TEST_F(CollectionNestedFieldsTest, ErrorWhenObjectTypeUsedWithoutEnablingNestedFASSERT_EQ("Type `object` or `object[]` can be used only when nested fields are enabled by setting` ""enable_nested_fields` to true.", op.error());}++TEST_F(CollectionNestedFieldsTest, UpdateNestedDocument) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "enable_nested_fields": true,+ "fields": [+ {"name": "contributors", "type": "object", "optional": false},+ {"name": "title", "type": "string", "optional": false}+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto doc1 = R"({+ "id": "0",+ "title": "Title Alpha",+ "contributors": {"first_name": "John", "last_name": "Galt"}+ })"_json;++ auto add_op = coll1->add(doc1.dump(), CREATE);+ ASSERT_TRUE(add_op.ok());++ // update document partially++ doc1 = R"({+ "id": "0",+ "title": "Title Beta"+ })"_json;++ add_op = coll1->add(doc1.dump(), UPDATE);+ ASSERT_TRUE(add_op.ok());++ auto results = coll1->search("beta", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["found"].get<size_t>());++ // emplace document partially++ doc1 = R"({+ "id": "0",+ "title": "Title Gamma"+ })"_json;++ add_op = coll1->add(doc1.dump(), EMPLACE);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("gamma", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["found"].get<size_t>());++ // update a sub-field of an object+ doc1 = R"({+ "id": "0",+ "contributors": {"last_name": "Shaw"}+ })"_json;++ add_op = coll1->add(doc1.dump(), UPDATE);+ ASSERT_TRUE(add_op.ok());++ results = coll1->search("shaw", {"contributors"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["found"].get<size_t>());++ // should not be able to find the old name++ results = coll1->search("galt", {"contributors"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(0, results["found"].get<size_t>());+}