Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle missing optional object fields.

Kishore Nallan4e7c5560aa63
diff --git a/include/collection.h b/include/collection.h
index 6eda783a..fd5c80b1 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -163,6 +163,7 @@ private:
const std::unordered_map<std::string, field>& dyn_fields,
const tsl::htrie_map<char, field>& nested_fields,
const std::string& fallback_field_type,
+ bool is_update,
std::vector<field>& new_fields,
std::vector<field>& nested_fields_found,
bool enable_nested_fields);
@@ -238,6 +239,7 @@ private:
static Option<bool> flatten_and_identify_new_fields(nlohmann::json& doc, const std::vector<field>& nested_fields_found,
const tsl::htrie_map<char, field>& schema,
+ bool missing_is_ok,
std::vector<field>& new_fields);
bool is_nested_array(const nlohmann::json& obj, std::vector<std::string> path_parts, size_t part_i) const;
diff --git a/include/field.h b/include/field.h
index 8dc8ac55..49ef7336 100644
--- a/include/field.h
+++ b/include/field.h
@@ -425,7 +425,7 @@ struct field {
bool has_obj_array, std::unordered_map<std::string, field>& flattened_fields);
static Option<bool> flatten_doc(nlohmann::json& document, const std::vector<field>& nested_fields,
- std::vector<field>& flattened_fields);
+ bool missing_is_ok, std::vector<field>& flattened_fields);
static Option<bool> flatten_stored_doc(nlohmann::json& document, const tsl::htrie_map<char, field>& nested_fields);
};
diff --git a/src/collection.cpp b/src/collection.cpp
index d39a8e0e..1f1cee5a 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -264,6 +264,7 @@ nlohmann::json Collection::add_many(std::vector<std::string>& json_lines, nlohma
search_schema, dynamic_fields,
nested_fields,
fallback_field_type,
+ record.is_update,
new_fields, nested_fields_found,
enable_nested_fields);
if(!new_fields_op.ok()) {
@@ -3697,7 +3698,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
Option<bool> new_fields_op = detect_new_fields(document, DIRTY_VALUES::DROP,
updated_search_schema, new_dynamic_fields,
new_nested_fields,
- fallback_field_type,
+ fallback_field_type, false,
new_fields, nested_fields_found,
enable_nested_fields);
if(!new_fields_op.ok()) {
@@ -3859,6 +3860,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
const std::unordered_map<std::string, field>& dyn_fields,
const tsl::htrie_map<char, field>& nested_fields,
const std::string& fallback_field_type,
+ bool is_update,
std::vector<field>& new_fields,
std::vector<field>& nested_fields_found,
const bool enable_nested_fields) {
@@ -3930,7 +3932,7 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
}
if(enable_nested_fields) {
- return flatten_and_identify_new_fields(document, nested_fields_found, schema, new_fields);
+ return flatten_and_identify_new_fields(document, nested_fields_found, schema, is_update, new_fields);
}
return Option<bool>(true);
@@ -3939,13 +3941,14 @@ Option<bool> Collection::detect_new_fields(nlohmann::json& document,
Option<bool> Collection::flatten_and_identify_new_fields(nlohmann::json& doc,
const std::vector<field>& nested_fields_found,
const tsl::htrie_map<char, field>& schema,
+ bool missing_is_ok,
std::vector<field>& new_fields) {
if(nested_fields_found.empty()) {
return Option<bool>(true);
}
std::vector<field> flattened_fields;
- auto flatten_op = field::flatten_doc(doc, nested_fields_found, flattened_fields);
+ auto flatten_op = field::flatten_doc(doc, nested_fields_found, missing_is_ok, flattened_fields);
if(!flatten_op.ok()) {
return flatten_op;
}
diff --git a/src/field.cpp b/src/field.cpp
index 1f42243b..a5fbf1ad 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -799,7 +799,7 @@ Option<bool> field::flatten_field(nlohmann::json& doc, nlohmann::json& obj, cons
Option<bool> field::flatten_doc(nlohmann::json& document,
const std::vector<field>& nested_fields,
- std::vector<field>& flattened_fields) {
+ bool missing_is_ok, std::vector<field>& flattened_fields) {
std::unordered_map<std::string, field> flattened_fields_map;
@@ -817,7 +817,7 @@ Option<bool> field::flatten_doc(nlohmann::json& document,
continue;
}
- if(op.code() == 404 && nested_field.optional) {
+ if(op.code() == 404 && (missing_is_ok || nested_field.optional)) {
continue;
} else {
return op;
diff --git a/test/collection_nested_fields_test.cpp b/test/collection_nested_fields_test.cpp
index c14b4670..267957e1 100644
--- a/test/collection_nested_fields_test.cpp
+++ b/test/collection_nested_fields_test.cpp
@@ -56,7 +56,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
// array of objects
std::vector<field> flattened_fields;
nlohmann::json doc = nlohmann::json::parse(json_str);
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
ASSERT_EQ(5, flattened_fields.size());
for(const auto& f: flattened_fields) {
@@ -99,7 +99,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
field("company", field_types::OBJECT, false)
};
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
expected_json = R"(
{
@@ -126,14 +126,14 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
field("locations.address", field_types::OBJECT, false)
};
- ASSERT_FALSE(field::flatten_doc(doc, nested_fields, flattened_fields).ok()); // must be of type object_array
+ ASSERT_FALSE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok()); // must be of type object_array
nested_fields = {
field("locations.address", field_types::OBJECT_ARRAY, false)
};
flattened_fields.clear();
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
expected_json = R"(
{
@@ -165,7 +165,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObject) {
field("company.name", field_types::STRING, false)
};
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
expected_json = R"(
{
@@ -216,7 +216,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {
// array of objects
std::vector<field> flattened_fields;
nlohmann::json doc = nlohmann::json::parse(json_str);
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
ASSERT_EQ(5, flattened_fields.size());
for(const auto& f: flattened_fields) {
@@ -232,7 +232,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {
field("employees", field_types::OBJECT, false)
};
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
ASSERT_EQ(5, flattened_fields.size());
for(const auto& f: flattened_fields) {
@@ -252,7 +252,7 @@ TEST_F(CollectionNestedFieldsTest, TestNestedArrayField) {
field("employees.detail.tags", field_types::STRING_ARRAY, false),
};
- ASSERT_TRUE(field::flatten_doc(doc, nested_fields, flattened_fields).ok());
+ ASSERT_TRUE(field::flatten_doc(doc, nested_fields, false, flattened_fields).ok());
ASSERT_EQ(3, flattened_fields.size());
ASSERT_EQ("employees.detail.tags",flattened_fields[0].name);
@@ -277,7 +277,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {
std::vector<field> flattened_fields;
nlohmann::json doc = nlohmann::json::parse(json_str);
- auto flatten_op = field::flatten_doc(doc, nested_fields, flattened_fields);
+ auto flatten_op = field::flatten_doc(doc, nested_fields, false, flattened_fields);
ASSERT_FALSE(flatten_op.ok());
ASSERT_EQ("Field `locations` not found.", flatten_op.error());
@@ -286,7 +286,7 @@ TEST_F(CollectionNestedFieldsTest, FlattenJSONObjectHandleErrors) {
};
flattened_fields.clear();
- flatten_op = field::flatten_doc(doc, nested_fields, flattened_fields);
+ flatten_op = field::flatten_doc(doc, nested_fields, false, flattened_fields);
ASSERT_FALSE(flatten_op.ok());
ASSERT_EQ("Field `company` has an incorrect type.", flatten_op.error());
}
@@ -1946,3 +1946,70 @@ TEST_F(CollectionNestedFieldsTest, ErrorWhenObjectTypeUsedWithoutEnablingNestedF
ASSERT_EQ("Type `object` or `object[]` can be used only when nested fields are enabled by setting` "
"enable_nested_fields` to true.", op.error());
}
+
+TEST_F(CollectionNestedFieldsTest, UpdateNestedDocument) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "contributors", "type": "object", "optional": false},
+ {"name": "title", "type": "string", "optional": false}
+ ]
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ auto doc1 = R"({
+ "id": "0",
+ "title": "Title Alpha",
+ "contributors": {"first_name": "John", "last_name": "Galt"}
+ })"_json;
+
+ auto add_op = coll1->add(doc1.dump(), CREATE);
+ ASSERT_TRUE(add_op.ok());
+
+ // update document partially
+
+ doc1 = R"({
+ "id": "0",
+ "title": "Title Beta"
+ })"_json;
+
+ add_op = coll1->add(doc1.dump(), UPDATE);
+ ASSERT_TRUE(add_op.ok());
+
+ auto results = coll1->search("beta", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ // emplace document partially
+
+ doc1 = R"({
+ "id": "0",
+ "title": "Title Gamma"
+ })"_json;
+
+ add_op = coll1->add(doc1.dump(), EMPLACE);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll1->search("gamma", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ // update a sub-field of an object
+ doc1 = R"({
+ "id": "0",
+ "contributors": {"last_name": "Shaw"}
+ })"_json;
+
+ add_op = coll1->add(doc1.dump(), UPDATE);
+ ASSERT_TRUE(add_op.ok());
+
+ results = coll1->search("shaw", {"contributors"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["found"].get<size_t>());
+
+ // should not be able to find the old name
+
+ results = coll1->search("galt", {"contributors"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(0, results["found"].get<size_t>());
+}