Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow `null` reference value for an optional field. (#1601)

Harpreet Sangare07cdfb4aa01
diff --git a/src/collection.cpp b/src/collection.cpp
index a962007a..dd2e0ed1 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -86,8 +86,13 @@ uint32_t Collection::get_next_seq_id() {
}
Option<bool> single_value_filter_query(nlohmann::json& document, const std::string& field_name,
- const std::string& ref_field_type, std::string& filter_query) {
+ const std::string& ref_field_type, bool is_optional, std::string& filter_query) {
auto const& value = document[field_name];
+
+ if (is_optional && value.is_null()) {
+ return Option<bool>(422, "Optional field has `null` value.");
+ }
+
if (value.is_string() && ref_field_type == field_types::STRING) {
filter_query[filter_query.size() - 1] = '=';
filter_query += (" " + value.get<std::string>());
@@ -140,14 +145,17 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
}
auto const reference_helper_field = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;
- document[fields::reference_helper_fields] += reference_helper_field;
if (reference_field_name == "id") {
auto id_field_type_error_op = Option<bool>(400, "Field `" + field_name + "` must have string value.");
if (document[field_name].is_array()) {
document[reference_helper_field] = nlohmann::json::array();
+ document[fields::reference_helper_fields] += reference_helper_field;
+
for (const auto &item: document[field_name].items()) {
- if (!item.value().is_string()) {
+ if (optional && item.value().is_null()) {
+ continue;
+ } else if (!item.value().is_string()) {
return id_field_type_error_op;
}
@@ -171,6 +179,9 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
}
document[reference_helper_field] = ref_doc_id_op.get();
+ document[fields::reference_helper_fields] += reference_helper_field;
+ } else if (optional && document[field_name].is_null()) {
+ continue;
} else {
return id_field_type_error_op;
}
@@ -221,6 +232,7 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
}
document[reference_helper_field] = nlohmann::json::array();
+ document[fields::reference_helper_fields] += reference_helper_field;
nlohmann::json temp_doc; // To store singular values of `field_name` field.
std::vector<std::string> keys;
@@ -236,8 +248,11 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
temp_doc[field_name] = object_array[i].at(keys[1]);
auto single_value_filter_query_op = single_value_filter_query(temp_doc, field_name, ref_field_type,
- filter_query);
+ optional, filter_query);
if (!single_value_filter_query_op.ok()) {
+ if (single_value_filter_query_op.code() == 422) {
+ continue;
+ }
return single_value_filter_query_op;
}
@@ -271,23 +286,34 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
} else {
filter_query += "[";
}
+ bool filter_values_added = false;
for (const auto &item: document[field_name].items()) {
auto const& item_value = item.value();
if (item_value.is_string() && ref_field_type == field_types::STRING) {
filter_query += item_value.get<std::string>();
+ filter_values_added = true;
} else if (item_value.is_number_integer() && (ref_field_type == field_types::INT64 ||
(ref_field_type == field_types::INT32 && StringUtils::is_int32_t(std::to_string(item_value.get<int64_t>()))))) {
filter_query += std::to_string(item_value.get<int64_t>());
+ filter_values_added = true;
+ } else if (optional && item_value.is_null()) {
+ continue;
} else {
return Option<bool>(400, "Field `" + field_name + "` must only have `" + ref_field_type + "` values.");
}
filter_query += ",";
}
+ if (!filter_values_added) {
+ continue;
+ }
filter_query[filter_query.size() - 1] = ']';
} else {
auto single_value_filter_query_op = single_value_filter_query(document, field_name, ref_field_type,
- filter_query);
+ optional, filter_query);
if (!single_value_filter_query_op.ok()) {
+ if (single_value_filter_query_op.code() == 422) {
+ continue;
+ }
return single_value_filter_query_op;
}
}
@@ -300,6 +326,8 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
if (document[field_name].is_array()) {
document[reference_helper_field] = nlohmann::json::array();
+ document[fields::reference_helper_fields] += reference_helper_field;
+
for (uint32_t i = 0; i < filter_result.count; i++) {
document[reference_helper_field] += filter_result.docs[i];
}
@@ -314,6 +342,7 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
}
document[reference_helper_field] = filter_result.docs[0];
+ document[fields::reference_helper_fields] += reference_helper_field;
}
}
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index 84d593cf..bdc9d22e 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -403,12 +403,33 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
ASSERT_EQ(0, doc["multi_id_reference_sequence_id"][0]);
ASSERT_EQ(1, doc["multi_id_reference_sequence_id"][1]);
+ id_ref_json = R"({
+ "id_reference": null
+ })"_json;
+ add_doc_op = id_ref_collection->add(id_ref_json.dump());
+ ASSERT_TRUE(add_doc_op.ok());
+
+ doc = id_ref_collection->get("3").get();
+ ASSERT_EQ(0, doc.count("id_reference_sequence_id"));
+ ASSERT_EQ(0, doc.count("multi_id_reference_sequence_id"));
+ ASSERT_EQ(1, doc.count(".ref"));
+ ASSERT_EQ(0, doc[".ref"].size());
+
+ id_ref_json = R"({
+ "multi_id_reference": [null]
+ })"_json;
+ add_doc_op = id_ref_collection->add(id_ref_json.dump());
+ ASSERT_FALSE(add_doc_op.ok());
+ ASSERT_EQ("Field `multi_id_reference` must be an array of string.", add_doc_op.error());
+
+ // Reference helper field is not returned in the search response.
auto result = id_ref_collection->search("*", {}, "", {}, {}, {0}).get();
- ASSERT_EQ(3, result["found"].get<size_t>());
- ASSERT_EQ(3, result["hits"].size());
- ASSERT_EQ(0, result["hits"][0]["document"].count("multi_id_reference_sequence_id"));
+ ASSERT_EQ(4, result["found"].get<size_t>());
+ ASSERT_EQ(4, result["hits"].size());
+ ASSERT_EQ(0, result["hits"][0]["document"].count("id_reference_sequence_id"));
ASSERT_EQ(0, result["hits"][1]["document"].count("multi_id_reference_sequence_id"));
- ASSERT_EQ(0, result["hits"][2]["document"].count("id_reference_sequence_id"));
+ ASSERT_EQ(0, result["hits"][2]["document"].count("multi_id_reference_sequence_id"));
+ ASSERT_EQ(0, result["hits"][3]["document"].count("id_reference_sequence_id"));
collectionManager.drop_collection("Customers");
collectionManager.drop_collection("Products");
@@ -500,6 +521,13 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
ASSERT_FALSE(add_doc_op.ok());
ASSERT_EQ("Field `ref_string_array_field` must only have `string` values.", add_doc_op.error());
+ doc_json = R"({
+ "ref_string_array_field": [null]
+ })"_json;
+ add_doc_op = coll2->add(doc_json.dump());
+ ASSERT_FALSE(add_doc_op.ok());
+ ASSERT_EQ("Field `ref_string_array_field` must be an array of string.", add_doc_op.error());
+
collectionManager.drop_collection("coll2");
temp_json = schema_json;
collection_create_op = collectionManager.create_collection(temp_json);
@@ -519,6 +547,17 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
ASSERT_EQ(1, doc[".ref"].size());
ASSERT_EQ("ref_string_field_sequence_id", doc[".ref"][0]);
+ doc_json = R"({
+ "ref_string_field": null
+ })"_json;
+ add_doc_op = coll2->add(doc_json.dump());
+ ASSERT_TRUE(add_doc_op.ok());
+
+ doc = coll2->get("1").get();
+ ASSERT_EQ(0, doc.count("ref_string_field_sequence_id"));
+ ASSERT_EQ(1, doc.count(".ref"));
+ ASSERT_EQ(0, doc[".ref"].size());
+
doc_json = R"({
"ref_string_array_field": ["foo"]
})"_json;
@@ -528,7 +567,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
result = coll2->search("*", {}, "", {}, {}, {0}).get();
ASSERT_EQ(0, result["hits"][0]["document"]["ref_string_array_field_sequence_id"].size());
- doc = coll2->get("1").get();
+ doc = coll2->get("2").get();
ASSERT_EQ(1, doc.count("ref_string_array_field_sequence_id"));
ASSERT_EQ(0, doc["ref_string_array_field_sequence_id"].size());
ASSERT_EQ(1, doc.count(".ref"));
@@ -541,7 +580,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
add_doc_op = coll2->add(doc_json.dump());
ASSERT_TRUE(add_doc_op.ok());
- doc = coll2->get("2").get();
+ doc = coll2->get("3").get();
ASSERT_EQ(1, doc.count("ref_string_array_field_sequence_id"));
ASSERT_EQ(1, doc["ref_string_array_field_sequence_id"].size());
ASSERT_EQ(0, doc["ref_string_array_field_sequence_id"][0]);
@@ -552,7 +591,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
add_doc_op = coll2->add(doc_json.dump());
ASSERT_TRUE(add_doc_op.ok());
- doc = coll2->get("3").get();
+ doc = coll2->get("4").get();
ASSERT_EQ(1, doc.count("ref_string_array_field_sequence_id"));
ASSERT_EQ(2, doc["ref_string_array_field_sequence_id"].size());
ASSERT_EQ(0, doc["ref_string_array_field_sequence_id"][0]);