Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor cascade deletion logic. (#1868)

Harpreet Sangarfc5f2e216659
diff --git a/src/collection.cpp b/src/collection.cpp
index d3e0b29d..afc97923 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -130,6 +130,8 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
// Add reference helper fields in the document.
for (auto const& pair: reference_fields) {
auto field_name = pair.first;
+ auto const reference_helper_field = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;
+
auto const& field = schema.at(field_name);
auto const& optional = field.optional;
// Strict checking for presence of non-optional reference field during indexing operation.
@@ -138,13 +140,12 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
return Option<bool>(400, "Missing the required reference field `" + field_name
+ "` in the document.");
} else if (document.count(field_name) != 1) {
+ if (is_update) {
+ document[fields::reference_helper_fields] += reference_helper_field;
+ }
continue;
}
- if (document.count(fields::reference_helper_fields) == 0) {
- document[fields::reference_helper_fields] = nlohmann::json::array();
- }
-
auto reference_pair = pair.second;
auto reference_collection_name = reference_pair.collection;
auto reference_field_name = reference_pair.field;
@@ -155,8 +156,6 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
+ "` not found.");
}
- auto const reference_helper_field = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;
-
bool is_object_reference_field = flat_fields.count(field_name) != 0;
std::string object_key;
bool is_object_array = false;
@@ -240,6 +239,10 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
document[reference_helper_field] = ref_doc_id_op.get();
document[fields::reference_helper_fields] += reference_helper_field;
} else if (optional && document[field_name].is_null()) {
+ // Reference helper field should also be removed along with reference field.
+ if (is_update) {
+ document[reference_helper_field] = nullptr;
+ }
continue;
} else {
return id_field_type_error_op;
@@ -365,6 +368,10 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c
filter_query);
if (!single_value_filter_query_op.ok()) {
if (optional && single_value_filter_query_op.code() == 422) {
+ // Reference helper field should also be removed along with reference field.
+ if (is_update) {
+ document[reference_helper_field] = nullptr;
+ }
continue;
}
return single_value_filter_query_op;
@@ -4548,7 +4555,7 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, c
return;
}
- bool is_field_singular;
+ bool is_field_singular, is_field_optional;
{
std::unique_lock lock(mutex);
@@ -4557,6 +4564,7 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, c
return;
}
is_field_singular = it.value().is_singular();
+ is_field_optional = it.value().optional;
}
if (is_field_singular) {
@@ -4564,8 +4572,8 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, c
for (uint32_t i = 0; i < filter_result.count; i++) {
auto const& seq_id = filter_result.docs[i];
- nlohmann::json document;
- auto get_doc_op = get_document_from_store(get_seq_id_key(seq_id), document);
+ nlohmann::json existing_document;
+ auto get_doc_op = get_document_from_store(get_seq_id_key(seq_id), existing_document);
if (!get_doc_op.ok()) {
if (get_doc_op.code() == 404) {
@@ -4577,7 +4585,22 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, c
continue;
}
- remove_document(document, seq_id, remove_from_store);
+ bool multiple_ref_fields = existing_document.contains(fields::reference_helper_fields) &&
+ existing_document[fields::reference_helper_fields].size() > 1;
+
+ // If there are other references present and the reference of an optional field is removed, don't delete the
+ // document.
+ if (multiple_ref_fields && is_field_optional) {
+ auto const id = existing_document["id"].get<std::string>();
+
+ nlohmann::json update_document;
+ update_document["id"] = id;
+ update_document[field_name] = nullptr;
+
+ add(update_document.dump(), index_operation_t::UPDATE, id);
+ } else {
+ remove_document(existing_document, seq_id, remove_from_store);
+ }
}
} else {
std::string ref_coll_name, ref_field_name;
@@ -4661,7 +4684,22 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, c
continue;
}
- remove_document(existing_document, seq_id, remove_from_store);
+ bool multiple_ref_fields = existing_document.contains(fields::reference_helper_fields) &&
+ existing_document[fields::reference_helper_fields].size() > 1;
+
+ // If there are other references present and the reference of an optional field is removed, don't delete the
+ // document.
+ if (multiple_ref_fields && is_field_optional) {
+ auto const id = existing_document["id"].get<std::string>();
+
+ nlohmann::json update_document;
+ update_document["id"] = id;
+ update_document[field_name] = nullptr;
+
+ buffer.push_back(update_document.dump());
+ } else {
+ remove_document(existing_document, seq_id, remove_from_store);
+ }
}
nlohmann::json dummy;
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index ec72c958..d2a98e4e 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -412,8 +412,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
doc = id_ref_collection->get("3").get();
ASSERT_EQ(0, doc.count("id_reference_sequence_id"));
ASSERT_EQ(0, doc.count("multi_id_reference_sequence_id"));
- ASSERT_EQ(1, doc.count(".ref"));
- ASSERT_EQ(0, doc[".ref"].size());
+ ASSERT_EQ(0, doc.count(".ref"));
id_ref_json = R"({
"multi_id_reference": [null]
@@ -555,8 +554,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {
doc = coll2->get("1").get();
ASSERT_EQ(0, doc.count("ref_string_field_sequence_id"));
- ASSERT_EQ(1, doc.count(".ref"));
- ASSERT_EQ(0, doc[".ref"].size());
+ ASSERT_EQ(0, doc.count(".ref"));
doc_json = R"({
"ref_string_array_field": ["foo"]
@@ -4798,6 +4796,242 @@ TEST_F(CollectionJoinTest, CascadeDeletion) {
ASSERT_EQ(2, res_obj["hits"][0]["document"].at("split_members").size());
ASSERT_EQ("user_a", res_obj["hits"][0]["document"]["split_members"][0].at("user_id"));
ASSERT_EQ("user_c", res_obj["hits"][0]["document"]["split_members"][1].at("user_id"));
+
+ collectionManager.drop_collection("Users");
+ schema_json =
+ R"({
+ "name": "Users",
+ "fields": [
+ {"name": "user_id", "type": "string"},
+ {"name": "user_name", "type": "string"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "user_id": "user_a",
+ "user_name": "Roshan"
+ })"_json,
+ R"({
+ "user_id": "user_b",
+ "user_name": "Ruby"
+ })"_json,
+ R"({
+ "user_id": "user_c",
+ "user_name": "Joe"
+ })"_json,
+ R"({
+ "user_id": "user_d",
+ "user_name": "Aby"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Repos",
+ "fields": [
+ {"name": "repo_id", "type": "string"},
+ {"name": "repo_content", "type": "string"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "repo_id": "repo_a",
+ "repo_content": "body1"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "repo_content": "body2"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "repo_content": "body3"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "Links",
+ "fields": [
+ {"name": "repo_id", "type": "string", "reference": "Repos.repo_id", "optional": true},
+ {"name": "user_id", "type": "string", "reference": "Users.user_id"}
+ ]
+ })"_json;
+ documents = {
+ R"({
+ "repo_id": "repo_a",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_a",
+ "user_id": "user_c"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_a"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_d"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_a"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_b"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_c"
+ })"_json,
+ R"({
+ "repo_id": "repo_c",
+ "user_id": "user_d"
+ })"_json
+ };
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ auto links_collection = collectionManager.get_collection_unsafe("Links");
+ auto links_doc = links_collection->get("0").get();
+ ASSERT_EQ(1, links_doc.count(".ref"));
+ ASSERT_EQ(2, links_doc[".ref"].size());
+ ASSERT_EQ("user_id_sequence_id", links_doc[".ref"][0]);
+ ASSERT_EQ("repo_id_sequence_id", links_doc[".ref"][1]);
+ ASSERT_EQ(1, links_doc.count("user_id_sequence_id"));
+ ASSERT_EQ(1, links_doc["user_id_sequence_id"]);
+ ASSERT_EQ(1, links_doc.count("repo_id_sequence_id"));
+ ASSERT_EQ(0, links_doc["repo_id_sequence_id"]);
+
+ links_doc = links_collection->get("1").get();
+ ASSERT_EQ(1, links_doc.count(".ref"));
+ ASSERT_EQ(2, links_doc[".ref"].size());
+ ASSERT_EQ("user_id_sequence_id", links_doc[".ref"][0]);
+ ASSERT_EQ("repo_id_sequence_id", links_doc[".ref"][1]);
+ ASSERT_EQ(1, links_doc.count("user_id_sequence_id"));
+ ASSERT_EQ(2, links_doc["user_id_sequence_id"]);
+ ASSERT_EQ(1, links_doc.count("repo_id_sequence_id"));
+ ASSERT_EQ(0, links_doc["repo_id_sequence_id"]);
+
+ collectionManager.get_collection_unsafe("Repos")->remove("0");
+
+ // Only optional reference to repos was deleted, so the document will not be deleted.
+ links_collection = collectionManager.get_collection_unsafe("Links");
+ links_doc = links_collection->get("0").get();
+ ASSERT_EQ(1, links_doc.count(".ref"));
+ ASSERT_EQ(1, links_doc[".ref"].size());
+ ASSERT_EQ("user_id_sequence_id", links_doc[".ref"][0]);
+ ASSERT_EQ(1, links_doc.count("user_id_sequence_id"));
+ ASSERT_EQ(1, links_doc["user_id_sequence_id"]);
+ ASSERT_EQ(0, links_doc.count("repo_id_sequence_id"));
+
+ links_doc = links_collection->get("1").get();
+ ASSERT_EQ(1, links_doc.count(".ref"));
+ ASSERT_EQ(1, links_doc[".ref"].size());
+ ASSERT_EQ("user_id_sequence_id", links_doc[".ref"][0]);
+ ASSERT_EQ(1, links_doc.count("user_id_sequence_id"));
+ ASSERT_EQ(2, links_doc["user_id_sequence_id"]);
+ ASSERT_EQ(0, links_doc.count("repo_id_sequence_id"));
+
+ collectionManager.get_collection_unsafe("Users")->remove("2");
+
+ links_doc = links_collection->get("0").get();
+ ASSERT_EQ(1, links_doc.count(".ref"));
+ ASSERT_EQ(1, links_doc[".ref"].size());
+ ASSERT_EQ("user_id_sequence_id", links_doc[".ref"][0]);
+ ASSERT_EQ(1, links_doc.count("user_id_sequence_id"));
+ ASSERT_EQ(1, links_doc["user_id_sequence_id"]);
+
+ // Required reference to users was deleted, so the documents are removed.
+ auto get_op = links_collection->get("1");
+ ASSERT_FALSE(get_op.ok());
+ ASSERT_EQ("Could not find a document with id: 1", get_op.error());
+
+ get_op = links_collection->get("7");
+ ASSERT_FALSE(get_op.ok());
+ ASSERT_EQ("Could not find a document with id: 7", get_op.error());
+
+ schema_json =
+ R"({
+ "name": "Links_2",
+ "fields": [
+ {"name": "repo_id", "type": "string", "reference": "Repos.repo_id", "optional": true},
+ {"name": "user_id", "type": "string", "reference": "Users.user_id", "optional": true}
+ ]
+ })"_json;
+
+ documents = {
+ R"({
+ "repo_id": "repo_b",
+ "user_id": "user_b"
+ })"_json,
+ };
+
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ if (!add_op.ok()) {
+ LOG(INFO) << add_op.error();
+ }
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ auto links_2_collection = collectionManager.get_collection_unsafe("Links_2");
+ auto links_2_doc = links_2_collection->get("0").get();
+ ASSERT_EQ(1, links_2_doc.count(".ref"));
+ ASSERT_EQ(2, links_2_doc[".ref"].size());
+ ASSERT_EQ("user_id_sequence_id", links_2_doc[".ref"][0]);
+ ASSERT_EQ("repo_id_sequence_id", links_2_doc[".ref"][1]);
+ ASSERT_EQ(1, links_2_doc.count("user_id_sequence_id"));
+ ASSERT_EQ(1, links_2_doc["user_id_sequence_id"]);
+ ASSERT_EQ(1, links_2_doc.count("repo_id_sequence_id"));
+ ASSERT_EQ(1, links_2_doc["repo_id_sequence_id"]);
+
+ collectionManager.get_collection_unsafe("Users")->remove("1");
+
+ links_2_doc = links_2_collection->get("0").get();
+ ASSERT_EQ(1, links_2_doc.count(".ref"));
+ ASSERT_EQ(1, links_2_doc[".ref"].size());
+ ASSERT_EQ("repo_id_sequence_id", links_2_doc[".ref"][0]);
+ ASSERT_EQ(0, links_2_doc.count("user_id_sequence_id"));
+ ASSERT_EQ(1, links_2_doc.count("repo_id_sequence_id"));
+ ASSERT_EQ(1, links_2_doc["repo_id_sequence_id"]);
+
+ collectionManager.get_collection_unsafe("Repos")->remove("1");
+
+ // All references were deleted, so the document is removed.
+ get_op = links_2_collection->get("0");
+ ASSERT_FALSE(get_op.ok());
+ ASSERT_EQ("Could not find a document with id: 0", get_op.error());
}
TEST_F(CollectionJoinTest, SortByReference) {