Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor cascade deletion logic. (#1868)
Harpreet Sangarfc5f2e216659
diff --git a/src/collection.cpp b/src/collection.cppindex d3e0b29d..afc97923 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -130,6 +130,8 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c// Add reference helper fields in the document.for (auto const& pair: reference_fields) {auto field_name = pair.first;+ auto const reference_helper_field = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;+auto const& field = schema.at(field_name);auto const& optional = field.optional;// Strict checking for presence of non-optional reference field during indexing operation.@@ -138,13 +140,12 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, creturn Option<bool>(400, "Missing the required reference field `" + field_name+ "` in the document.");} else if (document.count(field_name) != 1) {+ if (is_update) {+ document[fields::reference_helper_fields] += reference_helper_field;+ }continue;}- if (document.count(fields::reference_helper_fields) == 0) {- document[fields::reference_helper_fields] = nlohmann::json::array();- }-auto reference_pair = pair.second;auto reference_collection_name = reference_pair.collection;auto reference_field_name = reference_pair.field;@@ -155,8 +156,6 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, c+ "` not found.");}- auto const reference_helper_field = field_name + fields::REFERENCE_HELPER_FIELD_SUFFIX;-bool is_object_reference_field = flat_fields.count(field_name) != 0;std::string object_key;bool is_object_array = false;@@ -240,6 +239,10 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, cdocument[reference_helper_field] = ref_doc_id_op.get();document[fields::reference_helper_fields] += reference_helper_field;} else if (optional && document[field_name].is_null()) {+ // Reference helper field should also be removed along with reference field.+ if (is_update) {+ document[reference_helper_field] = nullptr;+ }continue;} else {return id_field_type_error_op;@@ -365,6 +368,10 @@ Option<bool> Collection::add_reference_helper_fields(nlohmann::json& document, cfilter_query);if (!single_value_filter_query_op.ok()) {if (optional && single_value_filter_query_op.code() == 422) {+ // Reference helper field should also be removed along with reference field.+ if (is_update) {+ document[reference_helper_field] = nullptr;+ }continue;}return single_value_filter_query_op;@@ -4548,7 +4555,7 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, creturn;}- bool is_field_singular;+ bool is_field_singular, is_field_optional;{std::unique_lock lock(mutex);@@ -4557,6 +4564,7 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, creturn;}is_field_singular = it.value().is_singular();+ is_field_optional = it.value().optional;}if (is_field_singular) {@@ -4564,8 +4572,8 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, cfor (uint32_t i = 0; i < filter_result.count; i++) {auto const& seq_id = filter_result.docs[i];- nlohmann::json document;- auto get_doc_op = get_document_from_store(get_seq_id_key(seq_id), document);+ nlohmann::json existing_document;+ auto get_doc_op = get_document_from_store(get_seq_id_key(seq_id), existing_document);if (!get_doc_op.ok()) {if (get_doc_op.code() == 404) {@@ -4577,7 +4585,22 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, ccontinue;}- remove_document(document, seq_id, remove_from_store);+ bool multiple_ref_fields = existing_document.contains(fields::reference_helper_fields) &&+ existing_document[fields::reference_helper_fields].size() > 1;++ // If there are other references present and the reference of an optional field is removed, don't delete the+ // document.+ if (multiple_ref_fields && is_field_optional) {+ auto const id = existing_document["id"].get<std::string>();++ nlohmann::json update_document;+ update_document["id"] = id;+ update_document[field_name] = nullptr;++ add(update_document.dump(), index_operation_t::UPDATE, id);+ } else {+ remove_document(existing_document, seq_id, remove_from_store);+ }}} else {std::string ref_coll_name, ref_field_name;@@ -4661,7 +4684,22 @@ void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, ccontinue;}- remove_document(existing_document, seq_id, remove_from_store);+ bool multiple_ref_fields = existing_document.contains(fields::reference_helper_fields) &&+ existing_document[fields::reference_helper_fields].size() > 1;++ // If there are other references present and the reference of an optional field is removed, don't delete the+ // document.+ if (multiple_ref_fields && is_field_optional) {+ auto const id = existing_document["id"].get<std::string>();++ nlohmann::json update_document;+ update_document["id"] = id;+ update_document[field_name] = nullptr;++ buffer.push_back(update_document.dump());+ } else {+ remove_document(existing_document, seq_id, remove_from_store);+ }}nlohmann::json dummy;diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cppindex ec72c958..d2a98e4e 100644--- a/test/collection_join_test.cpp+++ b/test/collection_join_test.cpp@@ -412,8 +412,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {doc = id_ref_collection->get("3").get();ASSERT_EQ(0, doc.count("id_reference_sequence_id"));ASSERT_EQ(0, doc.count("multi_id_reference_sequence_id"));- ASSERT_EQ(1, doc.count(".ref"));- ASSERT_EQ(0, doc[".ref"].size());+ ASSERT_EQ(0, doc.count(".ref"));id_ref_json = R"({"multi_id_reference": [null]@@ -555,8 +554,7 @@ TEST_F(CollectionJoinTest, IndexDocumentHavingReferenceField) {doc = coll2->get("1").get();ASSERT_EQ(0, doc.count("ref_string_field_sequence_id"));- ASSERT_EQ(1, doc.count(".ref"));- ASSERT_EQ(0, doc[".ref"].size());+ ASSERT_EQ(0, doc.count(".ref"));doc_json = R"({"ref_string_array_field": ["foo"]@@ -4798,6 +4796,242 @@ TEST_F(CollectionJoinTest, CascadeDeletion) {ASSERT_EQ(2, res_obj["hits"][0]["document"].at("split_members").size());ASSERT_EQ("user_a", res_obj["hits"][0]["document"]["split_members"][0].at("user_id"));ASSERT_EQ("user_c", res_obj["hits"][0]["document"]["split_members"][1].at("user_id"));++ collectionManager.drop_collection("Users");+ schema_json =+ R"({+ "name": "Users",+ "fields": [+ {"name": "user_id", "type": "string"},+ {"name": "user_name", "type": "string"}+ ]+ })"_json;+ documents = {+ R"({+ "user_id": "user_a",+ "user_name": "Roshan"+ })"_json,+ R"({+ "user_id": "user_b",+ "user_name": "Ruby"+ })"_json,+ R"({+ "user_id": "user_c",+ "user_name": "Joe"+ })"_json,+ R"({+ "user_id": "user_d",+ "user_name": "Aby"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "Repos",+ "fields": [+ {"name": "repo_id", "type": "string"},+ {"name": "repo_content", "type": "string"}+ ]+ })"_json;+ documents = {+ R"({+ "repo_id": "repo_a",+ "repo_content": "body1"+ })"_json,+ R"({+ "repo_id": "repo_b",+ "repo_content": "body2"+ })"_json,+ R"({+ "repo_id": "repo_c",+ "repo_content": "body3"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ schema_json =+ R"({+ "name": "Links",+ "fields": [+ {"name": "repo_id", "type": "string", "reference": "Repos.repo_id", "optional": true},+ {"name": "user_id", "type": "string", "reference": "Users.user_id"}+ ]+ })"_json;+ documents = {+ R"({+ "repo_id": "repo_a",+ "user_id": "user_b"+ })"_json,+ R"({+ "repo_id": "repo_a",+ "user_id": "user_c"+ })"_json,+ R"({+ "repo_id": "repo_b",+ "user_id": "user_a"+ })"_json,+ R"({+ "repo_id": "repo_b",+ "user_id": "user_b"+ })"_json,+ R"({+ "repo_id": "repo_b",+ "user_id": "user_d"+ })"_json,+ R"({+ "repo_id": "repo_c",+ "user_id": "user_a"+ })"_json,+ R"({+ "repo_id": "repo_c",+ "user_id": "user_b"+ })"_json,+ R"({+ "repo_id": "repo_c",+ "user_id": "user_c"+ })"_json,+ R"({+ "repo_id": "repo_c",+ "user_id": "user_d"+ })"_json+ };+ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());++ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ auto links_collection = collectionManager.get_collection_unsafe("Links");+ auto links_doc = links_collection->get("0").get();+ ASSERT_EQ(1, links_doc.count(".ref"));+ ASSERT_EQ(2, links_doc[".ref"].size());+ ASSERT_EQ("user_id_sequence_id", links_doc[".ref"][0]);+ ASSERT_EQ("repo_id_sequence_id", links_doc[".ref"][1]);+ ASSERT_EQ(1, links_doc.count("user_id_sequence_id"));+ ASSERT_EQ(1, links_doc["user_id_sequence_id"]);+ ASSERT_EQ(1, links_doc.count("repo_id_sequence_id"));+ ASSERT_EQ(0, links_doc["repo_id_sequence_id"]);++ links_doc = links_collection->get("1").get();+ ASSERT_EQ(1, links_doc.count(".ref"));+ ASSERT_EQ(2, links_doc[".ref"].size());+ ASSERT_EQ("user_id_sequence_id", links_doc[".ref"][0]);+ ASSERT_EQ("repo_id_sequence_id", links_doc[".ref"][1]);+ ASSERT_EQ(1, links_doc.count("user_id_sequence_id"));+ ASSERT_EQ(2, links_doc["user_id_sequence_id"]);+ ASSERT_EQ(1, links_doc.count("repo_id_sequence_id"));+ ASSERT_EQ(0, links_doc["repo_id_sequence_id"]);++ collectionManager.get_collection_unsafe("Repos")->remove("0");++ // Only optional reference to repos was deleted, so the document will not be deleted.+ links_collection = collectionManager.get_collection_unsafe("Links");+ links_doc = links_collection->get("0").get();+ ASSERT_EQ(1, links_doc.count(".ref"));+ ASSERT_EQ(1, links_doc[".ref"].size());+ ASSERT_EQ("user_id_sequence_id", links_doc[".ref"][0]);+ ASSERT_EQ(1, links_doc.count("user_id_sequence_id"));+ ASSERT_EQ(1, links_doc["user_id_sequence_id"]);+ ASSERT_EQ(0, links_doc.count("repo_id_sequence_id"));++ links_doc = links_collection->get("1").get();+ ASSERT_EQ(1, links_doc.count(".ref"));+ ASSERT_EQ(1, links_doc[".ref"].size());+ ASSERT_EQ("user_id_sequence_id", links_doc[".ref"][0]);+ ASSERT_EQ(1, links_doc.count("user_id_sequence_id"));+ ASSERT_EQ(2, links_doc["user_id_sequence_id"]);+ ASSERT_EQ(0, links_doc.count("repo_id_sequence_id"));++ collectionManager.get_collection_unsafe("Users")->remove("2");++ links_doc = links_collection->get("0").get();+ ASSERT_EQ(1, links_doc.count(".ref"));+ ASSERT_EQ(1, links_doc[".ref"].size());+ ASSERT_EQ("user_id_sequence_id", links_doc[".ref"][0]);+ ASSERT_EQ(1, links_doc.count("user_id_sequence_id"));+ ASSERT_EQ(1, links_doc["user_id_sequence_id"]);++ // Required reference to users was deleted, so the documents are removed.+ auto get_op = links_collection->get("1");+ ASSERT_FALSE(get_op.ok());+ ASSERT_EQ("Could not find a document with id: 1", get_op.error());++ get_op = links_collection->get("7");+ ASSERT_FALSE(get_op.ok());+ ASSERT_EQ("Could not find a document with id: 7", get_op.error());++ schema_json =+ R"({+ "name": "Links_2",+ "fields": [+ {"name": "repo_id", "type": "string", "reference": "Repos.repo_id", "optional": true},+ {"name": "user_id", "type": "string", "reference": "Users.user_id", "optional": true}+ ]+ })"_json;++ documents = {+ R"({+ "repo_id": "repo_b",+ "user_id": "user_b"+ })"_json,+ };++ collection_create_op = collectionManager.create_collection(schema_json);+ ASSERT_TRUE(collection_create_op.ok());+ for (auto const &json: documents) {+ auto add_op = collection_create_op.get()->add(json.dump());+ if (!add_op.ok()) {+ LOG(INFO) << add_op.error();+ }+ ASSERT_TRUE(add_op.ok());+ }++ auto links_2_collection = collectionManager.get_collection_unsafe("Links_2");+ auto links_2_doc = links_2_collection->get("0").get();+ ASSERT_EQ(1, links_2_doc.count(".ref"));+ ASSERT_EQ(2, links_2_doc[".ref"].size());+ ASSERT_EQ("user_id_sequence_id", links_2_doc[".ref"][0]);+ ASSERT_EQ("repo_id_sequence_id", links_2_doc[".ref"][1]);+ ASSERT_EQ(1, links_2_doc.count("user_id_sequence_id"));+ ASSERT_EQ(1, links_2_doc["user_id_sequence_id"]);+ ASSERT_EQ(1, links_2_doc.count("repo_id_sequence_id"));+ ASSERT_EQ(1, links_2_doc["repo_id_sequence_id"]);++ collectionManager.get_collection_unsafe("Users")->remove("1");++ links_2_doc = links_2_collection->get("0").get();+ ASSERT_EQ(1, links_2_doc.count(".ref"));+ ASSERT_EQ(1, links_2_doc[".ref"].size());+ ASSERT_EQ("repo_id_sequence_id", links_2_doc[".ref"][0]);+ ASSERT_EQ(0, links_2_doc.count("user_id_sequence_id"));+ ASSERT_EQ(1, links_2_doc.count("repo_id_sequence_id"));+ ASSERT_EQ(1, links_2_doc["repo_id_sequence_id"]);++ collectionManager.get_collection_unsafe("Repos")->remove("1");++ // All references were deleted, so the document is removed.+ get_op = links_2_collection->get("0");+ ASSERT_FALSE(get_op.ok());+ ASSERT_EQ("Could not find a document with id: 0", get_op.error());}TEST_F(CollectionJoinTest, SortByReference) {