Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix cascade deletion in case of a reference array field. (#1844)

Harpreet Sangard9b83a9fc7a0
diff --git a/include/collection.h b/include/collection.h
index fb0b6fd4..051af42c 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -616,7 +616,8 @@ public:
Option<nlohmann::json> get(const std::string & id) const;
- void remove_ref_docs(const uint32_t& count, uint32_t const* const docs, bool remove_from_store = true);
+ void cascade_remove_docs(const std::string& ref_helper_field_name, const uint32_t& ref_seq_id,
+ const nlohmann::json& ref_doc, bool remove_from_store = true);
Option<std::string> remove(const std::string & id, bool remove_from_store = true);
diff --git a/src/collection.cpp b/src/collection.cpp
index d9fe2a82..61225ea9 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -4499,58 +4499,169 @@ Option<nlohmann::json> Collection::get(const std::string & id) const {
}
void Collection::remove_document(nlohmann::json & document, const uint32_t seq_id, bool remove_from_store) {
- const std::string& id = document["id"];
+ spp::sparse_hash_map<std::string, std::string> referenced_in_copy;
+ {
+ std::unique_lock lock(mutex);
+ referenced_in_copy = referenced_in;
+ }
+
+ // Cascade delete all the references.
+ if (!referenced_in_copy.empty()) {
+ CollectionManager& collectionManager = CollectionManager::get_instance();
+ for (const auto &item: referenced_in_copy) {
+ auto coll = collectionManager.get_collection(item.first);
+ if (coll != nullptr) {
+ coll->cascade_remove_docs(item.second, seq_id, document, remove_from_store);
+ }
+ }
+ }
{
std::unique_lock lock(mutex);
index->remove(seq_id, document, {}, false);
- if(num_documents != 0) {
+ if (num_documents != 0) {
num_documents -= 1;
}
}
if(remove_from_store) {
+ const std::string& id = document["id"];
+
store->remove(get_doc_id_key(id));
store->remove(get_seq_id_key(seq_id));
}
+}
- if (referenced_in.empty()) {
+void Collection::cascade_remove_docs(const std::string& ref_helper_field_name, const uint32_t& ref_seq_id,
+ const nlohmann::json& ref_doc, bool remove_from_store) {
+ auto field_name = ref_helper_field_name.substr(0, ref_helper_field_name.size() - fields::REFERENCE_HELPER_FIELD_SUFFIX.size());
+
+ filter_result_t filter_result;
+ get_filter_ids(ref_helper_field_name + ":" + std::to_string(ref_seq_id), filter_result);
+
+ if (filter_result.count == 0) {
return;
}
- CollectionManager& collectionManager = CollectionManager::get_instance();
- // Cascade delete all the references.
- for (const auto &item: referenced_in) {
- auto ref_coll = collectionManager.get_collection(item.first);
- if (ref_coll != nullptr) {
- // Delete all the docs where reference helper field has value `seq_id`.
- filter_result_t filter_result;
- ref_coll->get_filter_ids(item.second + ":" + std::to_string(seq_id), filter_result);
+ bool is_field_singular;
+ {
+ std::unique_lock lock(mutex);
- ref_coll->remove_ref_docs(filter_result.count, filter_result.docs, remove_from_store);
+ auto it = search_schema.find(field_name);
+ if (it == search_schema.end()) {
+ return;
}
+ is_field_singular = it.value().is_singular();
}
-}
-void Collection::remove_ref_docs(const uint32_t& count, uint32_t const* const docs, bool remove_from_store) {
- for (uint32_t i = 0; i < count; i++) {
- auto const& seq_id = docs[i];
+ if (is_field_singular) {
+ // Delete all the docs where reference helper field has value `seq_id`.
+ for (uint32_t i = 0; i < filter_result.count; i++) {
+ auto const& seq_id = filter_result.docs[i];
- nlohmann::json document;
- auto get_doc_op = get_document_from_store(get_seq_id_key(seq_id), document);
+ nlohmann::json document;
+ auto get_doc_op = get_document_from_store(get_seq_id_key(seq_id), document);
+
+ if (!get_doc_op.ok()) {
+ if (get_doc_op.code() == 404) {
+ LOG(ERROR) << "`" << name << "` collection: Sequence ID `" << seq_id << "` exists, but document is missing.";
+ continue;
+ }
- if( !get_doc_op.ok()) {
- if (get_doc_op.code() == 404) {
- LOG(ERROR) << "`" << name << "` collection: Sequence ID `" << seq_id << "` exists, but document is missing.";
+ LOG(ERROR) << "`" << name << "` collection: " << get_doc_op.error();
continue;
}
- LOG(ERROR) << "`" << name << "` collection: " << get_doc_op.error();
- continue;
+ remove_document(document, seq_id, remove_from_store);
+ }
+ } else {
+ std::string ref_coll_name, ref_field_name;
+ {
+ std::unique_lock lock(mutex);
+
+ auto ref_it = reference_fields.find(field_name);
+ if (ref_it == reference_fields.end()) {
+ return;
+ }
+ ref_coll_name = ref_it->second.collection;
+ ref_field_name = ref_it->second.field;
+ }
+
+ if (ref_doc.count(ref_field_name) == 0) {
+ LOG(ERROR) << "`" << ref_coll_name << "` collection doc `" << ref_doc.dump() << "` is missing `" <<
+ ref_field_name << "` field.";
+ return;
+ } else if (ref_doc.at(ref_field_name).is_array()) {
+ LOG(ERROR) << "`" << ref_coll_name << "` collection doc `" << ref_doc.dump() << "` field `" <<
+ ref_field_name << "` is an array.";
+ return;
+ }
+
+ std::vector<std::string> buffer;
+ buffer.reserve(filter_result.count);
+
+ // Delete all references to `seq_id` in the docs.
+ for (uint32_t i = 0; i < filter_result.count; i++) {
+ auto const& seq_id = filter_result.docs[i];
+
+ nlohmann::json existing_document;
+ auto get_doc_op = get_document_from_store(get_seq_id_key(seq_id), existing_document);
+
+ if (!get_doc_op.ok()) {
+ if (get_doc_op.code() == 404) {
+ LOG(ERROR) << "`" << name << "` collection: Sequence ID `" << seq_id << "` exists, but document is missing.";
+ continue;
+ }
+
+ LOG(ERROR) << "`" << name << "` collection: " << get_doc_op.error();
+ continue;
+ }
+
+ if (existing_document.count("id") == 0) {
+ LOG(ERROR) << "`" << name << "` collection doc `" << existing_document.dump() << "` is missing `id` field.";
+ } else if (existing_document.count(field_name) == 0) {
+ LOG(ERROR) << "`" << name << "` collection doc `" << existing_document.dump() << "` is missing `" <<
+ field_name << "` field.";
+ } else if (!existing_document.at(field_name).is_array()) {
+ LOG(ERROR) << "`" << name << "` collection doc `" << existing_document.dump() << "` field `" <<
+ field_name << "` is not an array.";
+ } else if (existing_document.at(field_name).empty()) {
+ LOG(ERROR) << "`" << name << "` collection doc `" << existing_document.dump() << "` field `" <<
+ field_name << "` is empty.";
+ } else if (existing_document.at(field_name)[0].type() != ref_doc.at(ref_field_name).type()) {
+ LOG(ERROR) << "`" << name << "` collection doc `" << existing_document.dump() <<
+ "` at field `" << field_name << "` elements do not match the type of `" << ref_coll_name <<
+ "` collection doc `"<< ref_doc.dump() << "` at field `" << ref_field_name << "`.";
+ }
+ // If there are more than one references present in this document, we cannot delete the whole doc. Only remove
+ // `ref_seq_id` from reference helper field.
+ else if (existing_document.at(field_name).size() > 1) {
+ nlohmann::json update_document;
+ update_document["id"] = existing_document["id"].get<std::string>();
+ update_document[field_name] = nlohmann::json::array();
+
+ auto removed_ref_value_found = false;
+ for (const auto& ref_value: existing_document.at(field_name)) {
+ if (ref_value == ref_doc.at(ref_field_name)) {
+ removed_ref_value_found = true;
+ continue;
+ }
+
+ update_document[field_name] += ref_value;
+ }
+
+ if (removed_ref_value_found) {
+ buffer.push_back(update_document.dump());
+ }
+ continue;
+ }
+
+ remove_document(existing_document, seq_id, remove_from_store);
}
- remove_document(document, seq_id, remove_from_store);
+ nlohmann::json dummy;
+ add_many(buffer, dummy, index_operation_t::UPDATE);
}
}
diff --git a/test/collection_join_test.cpp b/test/collection_join_test.cpp
index 1f89adea..ba579a66 100644
--- a/test/collection_join_test.cpp
+++ b/test/collection_join_test.cpp
@@ -4710,6 +4710,94 @@ TEST_F(CollectionJoinTest, CascadeDeletion) {
ASSERT_EQ("2", res_obj["hits"][1]["document"].at("leadId"));
ASSERT_EQ("2", res_obj["hits"][1]["document"].at("documentId"));
+
+ schema_json =
+ R"({
+ "name": "split_members",
+ "fields": [
+ { "name": "user_id", "type": "string" }
+ ]
+ })"_json;
+
+ documents = {
+ R"({"user_id": "user_a"})"_json,
+ R"({"user_id": "user_b"})"_json,
+ R"({"user_id": "user_c"})"_json
+ };
+
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ schema_json =
+ R"({
+ "name": "splits",
+ "fields": [
+ { "name": "name", "type": "string" },
+ { "name": "members", "type": "string[]", "reference": "split_members.user_id" }
+ ]
+ })"_json;
+
+ documents = {
+ R"({
+ "name": "foo",
+ "members": ["user_a", "user_b", "user_c"]
+ })"_json,
+ R"({
+ "name": "bar",
+ "members": ["user_b"]
+ })"_json,
+ };
+
+ collection_create_op = collectionManager.create_collection(schema_json);
+ ASSERT_TRUE(collection_create_op.ok());
+ for (auto const &json: documents) {
+ auto add_op = collection_create_op.get()->add(json.dump());
+ ASSERT_TRUE(add_op.ok());
+ }
+
+ req_params = {
+ {"collection", "splits"},
+ {"q", "*"},
+ {"include_fields", "$split_members(*)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(2, res_obj["found"].get<size_t>());
+
+ ASSERT_EQ("bar", res_obj["hits"][0]["document"].at("name"));
+ ASSERT_EQ(1, res_obj["hits"][0]["document"].at("split_members").size());
+ ASSERT_EQ("user_b", res_obj["hits"][0]["document"]["split_members"][0].at("user_id"));
+
+ ASSERT_EQ("foo", res_obj["hits"][1]["document"].at("name"));
+ ASSERT_EQ(3, res_obj["hits"][1]["document"].at("split_members").size());
+ ASSERT_EQ("user_a", res_obj["hits"][1]["document"]["split_members"][0].at("user_id"));
+ ASSERT_EQ("user_b", res_obj["hits"][1]["document"]["split_members"][1].at("user_id"));
+ ASSERT_EQ("user_c", res_obj["hits"][1]["document"]["split_members"][2].at("user_id"));
+
+ // Remove `user_b`.
+ collectionManager.get_collection_unsafe("split_members")->remove("1");
+
+ req_params = {
+ {"collection", "splits"},
+ {"q", "*"},
+ {"include_fields", "$split_members(*)"}
+ };
+ search_op = collectionManager.do_search(req_params, embedded_params, json_res, now_ts);
+ ASSERT_TRUE(search_op.ok());
+
+ res_obj = nlohmann::json::parse(json_res);
+ ASSERT_EQ(1, res_obj["found"].get<size_t>());
+
+ ASSERT_EQ("foo", res_obj["hits"][0]["document"].at("name"));
+ ASSERT_EQ(2, res_obj["hits"][0]["document"].at("split_members").size());
+ ASSERT_EQ("user_a", res_obj["hits"][0]["document"]["split_members"][0].at("user_id"));
+ ASSERT_EQ("user_c", res_obj["hits"][0]["document"]["split_members"][1].at("user_id"));
}
TEST_F(CollectionJoinTest, SortByReference) {