Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Replicate deletion of document and dropping of collection.

Kishore Nallan58bc73312d4d
diff --git a/include/collection.h b/include/collection.h
index 099e3c2b..b6bf155e 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -168,7 +168,7 @@ public:
Option<nlohmann::json> get(const std::string & id);
- Option<std::string> remove(const std::string & id);
+ Option<std::string> remove(const std::string & id, const bool remove_from_store = true);
void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const int & candidate_rank,
Topster<100> &topster, const std::vector<art_leaf *> & query_suggestion, const uint32_t *result_ids,
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 5bdf4895..8d081bfa 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -61,7 +61,7 @@ public:
std::vector<Collection*> get_collections();
- Option<bool> drop_collection(std::string collection_name);
+ Option<bool> drop_collection(std::string collection_name, const bool remove_from_store = true);
uint32_t get_next_collection_id();
diff --git a/include/replicator.h b/include/replicator.h
index bfe3bdc4..0b76564c 100644
--- a/include/replicator.h
+++ b/include/replicator.h
@@ -11,16 +11,13 @@
struct ReplicationEvent {
- std::mutex m;
- std::condition_variable cv;
std::string type;
- uint32_t collection_id;
std::string key;
std::string value;
ReplicationEvent(const std::string& type, const uint32_t collection_id,
const std::string& key, const std::string& value):
- type(type), collection_id(collection_id), key(key), value(value) {
+ type(type), key(key), value(value) {
}
};
@@ -64,6 +61,18 @@ public:
}
void Delete(const rocksdb::Slice& key) {
+ std::vector<std::string> parts;
+ StringUtils::split(key.ToString(), parts, "_");
+
+ if(parts.size() == 3 && parts[1] == Collection::DOC_ID_PREFIX) {
+ ReplicationEvent* replication_event = new ReplicationEvent("REMOVE_DOCUMENT", 0, key.ToString(), "");
+ server->send_message(REPLICATION_EVENT_MSG, replication_event);
+ }
+
+ if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_META_PREFIX) {
+ ReplicationEvent* replication_event = new ReplicationEvent("DROP_COLLECTION", 0, key.ToString(), "");
+ server->send_message(REPLICATION_EVENT_MSG, replication_event);
+ }
}
@@ -128,7 +137,9 @@ public:
if(replication_event->type == "ADD_DOCUMENT") {
CollectionManager & collection_manager = CollectionManager::get_instance();
- Collection* collection = collection_manager.get_collection_with_id(replication_event->collection_id);
+ std::vector<std::string> parts;
+ StringUtils::split(replication_event->key, parts, "_"); // collection_id, seq_id_prefix, seq_id
+ Collection* collection = collection_manager.get_collection_with_id(std::stoi(parts[0]));
nlohmann::json document = nlohmann::json::parse(replication_event->value);
// last 4 bytes of the key would be the serialized version of the sequence id
@@ -139,13 +150,24 @@ public:
if(replication_event->type == "INCR_COLLECTION_NEXT_SEQ") {
CollectionManager & collection_manager = CollectionManager::get_instance();
- std::string collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_NEXT_SEQ_PREFIX)+1);
+ const std::string & collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_NEXT_SEQ_PREFIX)+1);
Collection* collection = collection_manager.get_collection(collection_name);
collection->increment_next_seq_id_field();
}
- if(replication_event->type == "REMOVE") {
+ if(replication_event->type == "REMOVE_DOCUMENT") {
+ std::vector<std::string> parts;
+ StringUtils::split(replication_event->key, parts, "_"); // collection_id, doc_id_prefix, doc_id
+ CollectionManager & collection_manager = CollectionManager::get_instance();
+ Collection* collection = collection_manager.get_collection_with_id(std::stoi(parts[0]));
+ collection->remove(parts[2], false);
+ }
+ if(replication_event->type == "DROP_COLLECTION") {
+ CollectionManager & collection_manager = CollectionManager::get_instance();
+ // <collection_meta_prefix>_<collection_name>
+ const std::string & collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_META_PREFIX)+1);
+ collection_manager.drop_collection(collection_name, false);
}
delete replication_event;
diff --git a/src/collection.cpp b/src/collection.cpp
index b29c710a..a90d9dd1 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1254,7 +1254,7 @@ Option<nlohmann::json> Collection::get(const std::string & id) {
return Option<nlohmann::json>(document);
}
-Option<std::string> Collection::remove(const std::string & id) {
+Option<std::string> Collection::remove(const std::string & id, const bool remove_from_store) {
std::string seq_id_str;
StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);
@@ -1379,8 +1379,10 @@ Option<std::string> Collection::remove(const std::string & id) {
field_doc_value_map.second->erase(seq_id);
}
- store->remove(get_doc_id_key(id));
- store->remove(get_seq_id_key(seq_id));
+ if(remove_from_store) {
+ store->remove(get_doc_id_key(id));
+ store->remove(get_seq_id_key(seq_id));
+ }
num_documents -= 1;
@@ -1410,7 +1412,7 @@ uint32_t Collection::deserialize_seq_id_key(std::string serialized_seq_id) {
}
std::string Collection::get_doc_id_key(const std::string & doc_id) {
- return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + doc_id;
+ return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_" + doc_id;
}
std::string Collection::get_name() {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 1701221c..5d19157f 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -184,23 +184,27 @@ std::vector<Collection*> CollectionManager::get_collections() {
return collection_vec;
}
-Option<bool> CollectionManager::drop_collection(std::string collection_name) {
+Option<bool> CollectionManager::drop_collection(std::string collection_name, const bool remove_from_store) {
Collection* collection = get_collection(collection_name);
if(collection == nullptr) {
return Option<bool>(404, "No collection with name `" + collection_name + "` found.");
}
- store->remove(Collection::get_meta_key(collection_name));
- store->remove(Collection::get_next_seq_id_key(collection_name));
+ if(remove_from_store) {
+ const std::string &collection_id_str = std::to_string(collection->get_collection_id());
- const std::string &collection_id_str = std::to_string(collection->get_collection_id());
- rocksdb::Iterator* iter = store->scan(collection_id_str);
- while(iter->Valid() && iter->key().starts_with(collection_id_str)) {
- store->remove(iter->key().ToString());
- iter->Next();
- }
+ // Note: The following order of dropping documents first before dropping collection meta is important for
+ // replication to work properly!
+ rocksdb::Iterator* iter = store->scan(collection_id_str);
+ while(iter->Valid() && iter->key().starts_with(collection_id_str)) {
+ store->remove(iter->key().ToString());
+ iter->Next();
+ }
+ delete iter;
- delete iter;
+ store->remove(Collection::get_next_seq_id_key(collection_name));
+ store->remove(Collection::get_meta_key(collection_name));
+ }
collections.erase(Collection::get_meta_key(collection_name));
collection_id_names.erase(collection->get_collection_id());
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 777bb592..dcc0d281 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -154,6 +154,9 @@ TEST_F(CollectionManagerTest, DropCollectionCleanly) {
infile.close();
+ ASSERT_FALSE(nullptr == collectionManager.get_collection_with_id(0));
+ ASSERT_FALSE(nullptr == collectionManager.get_collection("collection1"));
+
collectionManager.drop_collection("collection1");
rocksdb::Iterator* it = store->get_iterator();
@@ -168,6 +171,7 @@ TEST_F(CollectionManagerTest, DropCollectionCleanly) {
ASSERT_TRUE(it->status().ok());
ASSERT_EQ(nullptr, collectionManager.get_collection("collection1"));
+ ASSERT_EQ(nullptr, collectionManager.get_collection_with_id(0));
ASSERT_EQ(1, collectionManager.get_next_collection_id());
delete it;