Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Replicate deletion of document and dropping of collection.
Kishore Nallan58bc73312d4d
diff --git a/include/collection.h b/include/collection.hindex 099e3c2b..b6bf155e 100644--- a/include/collection.h+++ b/include/collection.h@@ -168,7 +168,7 @@ public:Option<nlohmann::json> get(const std::string & id);- Option<std::string> remove(const std::string & id);+ Option<std::string> remove(const std::string & id, const bool remove_from_store = true);void score_results(const std::vector<sort_by> & sort_fields, const int & query_index, const int & candidate_rank,Topster<100> &topster, const std::vector<art_leaf *> & query_suggestion, const uint32_t *result_ids,diff --git a/include/collection_manager.h b/include/collection_manager.hindex 5bdf4895..8d081bfa 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -61,7 +61,7 @@ public:std::vector<Collection*> get_collections();- Option<bool> drop_collection(std::string collection_name);+ Option<bool> drop_collection(std::string collection_name, const bool remove_from_store = true);uint32_t get_next_collection_id();diff --git a/include/replicator.h b/include/replicator.hindex bfe3bdc4..0b76564c 100644--- a/include/replicator.h+++ b/include/replicator.h@@ -11,16 +11,13 @@struct ReplicationEvent {- std::mutex m;- std::condition_variable cv;std::string type;- uint32_t collection_id;std::string key;std::string value;ReplicationEvent(const std::string& type, const uint32_t collection_id,const std::string& key, const std::string& value):- type(type), collection_id(collection_id), key(key), value(value) {+ type(type), key(key), value(value) {}};@@ -64,6 +61,18 @@ public:}void Delete(const rocksdb::Slice& key) {+ std::vector<std::string> parts;+ StringUtils::split(key.ToString(), parts, "_");++ if(parts.size() == 3 && parts[1] == Collection::DOC_ID_PREFIX) {+ ReplicationEvent* replication_event = new ReplicationEvent("REMOVE_DOCUMENT", 0, key.ToString(), "");+ server->send_message(REPLICATION_EVENT_MSG, replication_event);+ }++ if(parts.size() >= 2 && parts[0] == Collection::COLLECTION_META_PREFIX) {+ ReplicationEvent* replication_event = new ReplicationEvent("DROP_COLLECTION", 0, key.ToString(), "");+ server->send_message(REPLICATION_EVENT_MSG, replication_event);+ }}@@ -128,7 +137,9 @@ public:if(replication_event->type == "ADD_DOCUMENT") {CollectionManager & collection_manager = CollectionManager::get_instance();- Collection* collection = collection_manager.get_collection_with_id(replication_event->collection_id);+ std::vector<std::string> parts;+ StringUtils::split(replication_event->key, parts, "_"); // collection_id, seq_id_prefix, seq_id+ Collection* collection = collection_manager.get_collection_with_id(std::stoi(parts[0]));nlohmann::json document = nlohmann::json::parse(replication_event->value);// last 4 bytes of the key would be the serialized version of the sequence id@@ -139,13 +150,24 @@ public:if(replication_event->type == "INCR_COLLECTION_NEXT_SEQ") {CollectionManager & collection_manager = CollectionManager::get_instance();- std::string collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_NEXT_SEQ_PREFIX)+1);+ const std::string & collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_NEXT_SEQ_PREFIX)+1);Collection* collection = collection_manager.get_collection(collection_name);collection->increment_next_seq_id_field();}- if(replication_event->type == "REMOVE") {+ if(replication_event->type == "REMOVE_DOCUMENT") {+ std::vector<std::string> parts;+ StringUtils::split(replication_event->key, parts, "_"); // collection_id, doc_id_prefix, doc_id+ CollectionManager & collection_manager = CollectionManager::get_instance();+ Collection* collection = collection_manager.get_collection_with_id(std::stoi(parts[0]));+ collection->remove(parts[2], false);+ }+ if(replication_event->type == "DROP_COLLECTION") {+ CollectionManager & collection_manager = CollectionManager::get_instance();+ // <collection_meta_prefix>_<collection_name>+ const std::string & collection_name = replication_event->key.substr(strlen(Collection::COLLECTION_META_PREFIX)+1);+ collection_manager.drop_collection(collection_name, false);}delete replication_event;diff --git a/src/collection.cpp b/src/collection.cppindex b29c710a..a90d9dd1 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1254,7 +1254,7 @@ Option<nlohmann::json> Collection::get(const std::string & id) {return Option<nlohmann::json>(document);}-Option<std::string> Collection::remove(const std::string & id) {+Option<std::string> Collection::remove(const std::string & id, const bool remove_from_store) {std::string seq_id_str;StoreStatus status = store->get(get_doc_id_key(id), seq_id_str);@@ -1379,8 +1379,10 @@ Option<std::string> Collection::remove(const std::string & id) {field_doc_value_map.second->erase(seq_id);}- store->remove(get_doc_id_key(id));- store->remove(get_seq_id_key(seq_id));+ if(remove_from_store) {+ store->remove(get_doc_id_key(id));+ store->remove(get_seq_id_key(seq_id));+ }num_documents -= 1;@@ -1410,7 +1412,7 @@ uint32_t Collection::deserialize_seq_id_key(std::string serialized_seq_id) {}std::string Collection::get_doc_id_key(const std::string & doc_id) {- return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + doc_id;+ return std::to_string(collection_id) + "_" + DOC_ID_PREFIX + "_" + doc_id;}std::string Collection::get_name() {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 1701221c..5d19157f 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -184,23 +184,27 @@ std::vector<Collection*> CollectionManager::get_collections() {return collection_vec;}-Option<bool> CollectionManager::drop_collection(std::string collection_name) {+Option<bool> CollectionManager::drop_collection(std::string collection_name, const bool remove_from_store) {Collection* collection = get_collection(collection_name);if(collection == nullptr) {return Option<bool>(404, "No collection with name `" + collection_name + "` found.");}- store->remove(Collection::get_meta_key(collection_name));- store->remove(Collection::get_next_seq_id_key(collection_name));+ if(remove_from_store) {+ const std::string &collection_id_str = std::to_string(collection->get_collection_id());- const std::string &collection_id_str = std::to_string(collection->get_collection_id());- rocksdb::Iterator* iter = store->scan(collection_id_str);- while(iter->Valid() && iter->key().starts_with(collection_id_str)) {- store->remove(iter->key().ToString());- iter->Next();- }+ // Note: The following order of dropping documents first before dropping collection meta is important for+ // replication to work properly!+ rocksdb::Iterator* iter = store->scan(collection_id_str);+ while(iter->Valid() && iter->key().starts_with(collection_id_str)) {+ store->remove(iter->key().ToString());+ iter->Next();+ }+ delete iter;- delete iter;+ store->remove(Collection::get_next_seq_id_key(collection_name));+ store->remove(Collection::get_meta_key(collection_name));+ }collections.erase(Collection::get_meta_key(collection_name));collection_id_names.erase(collection->get_collection_id());diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 777bb592..dcc0d281 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -154,6 +154,9 @@ TEST_F(CollectionManagerTest, DropCollectionCleanly) {infile.close();+ ASSERT_FALSE(nullptr == collectionManager.get_collection_with_id(0));+ ASSERT_FALSE(nullptr == collectionManager.get_collection("collection1"));+collectionManager.drop_collection("collection1");rocksdb::Iterator* it = store->get_iterator();@@ -168,6 +171,7 @@ TEST_F(CollectionManagerTest, DropCollectionCleanly) {ASSERT_TRUE(it->status().ok());ASSERT_EQ(nullptr, collectionManager.get_collection("collection1"));+ ASSERT_EQ(nullptr, collectionManager.get_collection_with_id(0));ASSERT_EQ(1, collectionManager.get_next_collection_id());delete it;