Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle import of raw db data into snapshot.
kishorencb7f9a2ca2884
diff --git a/include/raft_server.h b/include/raft_server.hindex 49d73437..e1ec6956 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -12,6 +12,7 @@#include "http_data.h"class Store;+class ReplicationState;// Implements the callback for the state machineclass ReplicationClosure : public braft::Closure {@@ -60,23 +61,14 @@ public:// Closure that fires when initial snapshot operation finishesclass InitSnapshotClosure : public braft::Closure {private:- std::promise<bool>* ready;+ ReplicationState* replication_state;public:- InitSnapshotClosure(std::promise<bool>* ready): ready(ready) {}+ InitSnapshotClosure(ReplicationState* replication_state): replication_state(replication_state) {}~InitSnapshotClosure() {}- void Run() {- // Auto delete this after Run()- std::unique_ptr<InitSnapshotClosure> self_guard(this);-- if(status().ok()) {- LOG(INFO) << "Init snapshot succeeded!";- } else {- LOG(ERROR) << "Init snapshot failed, error: " << status().error_str();- }- }+ void Run();};@@ -96,6 +88,7 @@ private:butil::atomic<bool> has_initialized;std::promise<bool>* ready;+ bool create_init_db_snapshot;public:@@ -103,7 +96,8 @@ public:static constexpr const char* meta_dir_name = "meta";static constexpr const char* snapshot_dir_name = "snapshot";- ReplicationState(Store* store, http_message_dispatcher* message_dispatcher, std::promise<bool>* ready);+ ReplicationState(Store* store, http_message_dispatcher* message_dispatcher,+ std::promise<bool>* ready, bool create_init_db_snapshot);~ReplicationState() {delete node;@@ -140,6 +134,8 @@ public:}}+ int init_db();+rocksdb::DB *get_db() const;static constexpr const char* REPLICATION_MSG = "raft_replication";@@ -164,21 +160,17 @@ private:void on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done);- int init_db();-int on_snapshot_load(braft::SnapshotReader* reader);void on_leader_start(int64_t term) {leader_term.store(term, butil::memory_order_release);- /*if(!has_initialized.load()) {- has_initialized.store(true, butil::memory_order_release);- ready->set_value(true);- }*/- // have to write a dummy write, otherwise snapshot will not trigger- //http_req* request = new http_req(nullptr, "SELF", 0, {}, "INIT");- //http_res* response = new http_res();- //write(request, response);+ // have to do a dummy write, otherwise snapshot will not trigger+ if(create_init_db_snapshot) {+ http_req* request = new http_req(nullptr, "PRIVATE", 0, {}, "INIT_SNAPSHOT");+ http_res* response = new http_res();+ write(request, response);+ }LOG(INFO) << "Node becomes leader, term: " << term;}@@ -201,10 +193,6 @@ private:}void on_start_following(const ::braft::LeaderChangeContext& ctx) {- if(!has_initialized.load()) {- has_initialized.store(true, butil::memory_order_release);- ready->set_value(true);- }LOG(INFO) << "Node start following " << ctx;}diff --git a/src/raft_server.cpp b/src/raft_server.cppindex c3c25027..e6678b87 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -51,7 +51,9 @@ int ReplicationState::start(int port, int election_timeout_ms, int snapshot_inteif(snapshot_exists) {// we will be assured of on_snapshot() firing and we will wait for that to return the promise- } else {+ } else if(!create_init_db_snapshot) {+ // `create_init_db_snapshot` will be handled only after leader starts, otherwise:+LOG(INFO) << "Snapshot does not exist. We will remove db dir and init db fresh.";if (!butil::DeleteFile(butil::FilePath(db_path), true)) {@@ -158,11 +160,13 @@ void ReplicationState::on_apply(braft::Iterator& iter) {request = remote_request;}- if(request->http_method == "SELF") {+ if(request->http_method == "PRIVATE" && request->body == "INIT_SNAPSHOT") {// We attempt to trigger a cold snapshot against an existing stand-alone DB for backward compatibility- InitSnapshotClosure* init_snapshot_closure = new InitSnapshotClosure(ready);+ InitSnapshotClosure* init_snapshot_closure = new InitSnapshotClosure(this);node->snapshot(init_snapshot_closure);- return ;+ delete request;+ delete response;+ continue ;}// Now that the log has been parsed, perform the actual operation@@ -303,9 +307,23 @@ rocksdb::DB *ReplicationState::get_db() const {return db;}-ReplicationState::ReplicationState(Store *store, http_message_dispatcher *message_dispatcher, std::promise<bool> *ready):- node(nullptr), leader_term(-1), message_dispatcher(message_dispatcher), has_initialized(false), ready(ready) {+ReplicationState::ReplicationState(Store *store, http_message_dispatcher *message_dispatcher,+ std::promise<bool> *ready, bool create_init_db_snapshot):+ node(nullptr), leader_term(-1), message_dispatcher(message_dispatcher),+ has_initialized(false), ready(ready), create_init_db_snapshot(create_init_db_snapshot) {db = store->_get_db_unsafe();db_path = store->get_state_dir_path();db_options = store->get_db_options();}++void InitSnapshotClosure::Run() {+ // Auto delete this after Run()+ std::unique_ptr<InitSnapshotClosure> self_guard(this);++ if(status().ok()) {+ LOG(INFO) << "Init snapshot succeeded!";+ replication_state->init_db();+ } else {+ LOG(ERROR) << "Init snapshot failed, error: " << status().error_str();+ }+}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 63c6267a..8a38b6e3 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -228,6 +228,8 @@ int run_server(const Config & config, const std::string & version,std::string db_dir = config.get_data_dir() + "/db";std::string state_dir = config.get_data_dir() + "/state";+ bool create_init_db_snapshot = false; // for importing raw DB from earlier versions+if(!directory_exists(db_dir) && file_exists(data_dir+"/CURRENT") && file_exists(data_dir+"/IDENTITY")) {LOG(INFO) << "Migrating contents of data directory in a `db` sub-directory, as per the new data layout.";bool moved = mv_dir(data_dir, db_dir);@@ -236,6 +238,8 @@ int run_server(const Config & config, const std::string & version,LOG(ERR) << "NOTE: Please move remaining files manually. Failure to do so **WILL** lead to **DATA LOSS**.";return 1;}++ create_init_db_snapshot = true;}Store store(db_dir);@@ -264,7 +268,7 @@ int run_server(const Config & config, const std::string & version,std::promise<bool> ready_promise;std::future<bool> ready_future = ready_promise.get_future();- ReplicationState replication_state(&store, server->get_message_dispatcher(), &ready_promise);+ ReplicationState replication_state(&store, server->get_message_dispatcher(), &ready_promise, create_init_db_snapshot);std::thread raft_thread([&replication_state, &config, &state_dir]() {std::string path_to_peers = config.get_raft_peers();