Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Copy only state snapshot directory for on-demand snapshots.
kishorencf085354824d2
diff --git a/include/raft_server.h b/include/raft_server.hindex b78364aa..f6012098 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -81,8 +81,9 @@ private:public:- OnDemandSnapshotClosure(ReplicationState* replication_state, const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res):- replication_state(replication_state), req(req), res(res) {}+ OnDemandSnapshotClosure(ReplicationState *replication_state, const std::shared_ptr<http_req> &req,+ const std::shared_ptr<http_res> &res) :+ replication_state(replication_state), req(req), res(res) {}~OnDemandSnapshotClosure() {}@@ -121,6 +122,10 @@ private:int election_timeout_interval_ms;+ std::mutex mcv;+ std::condition_variable cv;+ bool ready;+public:static constexpr const char* log_dir_name = "log";@@ -193,6 +198,18 @@ public:http_message_dispatcher* get_message_dispatcher() const;+ void wait() {+ auto lk = std::unique_lock<std::mutex>(mcv);+ cv.wait(lk, [&] { return ready; });+ ready = false;+ }++ void notify() {+ std::lock_guard<std::mutex> lk(mcv);+ ready = true;+ cv.notify_all();+ }+static constexpr const char* REPLICATION_MSG = "raft_replication";private:@@ -206,7 +223,6 @@ private:ReplicationState* replication_state;braft::SnapshotWriter* writer;std::string state_dir_path;- std::string db_dir_path;std::string db_snapshot_path;std::string ext_snapshot_path;braft::Closure* done;diff --git a/src/http_server.cpp b/src/http_server.cppindex 8c9969dc..9c431d5c 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -540,16 +540,16 @@ int HttpServer::process_request(const std::shared_ptr<http_req>& request, constauto api_handler = rpath->handler;// LOG(INFO) << "Before enqueue res: " << response- handler->http_server->get_thread_pool()->enqueue([http_server, api_handler, message_dispatcher,+ handler->http_server->get_thread_pool()->enqueue([http_server, rpath, message_dispatcher,request, response]() {// call the API handler- (api_handler)(request, response);+ (rpath->handler)(request, response);- deferred_req_res_t req_res{request, response, http_server};- message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, &req_res);-- std::this_thread::sleep_for(std::chrono::milliseconds(100));- response->wait();+ if(!rpath->async_res) {+ deferred_req_res_t req_res{request, response, http_server};+ message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, &req_res);+ response->wait();+ }});return 0;diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 91ce970e..a3f2166e 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -309,7 +309,6 @@ void* ReplicationState::save_snapshot(void* arg) {SnapshotArg* sa = static_cast<SnapshotArg*>(arg);std::unique_ptr<SnapshotArg> arg_guard(sa);- brpc::ClosureGuard done_guard(sa->done);// add the db snapshot files to writer statebutil::FileEnumerator dir_enum(butil::FilePath(sa->db_snapshot_path), false, butil::FileEnumerator::FILES);@@ -322,14 +321,26 @@ void* ReplicationState::save_snapshot(void* arg) {}}- // if an external snapshot is requested, copy both state and data directory into that+ sa->done->Run();++ // if an external snapshot is requested, copy latest snapshot directory into thatif(!sa->ext_snapshot_path.empty()) {LOG(INFO) << "Copying system snapshot to external snapshot directory at " << sa->ext_snapshot_path;- if(!butil::DirectoryExists(butil::FilePath(sa->ext_snapshot_path))) {- butil::CreateDirectory(butil::FilePath(sa->ext_snapshot_path), true);++ const butil::FilePath& dest_state_dir = butil::FilePath(sa->ext_snapshot_path + "/state");++ if(!butil::DirectoryExists(dest_state_dir)) {+ butil::CreateDirectory(dest_state_dir, true);}- butil::CopyDirectory(butil::FilePath(sa->state_dir_path), butil::FilePath(sa->ext_snapshot_path), true);- butil::CopyDirectory(butil::FilePath(sa->db_dir_path), butil::FilePath(sa->ext_snapshot_path), true);++ const butil::FilePath& src_snapshot_dir = butil::FilePath(sa->state_dir_path + "/snapshot");+ const butil::FilePath& src_meta_dir = butil::FilePath(sa->state_dir_path + "/meta");++ butil::CopyDirectory(src_snapshot_dir, dest_state_dir, true);+ butil::CopyDirectory(src_meta_dir, dest_state_dir, true);++ // notify on demand closure that external snapshotting is done+ sa->replication_state->notify();}// NOTE: *must* do a dummy write here since snapshots cannot be triggered if no write has happened since the@@ -359,7 +370,6 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Clarg->replication_state = this;arg->writer = writer;arg->state_dir_path = raft_dir_path;- arg->db_dir_path = store->get_state_dir_path();arg->db_snapshot_path = db_snapshot_path;arg->done = done;@@ -368,7 +378,7 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Clext_snapshot_path = "";}- // Start a new bthread to avoid blocking StateMachine since it could be slow to write data to disk+ // Start a new bthread to avoid blocking StateMachine for slower operations that don't need a blocking viewbthread_t tid;bthread_start_urgent(&tid, NULL, save_snapshot, arg);}@@ -520,7 +530,8 @@ ReplicationState::ReplicationState(Store *store, ThreadPool* thread_pool, http_mnode(nullptr), leader_term(-1), store(store), thread_pool(thread_pool),message_dispatcher(message_dispatcher),catchup_min_sequence_diff(catchup_min_sequence_diff), catch_up_threshold_percentage(catch_up_threshold_percentage),- api_uses_ssl(api_uses_ssl), create_init_db_snapshot(create_init_db_snapshot), shut_down(quit_service) {+ api_uses_ssl(api_uses_ssl), create_init_db_snapshot(create_init_db_snapshot), shut_down(quit_service),+ ready(false) {}@@ -623,9 +634,10 @@ void InitSnapshotClosure::Run() {}void OnDemandSnapshotClosure::Run() {- // Auto delete this after Run()+ // Auto delete this after Done()std::unique_ptr<OnDemandSnapshotClosure> self_guard(this);+ replication_state->wait(); // until on demand snapshotting completesreplication_state->set_ext_snapshot_path("");req->last_chunk_aggregate = true;