Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle import of raw db data into snapshot.

kishorencb7f9a2ca2884
diff --git a/include/raft_server.h b/include/raft_server.h
index 49d73437..e1ec6956 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -12,6 +12,7 @@
#include "http_data.h"
class Store;
+class ReplicationState;
// Implements the callback for the state machine
class ReplicationClosure : public braft::Closure {
@@ -60,23 +61,14 @@ public:
// Closure that fires when initial snapshot operation finishes
class InitSnapshotClosure : public braft::Closure {
private:
- std::promise<bool>* ready;
+ ReplicationState* replication_state;
public:
- InitSnapshotClosure(std::promise<bool>* ready): ready(ready) {}
+ InitSnapshotClosure(ReplicationState* replication_state): replication_state(replication_state) {}
~InitSnapshotClosure() {}
- void Run() {
- // Auto delete this after Run()
- std::unique_ptr<InitSnapshotClosure> self_guard(this);
-
- if(status().ok()) {
- LOG(INFO) << "Init snapshot succeeded!";
- } else {
- LOG(ERROR) << "Init snapshot failed, error: " << status().error_str();
- }
- }
+ void Run();
};
@@ -96,6 +88,7 @@ private:
butil::atomic<bool> has_initialized;
std::promise<bool>* ready;
+ bool create_init_db_snapshot;
public:
@@ -103,7 +96,8 @@ public:
static constexpr const char* meta_dir_name = "meta";
static constexpr const char* snapshot_dir_name = "snapshot";
- ReplicationState(Store* store, http_message_dispatcher* message_dispatcher, std::promise<bool>* ready);
+ ReplicationState(Store* store, http_message_dispatcher* message_dispatcher,
+ std::promise<bool>* ready, bool create_init_db_snapshot);
~ReplicationState() {
delete node;
@@ -140,6 +134,8 @@ public:
}
}
+ int init_db();
+
rocksdb::DB *get_db() const;
static constexpr const char* REPLICATION_MSG = "raft_replication";
@@ -164,21 +160,17 @@ private:
void on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done);
- int init_db();
-
int on_snapshot_load(braft::SnapshotReader* reader);
void on_leader_start(int64_t term) {
leader_term.store(term, butil::memory_order_release);
- /*if(!has_initialized.load()) {
- has_initialized.store(true, butil::memory_order_release);
- ready->set_value(true);
- }*/
- // have to write a dummy write, otherwise snapshot will not trigger
- //http_req* request = new http_req(nullptr, "SELF", 0, {}, "INIT");
- //http_res* response = new http_res();
- //write(request, response);
+ // have to do a dummy write, otherwise snapshot will not trigger
+ if(create_init_db_snapshot) {
+ http_req* request = new http_req(nullptr, "PRIVATE", 0, {}, "INIT_SNAPSHOT");
+ http_res* response = new http_res();
+ write(request, response);
+ }
LOG(INFO) << "Node becomes leader, term: " << term;
}
@@ -201,10 +193,6 @@ private:
}
void on_start_following(const ::braft::LeaderChangeContext& ctx) {
- if(!has_initialized.load()) {
- has_initialized.store(true, butil::memory_order_release);
- ready->set_value(true);
- }
LOG(INFO) << "Node start following " << ctx;
}
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index c3c25027..e6678b87 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -51,7 +51,9 @@ int ReplicationState::start(int port, int election_timeout_ms, int snapshot_inte
if(snapshot_exists) {
// we will be assured of on_snapshot() firing and we will wait for that to return the promise
- } else {
+ } else if(!create_init_db_snapshot) {
+ // `create_init_db_snapshot` will be handled only after leader starts, otherwise:
+
LOG(INFO) << "Snapshot does not exist. We will remove db dir and init db fresh.";
if (!butil::DeleteFile(butil::FilePath(db_path), true)) {
@@ -158,11 +160,13 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
request = remote_request;
}
- if(request->http_method == "SELF") {
+ if(request->http_method == "PRIVATE" && request->body == "INIT_SNAPSHOT") {
// We attempt to trigger a cold snapshot against an existing stand-alone DB for backward compatibility
- InitSnapshotClosure* init_snapshot_closure = new InitSnapshotClosure(ready);
+ InitSnapshotClosure* init_snapshot_closure = new InitSnapshotClosure(this);
node->snapshot(init_snapshot_closure);
- return ;
+ delete request;
+ delete response;
+ continue ;
}
// Now that the log has been parsed, perform the actual operation
@@ -303,9 +307,23 @@ rocksdb::DB *ReplicationState::get_db() const {
return db;
}
-ReplicationState::ReplicationState(Store *store, http_message_dispatcher *message_dispatcher, std::promise<bool> *ready):
- node(nullptr), leader_term(-1), message_dispatcher(message_dispatcher), has_initialized(false), ready(ready) {
+ReplicationState::ReplicationState(Store *store, http_message_dispatcher *message_dispatcher,
+ std::promise<bool> *ready, bool create_init_db_snapshot):
+ node(nullptr), leader_term(-1), message_dispatcher(message_dispatcher),
+ has_initialized(false), ready(ready), create_init_db_snapshot(create_init_db_snapshot) {
db = store->_get_db_unsafe();
db_path = store->get_state_dir_path();
db_options = store->get_db_options();
}
+
+void InitSnapshotClosure::Run() {
+ // Auto delete this after Run()
+ std::unique_ptr<InitSnapshotClosure> self_guard(this);
+
+ if(status().ok()) {
+ LOG(INFO) << "Init snapshot succeeded!";
+ replication_state->init_db();
+ } else {
+ LOG(ERROR) << "Init snapshot failed, error: " << status().error_str();
+ }
+}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 63c6267a..8a38b6e3 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -228,6 +228,8 @@ int run_server(const Config & config, const std::string & version,
std::string db_dir = config.get_data_dir() + "/db";
std::string state_dir = config.get_data_dir() + "/state";
+ bool create_init_db_snapshot = false; // for importing raw DB from earlier versions
+
if(!directory_exists(db_dir) && file_exists(data_dir+"/CURRENT") && file_exists(data_dir+"/IDENTITY")) {
LOG(INFO) << "Migrating contents of data directory in a `db` sub-directory, as per the new data layout.";
bool moved = mv_dir(data_dir, db_dir);
@@ -236,6 +238,8 @@ int run_server(const Config & config, const std::string & version,
LOG(ERR) << "NOTE: Please move remaining files manually. Failure to do so **WILL** lead to **DATA LOSS**.";
return 1;
}
+
+ create_init_db_snapshot = true;
}
Store store(db_dir);
@@ -264,7 +268,7 @@ int run_server(const Config & config, const std::string & version,
std::promise<bool> ready_promise;
std::future<bool> ready_future = ready_promise.get_future();
- ReplicationState replication_state(&store, server->get_message_dispatcher(), &ready_promise);
+ ReplicationState replication_state(&store, server->get_message_dispatcher(), &ready_promise, create_init_db_snapshot);
std::thread raft_thread([&replication_state, &config, &state_dir]() {
std::string path_to_peers = config.get_raft_peers();