Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle presence/absence of snapshot.
kishorenca52ec8099919
diff --git a/include/collection_manager.h b/include/collection_manager.hindex ce8cdb9a..c059bd57 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -56,11 +56,13 @@ public:CollectionManager(CollectionManager const&) = delete;void operator=(CollectionManager const&) = delete;- Option<bool> init(Store *store,- const size_t default_num_indices,- const std::string & auth_key,- const std::string & search_only_auth_key,- const size_t init_batch_size=1000);+ void init(Store *store,+ const size_t default_num_indices,+ const std::string & auth_key,+ const std::string & search_only_auth_key+ );++ Option<bool> load(const size_t init_batch_size=1000);// frees in-memory data structures when server is shutdown - helps us run a memory leak detecter properlyvoid dispose();diff --git a/include/config.h b/include/config.hindex dd51cd4b..401b9333 100644--- a/include/config.h+++ b/include/config.h@@ -16,7 +16,6 @@ private:std::string listen_address;uint32_t listen_port;- std::string raft_dir;uint32_t raft_port;std::string raft_peers;@@ -92,10 +91,6 @@ public:this->raft_port = raft_port;}- void set_raft_dir(const std::string & raft_dir) {- this->raft_dir = raft_dir;- }-void set_raft_peers(const std::string & raft_peers) {this->raft_peers = raft_peers;}@@ -154,10 +149,6 @@ public:return this->raft_port;}- std::string get_raft_dir() const {- return this->raft_dir;- }-std::string get_raft_peers() const {return this->raft_peers;}@@ -191,7 +182,6 @@ public:this->raft_port = std::stoi(get_env("TYPESENSE_RAFT_PORT"));}- this->raft_dir = get_env("TYPESENSE_RAFT_DIR");this->raft_peers = get_env("TYPESENSE_RAFT_PEERS");this->master = get_env("TYPESENSE_MASTER");@@ -266,10 +256,6 @@ public:this->raft_port = reader.GetInteger("server", "raft-port", 8107);}- if(reader.Exists("server", "raft-dir")) {- this->raft_dir = reader.Get("server", "raft-dir", "");- }-if(reader.Exists("server", "raft-peers")) {this->raft_peers = reader.Get("server", "raft-peers", "");}@@ -320,10 +306,6 @@ public:this->raft_port = options.get<uint32_t>("raft-port");}- if(options.exist("raft-dir")) {- this->raft_dir = options.get<std::string>("raft-dir");- }-if(options.exist("raft-peers")) {this->raft_peers = options.get<std::string>("raft-peers");}@@ -344,14 +326,6 @@ public:return Option<bool>(500, "API key is not specified.");}- if(!raft_dir.empty() && raft_peers.empty()) {- return Option<bool>(500, "Argument --raft-peers is not specified.");- }-- if(!raft_peers.empty() && raft_dir.empty()) {- return Option<bool>(500, "Argument --raft-dir is not specified.");- }-return Option<bool>(true);}};diff --git a/include/file_utils.h b/include/file_utils.hnew file mode 100644index 00000000..e7bba268--- /dev/null+++ b/include/file_utils.h@@ -0,0 +1,14 @@+bool directory_exists(const std::string & dir_path);++bool file_exists(const std::string & file_path);++// tries to hard link first+bool copy_dir(const std::string& from_path, const std::string& to_path);++bool mv_dir(const std::string& from_path, const std::string& to_path);++bool rename_path(const std::string& from_path, const std::string& to_path);++bool delete_path(const std::string& path, bool recursive = true);++bool dir_enum_count(const std::string & path);\ No newline at end of filediff --git a/include/http_client.h b/include/http_client.hindex e7b0bb1a..bc141bfd 100644--- a/include/http_client.h+++ b/include/http_client.h@@ -2,6 +2,7 @@#include <string>#include <curl/curl.h>+#include "file_utils.h"/*NOTE: This is a really primitive blocking client meant only for specific Typesense use cases.diff --git a/include/raft_server.h b/include/raft_server.hindex 784a2eca..49d73437 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -11,6 +11,7 @@#include "http_data.h"+class Store;// Implements the callback for the state machineclass ReplicationClosure : public braft::Closure {@@ -56,29 +57,61 @@ public:}};+// Closure that fires when initial snapshot operation finishes+class InitSnapshotClosure : public braft::Closure {+private:+ std::promise<bool>* ready;+public:++ InitSnapshotClosure(std::promise<bool>* ready): ready(ready) {}++ ~InitSnapshotClosure() {}++ void Run() {+ // Auto delete this after Run()+ std::unique_ptr<InitSnapshotClosure> self_guard(this);++ if(status().ok()) {+ LOG(INFO) << "Init snapshot succeeded!";+ } else {+ LOG(ERROR) << "Init snapshot failed, error: " << status().error_str();+ }+ }+};++// Implements braft::StateMachine.class ReplicationState : public braft::StateMachine {private:- braft::Node* volatile _node;- butil::atomic<int64_t> _value;- butil::atomic<int64_t> _leader_term;+ static constexpr const char* db_snapshot_name = "db_snapshot";++ braft::Node* volatile node;+ butil::atomic<int64_t> leader_term;+rocksdb::DB* db;+ std::string db_path;+ rocksdb::Options db_options;+http_message_dispatcher* message_dispatcher;+ butil::atomic<bool> has_initialized;+ std::promise<bool>* ready;+public:- ReplicationState(http_message_dispatcher* message_dispatcher, rocksdb::DB* db):- _node(NULL), _value(0), _leader_term(-1), db(db),- message_dispatcher(message_dispatcher) {- }+ static constexpr const char* log_dir_name = "log";+ static constexpr const char* meta_dir_name = "meta";+ static constexpr const char* snapshot_dir_name = "snapshot";++ ReplicationState(Store* store, http_message_dispatcher* message_dispatcher, std::promise<bool>* ready);~ReplicationState() {- delete _node;+ delete node;}// Starts this nodeint start(int port, int election_timeout_ms, int snapshot_interval_s,- const std::string & data_path, const std::string & peers);+ const std::string & raft_dir, const std::string & peers);// Generic write method for synchronizing all writesvoid write(http_req* request, http_res* response);@@ -90,23 +123,25 @@ public:void refresh_peers(const std::string & peers);bool is_leader() const {- return _leader_term.load(butil::memory_order_acquire) > 0;+ return leader_term.load(butil::memory_order_acquire) > 0;}// Shut this node down.void shutdown() {- if (_node) {- _node->shutdown(NULL);+ if (node) {+ node->shutdown(nullptr);}}// Blocking this thread until the node is eventually down.void join() {- if (_node) {- _node->join();+ if (node) {+ node->join();}}+ rocksdb::DB *get_db() const;+static constexpr const char* REPLICATION_MSG = "raft_replication";private:@@ -129,17 +164,27 @@ private:void on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done);- static bool copy_snapshot(const std::string& from_path, const std::string& to_path);+ int init_db();int on_snapshot_load(braft::SnapshotReader* reader);void on_leader_start(int64_t term) {- _leader_term.store(term, butil::memory_order_release);+ leader_term.store(term, butil::memory_order_release);+ /*if(!has_initialized.load()) {+ has_initialized.store(true, butil::memory_order_release);+ ready->set_value(true);+ }*/++ // have to write a dummy write, otherwise snapshot will not trigger+ //http_req* request = new http_req(nullptr, "SELF", 0, {}, "INIT");+ //http_res* response = new http_res();+ //write(request, response);+LOG(INFO) << "Node becomes leader, term: " << term;}void on_leader_stop(const butil::Status& status) {- _leader_term.store(-1, butil::memory_order_release);+ leader_term.store(-1, butil::memory_order_release);LOG(INFO) << "Node stepped down : " << status;}@@ -155,11 +200,15 @@ private:LOG(INFO) << "Configuration of this group is " << conf;}- void on_stop_following(const ::braft::LeaderChangeContext& ctx) {- LOG(INFO) << "Node stops following " << ctx;- }-void on_start_following(const ::braft::LeaderChangeContext& ctx) {+ if(!has_initialized.load()) {+ has_initialized.store(true, butil::memory_order_release);+ ready->set_value(true);+ }LOG(INFO) << "Node start following " << ctx;}++ void on_stop_following(const ::braft::LeaderChangeContext& ctx) {+ LOG(INFO) << "Node stops following " << ctx;+ }};diff --git a/include/store.h b/include/store.hindex bad416db..609ec7d0 100644--- a/include/store.h+++ b/include/store.h@@ -44,7 +44,6 @@ class Store {private:const std::string state_dir_path;-rocksdb::DB *db;rocksdb::Options options;rocksdb::WriteOptions write_options;@@ -237,6 +236,14 @@ public:return db;}+ const std::string& get_state_dir_path() const {+ return state_dir_path;+ }++ const rocksdb::Options &get_db_options() const {+ return options;+ }+void print_memory_usage() {std::string index_usage;db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex dbc80236..11fba173 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -48,16 +48,17 @@ void CollectionManager::add_to_collections(Collection* collection) {collection_id_names.emplace(collection->get_collection_id(), collection->get_name());}-Option<bool> CollectionManager::init(Store *store,- const size_t default_num_indices,- const std::string & auth_key,- const std::string & search_only_auth_key,- const size_t init_batch_size) {+void CollectionManager::init(Store *store,+ const size_t default_num_indices,+ const std::string & auth_key,+ const std::string & search_only_auth_key) {this->store = store;this->auth_key = auth_key;this->search_only_auth_key = search_only_auth_key;this->default_num_indices = default_num_indices;+}+Option<bool> CollectionManager::load(const size_t init_batch_size) {std::string next_collection_id_str;StoreStatus next_coll_id_status = store->get(NEXT_COLLECTION_ID_KEY, next_collection_id_str);@@ -143,7 +144,7 @@ Option<bool> CollectionManager::init(Store *store,num_docs_read++;iter_batch[seq_id % collection->get_num_indices()].push_back(- index_record(0, seq_id, iter->value().ToString(), document)+ index_record(0, seq_id, iter->value().ToString(), document));if(num_docs_read % init_batch_size == 0) {@@ -194,6 +195,7 @@ Option<bool> CollectionManager::init(Store *store,return Option<bool>(true);}+void CollectionManager::dispose() {for(auto & name_collection: collections) {delete name_collection.second;diff --git a/src/file_utils.cpp b/src/file_utils.cppnew file mode 100644index 00000000..82035c08--- /dev/null+++ b/src/file_utils.cpp@@ -0,0 +1,104 @@++#include <butil/file_util.h>+#include <butil/files/file_enumerator.h>+#include <butil/string_printf.h>+#include <file_utils.h>+++bool directory_exists(const std::string & dir_path) {+ struct stat info;+ return stat(dir_path.c_str(), &info) == 0 && (info.st_mode & S_IFDIR);+}++bool file_exists(const std::string & file_path) {+ struct stat info;+ return stat(file_path.c_str(), &info) == 0;+}++// tries to hard link first+bool copy_dir(const std::string& from_path, const std::string& to_path) {+ struct stat from_stat;++ if (stat(from_path.c_str(), &from_stat) < 0 || !S_ISDIR(from_stat.st_mode)) {+ LOG(WARNING) << "stat " << from_path << " failed";+ return false;+ }++ if (!butil::CreateDirectory(butil::FilePath(to_path))) {+ LOG(WARNING) << "CreateDirectory " << to_path << " failed";+ return false;+ }++ butil::FileEnumerator dir_enum(butil::FilePath(from_path),false, butil::FileEnumerator::FILES);+ for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {+ std::string src_file(from_path);+ std::string dst_file(to_path);+ butil::string_appendf(&src_file, "/%s", name.BaseName().value().c_str());+ butil::string_appendf(&dst_file, "/%s", name.BaseName().value().c_str());++ if (0 != link(src_file.c_str(), dst_file.c_str())) {+ if (!butil::CopyFile(butil::FilePath(src_file), butil::FilePath(dst_file))) {+ LOG(WARNING) << "copy " << src_file << " to " << dst_file << " failed";+ return false;+ }+ }+ }++ return true;+}++bool mv_dir(const std::string& from_path, const std::string& to_path) {+ struct stat from_stat;++ if (stat(from_path.c_str(), &from_stat) < 0 || !S_ISDIR(from_stat.st_mode)) {+ LOG(WARNING) << "stat " << from_path << " failed";+ return false;+ }++ if (!butil::CreateDirectory(butil::FilePath(to_path))) {+ LOG(WARNING) << "CreateDirectory " << to_path << " failed";+ return false;+ }++ butil::FileEnumerator file_enum(butil::FilePath(from_path), false, butil::FileEnumerator::FILES+ | butil::FileEnumerator::DIRECTORIES);+ for (butil::FilePath name = file_enum.Next(); !name.empty(); name = file_enum.Next()) {+ std::string src_file(from_path);+ std::string dst_file(to_path);++ if(name.value() == to_path) {+ // handle edge case when moving a directory into a subdirectory+ continue;+ }++ butil::string_appendf(&src_file, "/%s", name.BaseName().value().c_str());+ butil::string_appendf(&dst_file, "/%s", name.BaseName().value().c_str());++ butil::File::Error error;+ if (!butil::ReplaceFile(butil::FilePath(src_file), butil::FilePath(dst_file), &error)) {+ LOG(WARNING) << "move " << src_file << " to " << dst_file << " failed: " << error;+ return false;+ }+ }++ return true;+}++bool rename_path(const std::string& from_path, const std::string& to_path) {+ return butil::Move(butil::FilePath(from_path), butil::FilePath(to_path));+}++bool delete_path(const std::string& path, bool recursive) {+ return butil::DeleteFile(butil::FilePath(path), recursive);+}++bool dir_enum_count(const std::string &path) {+ size_t count = 0;+ butil::FileEnumerator file_enum(butil::FilePath(path), false, butil::FileEnumerator::FILES+ | butil::FileEnumerator::DIRECTORIES);+ for (butil::FilePath name = file_enum.Next(); !name.empty(); name = file_enum.Next()) {+ count++;+ }++ return count;+}diff --git a/src/http_server.cpp b/src/http_server.cppindex 38ef650d..2bb8d3e3 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -23,7 +23,12 @@ HttpServer::HttpServer(const std::string & version, const std::string & listen_ah2o_config_init(&config);hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);register_handler(hostconf, "/", catch_all_handler);++ signal(SIGPIPE, SIG_IGN);+ h2o_context_init(&ctx, h2o_evloop_create(), &config);+message_dispatcher = new http_message_dispatcher;+ message_dispatcher->init(ctx.loop);}void HttpServer::on_accept(h2o_socket_t *listener, const char *err) {@@ -116,11 +121,7 @@ int HttpServer::create_listener(void) {}int HttpServer::run(ReplicationState* replication_state) {- signal(SIGPIPE, SIG_IGN);- h2o_context_init(&ctx, h2o_evloop_create(), &config);-this->replication_state = replication_state;- message_dispatcher->init(ctx.loop);if (create_listener() != 0) {LOG(ERR) << "Failed to listen on " << listen_address << ":" << listen_port << " - " << strerror(errno);diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 345bffa1..c3c25027 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -1,7 +1,10 @@+#include "store.h"#include "raft_server.h"#include <butil/files/file_enumerator.h>#include <thread>#include <string_utils.h>+#include <file_utils.h>+#include <collection_manager.h>#include "rocksdb/utilities/checkpoint.h"@@ -21,7 +24,8 @@ void ReplicationClosure::Run() {// State machine implementationint ReplicationState::start(int port, int election_timeout_ms, int snapshot_interval_s,- const std::string & data_path, const std::string & peers) {+ const std::string & raft_dir, const std::string & peers) {+butil::EndPoint addr(butil::my_ip(), port);braft::NodeOptions node_options;@@ -34,20 +38,50 @@ int ReplicationState::start(int port, int election_timeout_ms, int snapshot_intenode_options.fsm = this;node_options.node_owns_fsm = false;node_options.snapshot_interval_s = snapshot_interval_s;- std::string prefix = "local://" + data_path;- node_options.log_uri = prefix + "/log";- node_options.raft_meta_uri = prefix + "/raft_meta";- node_options.snapshot_uri = prefix + "/snapshot";+ std::string prefix = "local://" + raft_dir;+ node_options.log_uri = prefix + "/" + log_dir_name;+ node_options.raft_meta_uri = prefix + "/" + meta_dir_name;+ node_options.snapshot_uri = prefix + "/" + snapshot_dir_name;node_options.disable_cli = true;braft::Node* node = new braft::Node("default_group", braft::PeerId(addr));++ std::string snapshot_dir = raft_dir + "/" + snapshot_dir_name;+ bool snapshot_exists = dir_enum_count(snapshot_dir) > 0;++ if(snapshot_exists) {+ // we will be assured of on_snapshot() firing and we will wait for that to return the promise+ } else {+ LOG(INFO) << "Snapshot does not exist. We will remove db dir and init db fresh.";++ if (!butil::DeleteFile(butil::FilePath(db_path), true)) {+ LOG(WARNING) << "rm " << db_path << " failed";+ return -1;+ }++ init_db(); // TODO: handle error+ }+if (node->init(node_options) != 0) {LOG(ERROR) << "Fail to init raft node";delete node;return -1;}- _node = node;+ std::vector<std::string> peer_vec;+ StringUtils::split(peers, peer_vec, ",");++ if(peer_vec.size() == 1) {+ // NOTE: `reset_peers` is NOT safe to run on a cluster of nodes, but okay for standalone+ braft::Configuration conf;+ conf.parse_from(peers);+ auto status = node->reset_peers(conf);+ if(!status.ok()) {+ LOG(ERROR) << status.error_str();+ }+ }++ this->node = node;return 0;}@@ -57,8 +91,6 @@ void ReplicationState::write(http_req* request, http_res* response) {// TODO: return redirect(response);}- LOG(INFO) << "*** write thread id: " << std::this_thread::get_id();-// Serialize request to replicated WAL so that all the peers in the group receive it as well.// NOTE: actual write must be done only on the `on_apply` method to maintain consistency.@@ -73,10 +105,10 @@ void ReplicationState::write(http_req* request, http_res* response) {task.done = new ReplicationClosure(request, response);// To avoid ABA problem- task.expected_term = _leader_term.load(butil::memory_order_relaxed);+ task.expected_term = leader_term.load(butil::memory_order_relaxed);// Now the task is applied to the group, waiting for the result.- return _node->apply(task);+ return node->apply(task);}void ReplicationState::read(http_res* response) {@@ -93,8 +125,8 @@ void ReplicationState::read(http_res* response) {}void ReplicationState::redirect(http_res* response) {- if (_node) {- braft::PeerId leader = _node->leader_id();+ if (node) {+ braft::PeerId leader = node->leader_id();if (!leader.is_empty()) {// TODO: response->set_redirect(leader.to_string());}@@ -126,6 +158,13 @@ void ReplicationState::on_apply(braft::Iterator& iter) {request = remote_request;}+ if(request->http_method == "SELF") {+ // We attempt to trigger a cold snapshot against an existing stand-alone DB for backward compatibility+ InitSnapshotClosure* init_snapshot_closure = new InitSnapshotClosure(ready);+ node->snapshot(init_snapshot_closure);+ return ;+ }+// Now that the log has been parsed, perform the actual operation// Call http server thread for write and response back to client (if `response` is NOT null)// We use a future to block current thread until the async flow finishes@@ -134,9 +173,7 @@ void ReplicationState::on_apply(braft::Iterator& iter) {std::future<bool> future = promise.get_future();auto replication_arg = new AsyncIndexArg{request, response, &promise};message_dispatcher->send_message(REPLICATION_MSG, replication_arg);-- bool value = future.get();-+ future.get();}}@@ -145,33 +182,33 @@ void* ReplicationState::save_snapshot(void* arg) {std::unique_ptr<SnapshotArg> arg_guard(sa);brpc::ClosureGuard done_guard(sa->done);- std::string snapshot_path = sa->writer->get_path() + "/rocksdb_snapshot";+ std::string snapshot_path = sa->writer->get_path() + "/" + db_snapshot_name;- rocksdb::Checkpoint* checkpoint = NULL;+ rocksdb::Checkpoint* checkpoint = nullptr;rocksdb::Status status = rocksdb::Checkpoint::Create(sa->db, &checkpoint);if(!status.ok()) {LOG(WARNING) << "Checkpoint Create failed, msg:" << status.ToString();- return NULL;+ return nullptr;}std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);status = checkpoint->CreateCheckpoint(snapshot_path);if(!status.ok()) {LOG(WARNING) << "Checkpoint CreateCheckpoint failed, msg:" << status.ToString();- return NULL;+ return nullptr;}butil::FileEnumerator dir_enum(butil::FilePath(snapshot_path),false, butil::FileEnumerator::FILES);for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {- std::string file_name = "rocksdb_snapshot/" + name.BaseName().value();+ std::string file_name = std::string(db_snapshot_name) + "/" + name.BaseName().value();if (sa->writer->add_file(file_name) != 0) {sa->done->status().set_error(EIO, "Fail to add file to writer.");- return NULL;+ return nullptr;}}- return NULL;+ return nullptr;}void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done) {@@ -184,52 +221,54 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Clbthread_start_urgent(&tid, NULL, save_snapshot, arg);}-bool ReplicationState::copy_snapshot(const std::string& from_path, const std::string& to_path) {- struct stat from_stat;+int ReplicationState::init_db() {+ if (db != nullptr) {+ delete db;+ db = nullptr;+ }++ if (!butil::CreateDirectory(butil::FilePath(db_path))) {+ LOG(WARNING) << "CreateDirectory " << db_path << " failed";+ return -1;+ }- if (stat(from_path.c_str(), &from_stat) < 0 || !S_ISDIR(from_stat.st_mode)) {- LOG(WARNING) << "stat " << from_path << " failed";- return false;+ db_options.create_if_missing = true;+ rocksdb::Status status = rocksdb::DB::Open(db_options, db_path, &db);+ if (!status.ok()) {+ LOG(WARNING) << "Open DB " << db_path << " failed, msg: " << status.ToString();+ return -1;}- if (!butil::CreateDirectory(butil::FilePath(to_path))) {- LOG(WARNING) << "CreateDirectory " << to_path << " failed";- return false;+ LOG(NOTICE) << "DB open success!";+ LOG(INFO) << "Loading collections from disk...";++ Option<bool> init_op = CollectionManager::get_instance().load();++ if(init_op.ok()) {+ LOG(INFO) << "Finished loading collections from disk.";+ } else {+ LOG(ERROR)<< "Typesense failed to start. " << "Could not load collections from disk: " << init_op.error();+ return 1;}- butil::FileEnumerator dir_enum(butil::FilePath(from_path),false, butil::FileEnumerator::FILES);- for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {- std::string src_file(from_path);- std::string dst_file(to_path);- butil::string_appendf(&src_file, "/%s", name.BaseName().value().c_str());- butil::string_appendf(&dst_file, "/%s", name.BaseName().value().c_str());-- if (0 != link(src_file.c_str(), dst_file.c_str())) {- if (!butil::CopyFile(butil::FilePath(src_file), butil::FilePath(dst_file))) {- LOG(WARNING) << "copy " << src_file << " to " << dst_file << " failed";- return false;- }- }+ if(!has_initialized.load()) {+ has_initialized.store(true, butil::memory_order_release);+ ready->set_value(true);}- return true;+ return 0;}int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {- // reset rocksdb handle- if (db != NULL) {- delete db;- db = NULL;- }-CHECK(!is_leader()) << "Leader is not supposed to load snapshot";+ LOG(INFO) << "on_snapshot_load";+// Load snapshot from reader, replacing the running StateMachinestd::string snapshot_path = reader->get_path();- snapshot_path.append("/rocksdb_snapshot");+ snapshot_path.append(std::string("/") + db_snapshot_name);- std::string db_path = "./data/rocksdb_data"; // TODOif (!butil::DeleteFile(butil::FilePath(db_path), true)) {LOG(WARNING) << "rm " << db_path << " failed";return -1;@@ -238,27 +277,35 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {LOG(TRACE) << "rm " << db_path << " success";// tries to use link if possible, or else copies- if (!copy_snapshot(snapshot_path, db_path)) {+ if (!copy_dir(snapshot_path, db_path)) {LOG(WARNING) << "copy snapshot " << snapshot_path << " to " << db_path << " failed";return -1;}LOG(TRACE) << "copy snapshot " << snapshot_path << " to " << db_path << " success";- return 0;+ return init_db();}void ReplicationState::refresh_peers(const std::string & peers) {- if(db == NULL) {- LOG(ERROR) << "DB IS NULL!";- }-- if(_node && is_leader()) {+ if(node && is_leader()) {LOG(INFO) << "Refreshing peer config";+braft::Configuration conf;conf.parse_from(peers);+RefreshPeersClosure* refresh_peers_done = new RefreshPeersClosure;- _node->change_peers(conf, refresh_peers_done);+ node->change_peers(conf, refresh_peers_done);}}+rocksdb::DB *ReplicationState::get_db() const {+ return db;+}++ReplicationState::ReplicationState(Store *store, http_message_dispatcher *message_dispatcher, std::promise<bool> *ready):+ node(nullptr), leader_term(-1), message_dispatcher(message_dispatcher), has_initialized(false), ready(ready) {+ db = store->_get_db_unsafe();+ db_path = store->get_state_dir_path();+ db_options = store->get_db_options();+}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex b63f080e..63c6267a 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -10,6 +10,7 @@#include "core_api.h"#include "typesense_server_utils.h"+#include "file_utils.h"HttpServer* server;@@ -19,16 +20,6 @@ void catch_interrupt(int sig) {server->stop();}-bool directory_exists(const std::string & dir_path) {- struct stat info;- return stat(dir_path.c_str(), &info) == 0 && (info.st_mode & S_IFDIR);-}--bool file_exists(const std::string & file_path) {- struct stat info;- return stat(file_path.c_str(), &info) == 0;-}-Option<std::string> fetch_file_contents(const std::string & file_path) {if(!file_exists(file_path)) {return Option<std::string>(404, "Error reading file containing raft peers.");@@ -95,9 +86,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<std::string>("listen-address", 'h', "Address to which Typesense server binds.", false, "0.0.0.0");options.add<uint32_t>("listen-port", 'p', "Port on which Typesense server listens.", false, 8108);- options.add<std::string>("raft-dir", '\0', "Directory where raft data will be stored.", false);options.add<uint32_t>("raft-port", '\0', "Port on which Typesense raft service listens.", false, 8107);- options.add<std::string>("raft-peers", '\0', "Path to file having comma separated string of Raft node IPs.", false);+ options.add<std::string>("raft-peers", '\0', "Path to file with comma separated string of Raft node IPs.", false);options.add<std::string>("master", 'm', "To start the server as read-only replica, ""provide the master's address in http(s)://<master_address>:<master_port> format.",@@ -152,6 +142,77 @@ bool on_send_response(void *data) {return true;}+int start_raft_server(ReplicationState& replication_state, const std::string& state_dir,+ const std::string& path_to_peers, uint32_t raft_port) {+ // TODO: early returns must quit parent process as well++ const Option<std::string> & peers_op = fetch_file_contents(path_to_peers);+ if(!peers_op.ok()) {+ LOG(ERR) << peers_op.error();+ return -1;+ }++ const std::string & peer_ips_string = peers_op.get();++ if(peer_ips_string.empty()) {+ LOG(ERR) << "File containing raft peers is empty.";+ return -1;+ }++ // start raft server+ brpc::Server raft_server;++ if (braft::add_service(&raft_server, raft_port) != 0) {+ LOG(ERR) << "Failed to add raft service";+ return -1;+ }++ if (raft_server.Start(raft_port, NULL) != 0) {+ LOG(ERR) << "Failed to start raft server";+ return -1;+ }++ std::vector<std::string> peer_ips;+ StringUtils::split(peer_ips_string, peer_ips, ",");+ std::string peers = StringUtils::join(peer_ips, ":0,");++ if (replication_state.start(raft_port, 1000, 60, state_dir, peers) != 0) {+ LOG(ERR) << "Failed to start raft state";+ return -1;+ }++ LOG(INFO) << "Typesense raft service is running on " << raft_server.listen_address();++ // Wait until 'CTRL-C' is pressed. then Stop() and Join() the service+ size_t raft_counter = 0;+ while (!brpc::IsAskedToQuit()) {+ if(++raft_counter % 10 == 0) {+ // reset peer configuration periodically to identify change in cluster membership+ const Option<std::string> & refreshed_peers_op = fetch_file_contents(path_to_peers);+ if(!peers_op.ok()) {+ LOG(ERR) << "Error while refreshing peer configuration: " << refreshed_peers_op.error();+ continue;+ }++ const std::string & refreshed_peer_ips_string = refreshed_peers_op.get();+ replication_state.refresh_peers(refreshed_peer_ips_string);+ }++ sleep(1);+ }++ LOG(INFO) << "Typesense raft service is going to quit";++ // Stop counter before server+ replication_state.shutdown();+ raft_server.Stop(0);++ // Wait until all the processing tasks are over.+ replication_state.join();+ raft_server.Join();+ return 0;+}+int run_server(const Config & config, const std::string & version,void (*master_server_routes)(), void (*replica_server_routes)()) {@@ -163,22 +224,24 @@ int run_server(const Config & config, const std::string & version,return 1;}- Store store(config.get_data_dir());+ std::string data_dir = config.get_data_dir();+ std::string db_dir = config.get_data_dir() + "/db";+ std::string state_dir = config.get_data_dir() + "/state";- LOG(INFO) << "Loading collections from disk...";+ if(!directory_exists(db_dir) && file_exists(data_dir+"/CURRENT") && file_exists(data_dir+"/IDENTITY")) {+ LOG(INFO) << "Migrating contents of data directory in a `db` sub-directory, as per the new data layout.";+ bool moved = mv_dir(data_dir, db_dir);+ if(!moved) {+ LOG(ERR) << "CRITICAL ERROR! Failed to migrate all files in the data directory into a `db` sub-directory.";+ LOG(ERR) << "NOTE: Please move remaining files manually. Failure to do so **WILL** lead to **DATA LOSS**.";+ return 1;+ }+ }+ Store store(db_dir);CollectionManager & collectionManager = CollectionManager::get_instance();- Option<bool> init_op = collectionManager.init(&store,- config.get_indices_per_collection(),- config.get_api_key(),- config.get_search_only_api_key());-- if(init_op.ok()) {- LOG(INFO) << "Finished loading collections from disk.";- } else {- LOG(ERR)<< "Typesense failed to start. " << "Could not load collections from disk: " << init_op.error();- return 1;- }+ collectionManager.init(&store, config.get_indices_per_collection(),+ config.get_api_key(), config.get_search_only_api_key());curl_global_init(CURL_GLOBAL_SSL);@@ -197,6 +260,22 @@ int run_server(const Config & config, const std::string & version,server->on(REPLICATION_EVENT_MSG, Replicator::on_replication_event);server->on(ReplicationState::REPLICATION_MSG, async_write_request);+ // first we start the raft service+ std::promise<bool> ready_promise;+ std::future<bool> ready_future = ready_promise.get_future();++ ReplicationState replication_state(&store, server->get_message_dispatcher(), &ready_promise);++ std::thread raft_thread([&replication_state, &config, &state_dir]() {+ std::string path_to_peers = config.get_raft_peers();+ start_raft_server(replication_state, state_dir, path_to_peers, config.get_raft_port());+ });++ raft_thread.detach();++ // wait for raft service to be ready before starting http+ ready_future.get();+if(config.get_master().empty()) {master_server_routes();} else {@@ -220,86 +299,6 @@ int run_server(const Config & config, const std::string & version,replication_thread.detach();}- ReplicationState replication_state(server->get_message_dispatcher(), store._get_db_unsafe());-- std::thread raft_thread([&replication_state, &config]() {- std::string path_to_peers = config.get_raft_peers();-- if(path_to_peers.empty()) {- return 0;- }-- const Option<std::string> & peers_op = fetch_file_contents(path_to_peers);- if(!peers_op.ok()) {- LOG(ERR) << peers_op.error();- return -1;- }-- const std::string & peer_ips_string = peers_op.get();-- if(peer_ips_string.empty()) {- LOG(ERR) << "File containing raft peers is empty.";- return -1;- }-- // start raft server- brpc::Server raft_server;-- uint32_t raft_port = config.get_raft_port();-- if (braft::add_service(&raft_server, raft_port) != 0) {- LOG(ERR) << "Failed to add raft service";- return -1; // TODO: must quit parent process as well- }-- if (raft_server.Start(raft_port, NULL) != 0) {- LOG(ERR) << "Failed to start raft server";- return -1;- }-- std::vector<std::string> peer_ips;- StringUtils::split(peer_ips_string, peer_ips, ",");- std::string peers = StringUtils::join(peer_ips, ":0,");-- if (replication_state.start(raft_port, 1000, 60, config.get_raft_dir(), peers) != 0) {- LOG(ERR) << "Failed to start raft state";- return -1;- }-- LOG(INFO) << "Typesense raft service is running on " << raft_server.listen_address();-- // Wait until 'CTRL-C' is pressed. then Stop() and Join() the service- size_t raft_counter = 0;- while (!brpc::IsAskedToQuit()) {- if(++raft_counter % 10 == 0) {- // reset peer configuration periodically to identify change in cluster membership- const Option<std::string> & refreshed_peers_op = fetch_file_contents(path_to_peers);- if(!peers_op.ok()) {- LOG(ERR) << "Error while refreshing peer configuration: " << refreshed_peers_op.error();- continue;- }-- const std::string & refreshed_peer_ips_string = refreshed_peers_op.get();- replication_state.refresh_peers(refreshed_peer_ips_string);- }-- sleep(1);- }-- LOG(INFO) << "Typesense raft service is going to quit";-- // Stop counter before server- replication_state.shutdown();- raft_server.Stop(0);-- // Wait until all the processing tasks are over.- replication_state.join();- raft_server.Join();- return 0;- });-- raft_thread.detach();-int ret_code = server->run(&replication_state);// we are out of the event loop here