Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle presence/absence of snapshot.

kishorenca52ec8099919
diff --git a/include/collection_manager.h b/include/collection_manager.h
index ce8cdb9a..c059bd57 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -56,11 +56,13 @@ public:
CollectionManager(CollectionManager const&) = delete;
void operator=(CollectionManager const&) = delete;
- Option<bool> init(Store *store,
- const size_t default_num_indices,
- const std::string & auth_key,
- const std::string & search_only_auth_key,
- const size_t init_batch_size=1000);
+ void init(Store *store,
+ const size_t default_num_indices,
+ const std::string & auth_key,
+ const std::string & search_only_auth_key
+ );
+
+ Option<bool> load(const size_t init_batch_size=1000);
// frees in-memory data structures when server is shutdown - helps us run a memory leak detecter properly
void dispose();
diff --git a/include/config.h b/include/config.h
index dd51cd4b..401b9333 100644
--- a/include/config.h
+++ b/include/config.h
@@ -16,7 +16,6 @@ private:
std::string listen_address;
uint32_t listen_port;
- std::string raft_dir;
uint32_t raft_port;
std::string raft_peers;
@@ -92,10 +91,6 @@ public:
this->raft_port = raft_port;
}
- void set_raft_dir(const std::string & raft_dir) {
- this->raft_dir = raft_dir;
- }
-
void set_raft_peers(const std::string & raft_peers) {
this->raft_peers = raft_peers;
}
@@ -154,10 +149,6 @@ public:
return this->raft_port;
}
- std::string get_raft_dir() const {
- return this->raft_dir;
- }
-
std::string get_raft_peers() const {
return this->raft_peers;
}
@@ -191,7 +182,6 @@ public:
this->raft_port = std::stoi(get_env("TYPESENSE_RAFT_PORT"));
}
- this->raft_dir = get_env("TYPESENSE_RAFT_DIR");
this->raft_peers = get_env("TYPESENSE_RAFT_PEERS");
this->master = get_env("TYPESENSE_MASTER");
@@ -266,10 +256,6 @@ public:
this->raft_port = reader.GetInteger("server", "raft-port", 8107);
}
- if(reader.Exists("server", "raft-dir")) {
- this->raft_dir = reader.Get("server", "raft-dir", "");
- }
-
if(reader.Exists("server", "raft-peers")) {
this->raft_peers = reader.Get("server", "raft-peers", "");
}
@@ -320,10 +306,6 @@ public:
this->raft_port = options.get<uint32_t>("raft-port");
}
- if(options.exist("raft-dir")) {
- this->raft_dir = options.get<std::string>("raft-dir");
- }
-
if(options.exist("raft-peers")) {
this->raft_peers = options.get<std::string>("raft-peers");
}
@@ -344,14 +326,6 @@ public:
return Option<bool>(500, "API key is not specified.");
}
- if(!raft_dir.empty() && raft_peers.empty()) {
- return Option<bool>(500, "Argument --raft-peers is not specified.");
- }
-
- if(!raft_peers.empty() && raft_dir.empty()) {
- return Option<bool>(500, "Argument --raft-dir is not specified.");
- }
-
return Option<bool>(true);
}
};
diff --git a/include/file_utils.h b/include/file_utils.h
new file mode 100644
index 00000000..e7bba268
--- /dev/null
+++ b/include/file_utils.h
@@ -0,0 +1,14 @@
+bool directory_exists(const std::string & dir_path);
+
+bool file_exists(const std::string & file_path);
+
+// tries to hard link first
+bool copy_dir(const std::string& from_path, const std::string& to_path);
+
+bool mv_dir(const std::string& from_path, const std::string& to_path);
+
+bool rename_path(const std::string& from_path, const std::string& to_path);
+
+bool delete_path(const std::string& path, bool recursive = true);
+
+bool dir_enum_count(const std::string & path);
\ No newline at end of file
diff --git a/include/http_client.h b/include/http_client.h
index e7b0bb1a..bc141bfd 100644
--- a/include/http_client.h
+++ b/include/http_client.h
@@ -2,6 +2,7 @@
#include <string>
#include <curl/curl.h>
+#include "file_utils.h"
/*
NOTE: This is a really primitive blocking client meant only for specific Typesense use cases.
diff --git a/include/raft_server.h b/include/raft_server.h
index 784a2eca..49d73437 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -11,6 +11,7 @@
#include "http_data.h"
+class Store;
// Implements the callback for the state machine
class ReplicationClosure : public braft::Closure {
@@ -56,29 +57,61 @@ public:
}
};
+// Closure that fires when initial snapshot operation finishes
+class InitSnapshotClosure : public braft::Closure {
+private:
+ std::promise<bool>* ready;
+public:
+
+ InitSnapshotClosure(std::promise<bool>* ready): ready(ready) {}
+
+ ~InitSnapshotClosure() {}
+
+ void Run() {
+ // Auto delete this after Run()
+ std::unique_ptr<InitSnapshotClosure> self_guard(this);
+
+ if(status().ok()) {
+ LOG(INFO) << "Init snapshot succeeded!";
+ } else {
+ LOG(ERROR) << "Init snapshot failed, error: " << status().error_str();
+ }
+ }
+};
+
+
// Implements braft::StateMachine.
class ReplicationState : public braft::StateMachine {
private:
- braft::Node* volatile _node;
- butil::atomic<int64_t> _value;
- butil::atomic<int64_t> _leader_term;
+ static constexpr const char* db_snapshot_name = "db_snapshot";
+
+ braft::Node* volatile node;
+ butil::atomic<int64_t> leader_term;
+
rocksdb::DB* db;
+ std::string db_path;
+ rocksdb::Options db_options;
+
http_message_dispatcher* message_dispatcher;
+ butil::atomic<bool> has_initialized;
+ std::promise<bool>* ready;
+
public:
- ReplicationState(http_message_dispatcher* message_dispatcher, rocksdb::DB* db):
- _node(NULL), _value(0), _leader_term(-1), db(db),
- message_dispatcher(message_dispatcher) {
- }
+ static constexpr const char* log_dir_name = "log";
+ static constexpr const char* meta_dir_name = "meta";
+ static constexpr const char* snapshot_dir_name = "snapshot";
+
+ ReplicationState(Store* store, http_message_dispatcher* message_dispatcher, std::promise<bool>* ready);
~ReplicationState() {
- delete _node;
+ delete node;
}
// Starts this node
int start(int port, int election_timeout_ms, int snapshot_interval_s,
- const std::string & data_path, const std::string & peers);
+ const std::string & raft_dir, const std::string & peers);
// Generic write method for synchronizing all writes
void write(http_req* request, http_res* response);
@@ -90,23 +123,25 @@ public:
void refresh_peers(const std::string & peers);
bool is_leader() const {
- return _leader_term.load(butil::memory_order_acquire) > 0;
+ return leader_term.load(butil::memory_order_acquire) > 0;
}
// Shut this node down.
void shutdown() {
- if (_node) {
- _node->shutdown(NULL);
+ if (node) {
+ node->shutdown(nullptr);
}
}
// Blocking this thread until the node is eventually down.
void join() {
- if (_node) {
- _node->join();
+ if (node) {
+ node->join();
}
}
+ rocksdb::DB *get_db() const;
+
static constexpr const char* REPLICATION_MSG = "raft_replication";
private:
@@ -129,17 +164,27 @@ private:
void on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done);
- static bool copy_snapshot(const std::string& from_path, const std::string& to_path);
+ int init_db();
int on_snapshot_load(braft::SnapshotReader* reader);
void on_leader_start(int64_t term) {
- _leader_term.store(term, butil::memory_order_release);
+ leader_term.store(term, butil::memory_order_release);
+ /*if(!has_initialized.load()) {
+ has_initialized.store(true, butil::memory_order_release);
+ ready->set_value(true);
+ }*/
+
+ // have to write a dummy write, otherwise snapshot will not trigger
+ //http_req* request = new http_req(nullptr, "SELF", 0, {}, "INIT");
+ //http_res* response = new http_res();
+ //write(request, response);
+
LOG(INFO) << "Node becomes leader, term: " << term;
}
void on_leader_stop(const butil::Status& status) {
- _leader_term.store(-1, butil::memory_order_release);
+ leader_term.store(-1, butil::memory_order_release);
LOG(INFO) << "Node stepped down : " << status;
}
@@ -155,11 +200,15 @@ private:
LOG(INFO) << "Configuration of this group is " << conf;
}
- void on_stop_following(const ::braft::LeaderChangeContext& ctx) {
- LOG(INFO) << "Node stops following " << ctx;
- }
-
void on_start_following(const ::braft::LeaderChangeContext& ctx) {
+ if(!has_initialized.load()) {
+ has_initialized.store(true, butil::memory_order_release);
+ ready->set_value(true);
+ }
LOG(INFO) << "Node start following " << ctx;
}
+
+ void on_stop_following(const ::braft::LeaderChangeContext& ctx) {
+ LOG(INFO) << "Node stops following " << ctx;
+ }
};
diff --git a/include/store.h b/include/store.h
index bad416db..609ec7d0 100644
--- a/include/store.h
+++ b/include/store.h
@@ -44,7 +44,6 @@ class Store {
private:
const std::string state_dir_path;
-
rocksdb::DB *db;
rocksdb::Options options;
rocksdb::WriteOptions write_options;
@@ -237,6 +236,14 @@ public:
return db;
}
+ const std::string& get_state_dir_path() const {
+ return state_dir_path;
+ }
+
+ const rocksdb::Options &get_db_options() const {
+ return options;
+ }
+
void print_memory_usage() {
std::string index_usage;
db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index dbc80236..11fba173 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -48,16 +48,17 @@ void CollectionManager::add_to_collections(Collection* collection) {
collection_id_names.emplace(collection->get_collection_id(), collection->get_name());
}
-Option<bool> CollectionManager::init(Store *store,
- const size_t default_num_indices,
- const std::string & auth_key,
- const std::string & search_only_auth_key,
- const size_t init_batch_size) {
+void CollectionManager::init(Store *store,
+ const size_t default_num_indices,
+ const std::string & auth_key,
+ const std::string & search_only_auth_key) {
this->store = store;
this->auth_key = auth_key;
this->search_only_auth_key = search_only_auth_key;
this->default_num_indices = default_num_indices;
+}
+Option<bool> CollectionManager::load(const size_t init_batch_size) {
std::string next_collection_id_str;
StoreStatus next_coll_id_status = store->get(NEXT_COLLECTION_ID_KEY, next_collection_id_str);
@@ -143,7 +144,7 @@ Option<bool> CollectionManager::init(Store *store,
num_docs_read++;
iter_batch[seq_id % collection->get_num_indices()].push_back(
- index_record(0, seq_id, iter->value().ToString(), document)
+ index_record(0, seq_id, iter->value().ToString(), document)
);
if(num_docs_read % init_batch_size == 0) {
@@ -194,6 +195,7 @@ Option<bool> CollectionManager::init(Store *store,
return Option<bool>(true);
}
+
void CollectionManager::dispose() {
for(auto & name_collection: collections) {
delete name_collection.second;
diff --git a/src/file_utils.cpp b/src/file_utils.cpp
new file mode 100644
index 00000000..82035c08
--- /dev/null
+++ b/src/file_utils.cpp
@@ -0,0 +1,104 @@
+
+#include <butil/file_util.h>
+#include <butil/files/file_enumerator.h>
+#include <butil/string_printf.h>
+#include <file_utils.h>
+
+
+bool directory_exists(const std::string & dir_path) {
+ struct stat info;
+ return stat(dir_path.c_str(), &info) == 0 && (info.st_mode & S_IFDIR);
+}
+
+bool file_exists(const std::string & file_path) {
+ struct stat info;
+ return stat(file_path.c_str(), &info) == 0;
+}
+
+// tries to hard link first
+bool copy_dir(const std::string& from_path, const std::string& to_path) {
+ struct stat from_stat;
+
+ if (stat(from_path.c_str(), &from_stat) < 0 || !S_ISDIR(from_stat.st_mode)) {
+ LOG(WARNING) << "stat " << from_path << " failed";
+ return false;
+ }
+
+ if (!butil::CreateDirectory(butil::FilePath(to_path))) {
+ LOG(WARNING) << "CreateDirectory " << to_path << " failed";
+ return false;
+ }
+
+ butil::FileEnumerator dir_enum(butil::FilePath(from_path),false, butil::FileEnumerator::FILES);
+ for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {
+ std::string src_file(from_path);
+ std::string dst_file(to_path);
+ butil::string_appendf(&src_file, "/%s", name.BaseName().value().c_str());
+ butil::string_appendf(&dst_file, "/%s", name.BaseName().value().c_str());
+
+ if (0 != link(src_file.c_str(), dst_file.c_str())) {
+ if (!butil::CopyFile(butil::FilePath(src_file), butil::FilePath(dst_file))) {
+ LOG(WARNING) << "copy " << src_file << " to " << dst_file << " failed";
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+bool mv_dir(const std::string& from_path, const std::string& to_path) {
+ struct stat from_stat;
+
+ if (stat(from_path.c_str(), &from_stat) < 0 || !S_ISDIR(from_stat.st_mode)) {
+ LOG(WARNING) << "stat " << from_path << " failed";
+ return false;
+ }
+
+ if (!butil::CreateDirectory(butil::FilePath(to_path))) {
+ LOG(WARNING) << "CreateDirectory " << to_path << " failed";
+ return false;
+ }
+
+ butil::FileEnumerator file_enum(butil::FilePath(from_path), false, butil::FileEnumerator::FILES
+ | butil::FileEnumerator::DIRECTORIES);
+ for (butil::FilePath name = file_enum.Next(); !name.empty(); name = file_enum.Next()) {
+ std::string src_file(from_path);
+ std::string dst_file(to_path);
+
+ if(name.value() == to_path) {
+ // handle edge case when moving a directory into a subdirectory
+ continue;
+ }
+
+ butil::string_appendf(&src_file, "/%s", name.BaseName().value().c_str());
+ butil::string_appendf(&dst_file, "/%s", name.BaseName().value().c_str());
+
+ butil::File::Error error;
+ if (!butil::ReplaceFile(butil::FilePath(src_file), butil::FilePath(dst_file), &error)) {
+ LOG(WARNING) << "move " << src_file << " to " << dst_file << " failed: " << error;
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool rename_path(const std::string& from_path, const std::string& to_path) {
+ return butil::Move(butil::FilePath(from_path), butil::FilePath(to_path));
+}
+
+bool delete_path(const std::string& path, bool recursive) {
+ return butil::DeleteFile(butil::FilePath(path), recursive);
+}
+
+bool dir_enum_count(const std::string &path) {
+ size_t count = 0;
+ butil::FileEnumerator file_enum(butil::FilePath(path), false, butil::FileEnumerator::FILES
+ | butil::FileEnumerator::DIRECTORIES);
+ for (butil::FilePath name = file_enum.Next(); !name.empty(); name = file_enum.Next()) {
+ count++;
+ }
+
+ return count;
+}
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 38ef650d..2bb8d3e3 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -23,7 +23,12 @@ HttpServer::HttpServer(const std::string & version, const std::string & listen_a
h2o_config_init(&config);
hostconf = h2o_config_register_host(&config, h2o_iovec_init(H2O_STRLIT("default")), 65535);
register_handler(hostconf, "/", catch_all_handler);
+
+ signal(SIGPIPE, SIG_IGN);
+ h2o_context_init(&ctx, h2o_evloop_create(), &config);
+
message_dispatcher = new http_message_dispatcher;
+ message_dispatcher->init(ctx.loop);
}
void HttpServer::on_accept(h2o_socket_t *listener, const char *err) {
@@ -116,11 +121,7 @@ int HttpServer::create_listener(void) {
}
int HttpServer::run(ReplicationState* replication_state) {
- signal(SIGPIPE, SIG_IGN);
- h2o_context_init(&ctx, h2o_evloop_create(), &config);
-
this->replication_state = replication_state;
- message_dispatcher->init(ctx.loop);
if (create_listener() != 0) {
LOG(ERR) << "Failed to listen on " << listen_address << ":" << listen_port << " - " << strerror(errno);
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 345bffa1..c3c25027 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -1,7 +1,10 @@
+#include "store.h"
#include "raft_server.h"
#include <butil/files/file_enumerator.h>
#include <thread>
#include <string_utils.h>
+#include <file_utils.h>
+#include <collection_manager.h>
#include "rocksdb/utilities/checkpoint.h"
@@ -21,7 +24,8 @@ void ReplicationClosure::Run() {
// State machine implementation
int ReplicationState::start(int port, int election_timeout_ms, int snapshot_interval_s,
- const std::string & data_path, const std::string & peers) {
+ const std::string & raft_dir, const std::string & peers) {
+
butil::EndPoint addr(butil::my_ip(), port);
braft::NodeOptions node_options;
@@ -34,20 +38,50 @@ int ReplicationState::start(int port, int election_timeout_ms, int snapshot_inte
node_options.fsm = this;
node_options.node_owns_fsm = false;
node_options.snapshot_interval_s = snapshot_interval_s;
- std::string prefix = "local://" + data_path;
- node_options.log_uri = prefix + "/log";
- node_options.raft_meta_uri = prefix + "/raft_meta";
- node_options.snapshot_uri = prefix + "/snapshot";
+ std::string prefix = "local://" + raft_dir;
+ node_options.log_uri = prefix + "/" + log_dir_name;
+ node_options.raft_meta_uri = prefix + "/" + meta_dir_name;
+ node_options.snapshot_uri = prefix + "/" + snapshot_dir_name;
node_options.disable_cli = true;
braft::Node* node = new braft::Node("default_group", braft::PeerId(addr));
+
+ std::string snapshot_dir = raft_dir + "/" + snapshot_dir_name;
+ bool snapshot_exists = dir_enum_count(snapshot_dir) > 0;
+
+ if(snapshot_exists) {
+ // we will be assured of on_snapshot() firing and we will wait for that to return the promise
+ } else {
+ LOG(INFO) << "Snapshot does not exist. We will remove db dir and init db fresh.";
+
+ if (!butil::DeleteFile(butil::FilePath(db_path), true)) {
+ LOG(WARNING) << "rm " << db_path << " failed";
+ return -1;
+ }
+
+ init_db(); // TODO: handle error
+ }
+
if (node->init(node_options) != 0) {
LOG(ERROR) << "Fail to init raft node";
delete node;
return -1;
}
- _node = node;
+ std::vector<std::string> peer_vec;
+ StringUtils::split(peers, peer_vec, ",");
+
+ if(peer_vec.size() == 1) {
+ // NOTE: `reset_peers` is NOT safe to run on a cluster of nodes, but okay for standalone
+ braft::Configuration conf;
+ conf.parse_from(peers);
+ auto status = node->reset_peers(conf);
+ if(!status.ok()) {
+ LOG(ERROR) << status.error_str();
+ }
+ }
+
+ this->node = node;
return 0;
}
@@ -57,8 +91,6 @@ void ReplicationState::write(http_req* request, http_res* response) {
// TODO: return redirect(response);
}
- LOG(INFO) << "*** write thread id: " << std::this_thread::get_id();
-
// Serialize request to replicated WAL so that all the peers in the group receive it as well.
// NOTE: actual write must be done only on the `on_apply` method to maintain consistency.
@@ -73,10 +105,10 @@ void ReplicationState::write(http_req* request, http_res* response) {
task.done = new ReplicationClosure(request, response);
// To avoid ABA problem
- task.expected_term = _leader_term.load(butil::memory_order_relaxed);
+ task.expected_term = leader_term.load(butil::memory_order_relaxed);
// Now the task is applied to the group, waiting for the result.
- return _node->apply(task);
+ return node->apply(task);
}
void ReplicationState::read(http_res* response) {
@@ -93,8 +125,8 @@ void ReplicationState::read(http_res* response) {
}
void ReplicationState::redirect(http_res* response) {
- if (_node) {
- braft::PeerId leader = _node->leader_id();
+ if (node) {
+ braft::PeerId leader = node->leader_id();
if (!leader.is_empty()) {
// TODO: response->set_redirect(leader.to_string());
}
@@ -126,6 +158,13 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
request = remote_request;
}
+ if(request->http_method == "SELF") {
+ // We attempt to trigger a cold snapshot against an existing stand-alone DB for backward compatibility
+ InitSnapshotClosure* init_snapshot_closure = new InitSnapshotClosure(ready);
+ node->snapshot(init_snapshot_closure);
+ return ;
+ }
+
// Now that the log has been parsed, perform the actual operation
// Call http server thread for write and response back to client (if `response` is NOT null)
// We use a future to block current thread until the async flow finishes
@@ -134,9 +173,7 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
std::future<bool> future = promise.get_future();
auto replication_arg = new AsyncIndexArg{request, response, &promise};
message_dispatcher->send_message(REPLICATION_MSG, replication_arg);
-
- bool value = future.get();
-
+ future.get();
}
}
@@ -145,33 +182,33 @@ void* ReplicationState::save_snapshot(void* arg) {
std::unique_ptr<SnapshotArg> arg_guard(sa);
brpc::ClosureGuard done_guard(sa->done);
- std::string snapshot_path = sa->writer->get_path() + "/rocksdb_snapshot";
+ std::string snapshot_path = sa->writer->get_path() + "/" + db_snapshot_name;
- rocksdb::Checkpoint* checkpoint = NULL;
+ rocksdb::Checkpoint* checkpoint = nullptr;
rocksdb::Status status = rocksdb::Checkpoint::Create(sa->db, &checkpoint);
if(!status.ok()) {
LOG(WARNING) << "Checkpoint Create failed, msg:" << status.ToString();
- return NULL;
+ return nullptr;
}
std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);
status = checkpoint->CreateCheckpoint(snapshot_path);
if(!status.ok()) {
LOG(WARNING) << "Checkpoint CreateCheckpoint failed, msg:" << status.ToString();
- return NULL;
+ return nullptr;
}
butil::FileEnumerator dir_enum(butil::FilePath(snapshot_path),false, butil::FileEnumerator::FILES);
for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {
- std::string file_name = "rocksdb_snapshot/" + name.BaseName().value();
+ std::string file_name = std::string(db_snapshot_name) + "/" + name.BaseName().value();
if (sa->writer->add_file(file_name) != 0) {
sa->done->status().set_error(EIO, "Fail to add file to writer.");
- return NULL;
+ return nullptr;
}
}
- return NULL;
+ return nullptr;
}
void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done) {
@@ -184,52 +221,54 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Cl
bthread_start_urgent(&tid, NULL, save_snapshot, arg);
}
-bool ReplicationState::copy_snapshot(const std::string& from_path, const std::string& to_path) {
- struct stat from_stat;
+int ReplicationState::init_db() {
+ if (db != nullptr) {
+ delete db;
+ db = nullptr;
+ }
+
+ if (!butil::CreateDirectory(butil::FilePath(db_path))) {
+ LOG(WARNING) << "CreateDirectory " << db_path << " failed";
+ return -1;
+ }
- if (stat(from_path.c_str(), &from_stat) < 0 || !S_ISDIR(from_stat.st_mode)) {
- LOG(WARNING) << "stat " << from_path << " failed";
- return false;
+ db_options.create_if_missing = true;
+ rocksdb::Status status = rocksdb::DB::Open(db_options, db_path, &db);
+ if (!status.ok()) {
+ LOG(WARNING) << "Open DB " << db_path << " failed, msg: " << status.ToString();
+ return -1;
}
- if (!butil::CreateDirectory(butil::FilePath(to_path))) {
- LOG(WARNING) << "CreateDirectory " << to_path << " failed";
- return false;
+ LOG(NOTICE) << "DB open success!";
+ LOG(INFO) << "Loading collections from disk...";
+
+ Option<bool> init_op = CollectionManager::get_instance().load();
+
+ if(init_op.ok()) {
+ LOG(INFO) << "Finished loading collections from disk.";
+ } else {
+ LOG(ERROR)<< "Typesense failed to start. " << "Could not load collections from disk: " << init_op.error();
+ return 1;
}
- butil::FileEnumerator dir_enum(butil::FilePath(from_path),false, butil::FileEnumerator::FILES);
- for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {
- std::string src_file(from_path);
- std::string dst_file(to_path);
- butil::string_appendf(&src_file, "/%s", name.BaseName().value().c_str());
- butil::string_appendf(&dst_file, "/%s", name.BaseName().value().c_str());
-
- if (0 != link(src_file.c_str(), dst_file.c_str())) {
- if (!butil::CopyFile(butil::FilePath(src_file), butil::FilePath(dst_file))) {
- LOG(WARNING) << "copy " << src_file << " to " << dst_file << " failed";
- return false;
- }
- }
+ if(!has_initialized.load()) {
+ has_initialized.store(true, butil::memory_order_release);
+ ready->set_value(true);
}
- return true;
+ return 0;
}
int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
- // reset rocksdb handle
- if (db != NULL) {
- delete db;
- db = NULL;
- }
-
CHECK(!is_leader()) << "Leader is not supposed to load snapshot";
+ LOG(INFO) << "on_snapshot_load";
+
// Load snapshot from reader, replacing the running StateMachine
std::string snapshot_path = reader->get_path();
- snapshot_path.append("/rocksdb_snapshot");
+ snapshot_path.append(std::string("/") + db_snapshot_name);
- std::string db_path = "./data/rocksdb_data"; // TODO
if (!butil::DeleteFile(butil::FilePath(db_path), true)) {
LOG(WARNING) << "rm " << db_path << " failed";
return -1;
@@ -238,27 +277,35 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
LOG(TRACE) << "rm " << db_path << " success";
// tries to use link if possible, or else copies
- if (!copy_snapshot(snapshot_path, db_path)) {
+ if (!copy_dir(snapshot_path, db_path)) {
LOG(WARNING) << "copy snapshot " << snapshot_path << " to " << db_path << " failed";
return -1;
}
LOG(TRACE) << "copy snapshot " << snapshot_path << " to " << db_path << " success";
- return 0;
+ return init_db();
}
void ReplicationState::refresh_peers(const std::string & peers) {
- if(db == NULL) {
- LOG(ERROR) << "DB IS NULL!";
- }
-
- if(_node && is_leader()) {
+ if(node && is_leader()) {
LOG(INFO) << "Refreshing peer config";
+
braft::Configuration conf;
conf.parse_from(peers);
+
RefreshPeersClosure* refresh_peers_done = new RefreshPeersClosure;
- _node->change_peers(conf, refresh_peers_done);
+ node->change_peers(conf, refresh_peers_done);
}
}
+rocksdb::DB *ReplicationState::get_db() const {
+ return db;
+}
+
+ReplicationState::ReplicationState(Store *store, http_message_dispatcher *message_dispatcher, std::promise<bool> *ready):
+ node(nullptr), leader_term(-1), message_dispatcher(message_dispatcher), has_initialized(false), ready(ready) {
+ db = store->_get_db_unsafe();
+ db_path = store->get_state_dir_path();
+ db_options = store->get_db_options();
+}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index b63f080e..63c6267a 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -10,6 +10,7 @@
#include "core_api.h"
#include "typesense_server_utils.h"
+#include "file_utils.h"
HttpServer* server;
@@ -19,16 +20,6 @@ void catch_interrupt(int sig) {
server->stop();
}
-bool directory_exists(const std::string & dir_path) {
- struct stat info;
- return stat(dir_path.c_str(), &info) == 0 && (info.st_mode & S_IFDIR);
-}
-
-bool file_exists(const std::string & file_path) {
- struct stat info;
- return stat(file_path.c_str(), &info) == 0;
-}
-
Option<std::string> fetch_file_contents(const std::string & file_path) {
if(!file_exists(file_path)) {
return Option<std::string>(404, "Error reading file containing raft peers.");
@@ -95,9 +86,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("listen-address", 'h', "Address to which Typesense server binds.", false, "0.0.0.0");
options.add<uint32_t>("listen-port", 'p', "Port on which Typesense server listens.", false, 8108);
- options.add<std::string>("raft-dir", '\0', "Directory where raft data will be stored.", false);
options.add<uint32_t>("raft-port", '\0', "Port on which Typesense raft service listens.", false, 8107);
- options.add<std::string>("raft-peers", '\0', "Path to file having comma separated string of Raft node IPs.", false);
+ options.add<std::string>("raft-peers", '\0', "Path to file with comma separated string of Raft node IPs.", false);
options.add<std::string>("master", 'm', "To start the server as read-only replica, "
"provide the master's address in http(s)://<master_address>:<master_port> format.",
@@ -152,6 +142,77 @@ bool on_send_response(void *data) {
return true;
}
+int start_raft_server(ReplicationState& replication_state, const std::string& state_dir,
+ const std::string& path_to_peers, uint32_t raft_port) {
+ // TODO: early returns must quit parent process as well
+
+ const Option<std::string> & peers_op = fetch_file_contents(path_to_peers);
+ if(!peers_op.ok()) {
+ LOG(ERR) << peers_op.error();
+ return -1;
+ }
+
+ const std::string & peer_ips_string = peers_op.get();
+
+ if(peer_ips_string.empty()) {
+ LOG(ERR) << "File containing raft peers is empty.";
+ return -1;
+ }
+
+ // start raft server
+ brpc::Server raft_server;
+
+ if (braft::add_service(&raft_server, raft_port) != 0) {
+ LOG(ERR) << "Failed to add raft service";
+ return -1;
+ }
+
+ if (raft_server.Start(raft_port, NULL) != 0) {
+ LOG(ERR) << "Failed to start raft server";
+ return -1;
+ }
+
+ std::vector<std::string> peer_ips;
+ StringUtils::split(peer_ips_string, peer_ips, ",");
+ std::string peers = StringUtils::join(peer_ips, ":0,");
+
+ if (replication_state.start(raft_port, 1000, 60, state_dir, peers) != 0) {
+ LOG(ERR) << "Failed to start raft state";
+ return -1;
+ }
+
+ LOG(INFO) << "Typesense raft service is running on " << raft_server.listen_address();
+
+ // Wait until 'CTRL-C' is pressed. then Stop() and Join() the service
+ size_t raft_counter = 0;
+ while (!brpc::IsAskedToQuit()) {
+ if(++raft_counter % 10 == 0) {
+ // reset peer configuration periodically to identify change in cluster membership
+ const Option<std::string> & refreshed_peers_op = fetch_file_contents(path_to_peers);
+ if(!peers_op.ok()) {
+ LOG(ERR) << "Error while refreshing peer configuration: " << refreshed_peers_op.error();
+ continue;
+ }
+
+ const std::string & refreshed_peer_ips_string = refreshed_peers_op.get();
+ replication_state.refresh_peers(refreshed_peer_ips_string);
+ }
+
+ sleep(1);
+ }
+
+ LOG(INFO) << "Typesense raft service is going to quit";
+
+ // Stop counter before server
+ replication_state.shutdown();
+ raft_server.Stop(0);
+
+ // Wait until all the processing tasks are over.
+ replication_state.join();
+ raft_server.Join();
+ return 0;
+}
+
int run_server(const Config & config, const std::string & version,
void (*master_server_routes)(), void (*replica_server_routes)()) {
@@ -163,22 +224,24 @@ int run_server(const Config & config, const std::string & version,
return 1;
}
- Store store(config.get_data_dir());
+ std::string data_dir = config.get_data_dir();
+ std::string db_dir = config.get_data_dir() + "/db";
+ std::string state_dir = config.get_data_dir() + "/state";
- LOG(INFO) << "Loading collections from disk...";
+ if(!directory_exists(db_dir) && file_exists(data_dir+"/CURRENT") && file_exists(data_dir+"/IDENTITY")) {
+ LOG(INFO) << "Migrating contents of data directory in a `db` sub-directory, as per the new data layout.";
+ bool moved = mv_dir(data_dir, db_dir);
+ if(!moved) {
+ LOG(ERR) << "CRITICAL ERROR! Failed to migrate all files in the data directory into a `db` sub-directory.";
+ LOG(ERR) << "NOTE: Please move remaining files manually. Failure to do so **WILL** lead to **DATA LOSS**.";
+ return 1;
+ }
+ }
+ Store store(db_dir);
CollectionManager & collectionManager = CollectionManager::get_instance();
- Option<bool> init_op = collectionManager.init(&store,
- config.get_indices_per_collection(),
- config.get_api_key(),
- config.get_search_only_api_key());
-
- if(init_op.ok()) {
- LOG(INFO) << "Finished loading collections from disk.";
- } else {
- LOG(ERR)<< "Typesense failed to start. " << "Could not load collections from disk: " << init_op.error();
- return 1;
- }
+ collectionManager.init(&store, config.get_indices_per_collection(),
+ config.get_api_key(), config.get_search_only_api_key());
curl_global_init(CURL_GLOBAL_SSL);
@@ -197,6 +260,22 @@ int run_server(const Config & config, const std::string & version,
server->on(REPLICATION_EVENT_MSG, Replicator::on_replication_event);
server->on(ReplicationState::REPLICATION_MSG, async_write_request);
+ // first we start the raft service
+ std::promise<bool> ready_promise;
+ std::future<bool> ready_future = ready_promise.get_future();
+
+ ReplicationState replication_state(&store, server->get_message_dispatcher(), &ready_promise);
+
+ std::thread raft_thread([&replication_state, &config, &state_dir]() {
+ std::string path_to_peers = config.get_raft_peers();
+ start_raft_server(replication_state, state_dir, path_to_peers, config.get_raft_port());
+ });
+
+ raft_thread.detach();
+
+ // wait for raft service to be ready before starting http
+ ready_future.get();
+
if(config.get_master().empty()) {
master_server_routes();
} else {
@@ -220,86 +299,6 @@ int run_server(const Config & config, const std::string & version,
replication_thread.detach();
}
- ReplicationState replication_state(server->get_message_dispatcher(), store._get_db_unsafe());
-
- std::thread raft_thread([&replication_state, &config]() {
- std::string path_to_peers = config.get_raft_peers();
-
- if(path_to_peers.empty()) {
- return 0;
- }
-
- const Option<std::string> & peers_op = fetch_file_contents(path_to_peers);
- if(!peers_op.ok()) {
- LOG(ERR) << peers_op.error();
- return -1;
- }
-
- const std::string & peer_ips_string = peers_op.get();
-
- if(peer_ips_string.empty()) {
- LOG(ERR) << "File containing raft peers is empty.";
- return -1;
- }
-
- // start raft server
- brpc::Server raft_server;
-
- uint32_t raft_port = config.get_raft_port();
-
- if (braft::add_service(&raft_server, raft_port) != 0) {
- LOG(ERR) << "Failed to add raft service";
- return -1; // TODO: must quit parent process as well
- }
-
- if (raft_server.Start(raft_port, NULL) != 0) {
- LOG(ERR) << "Failed to start raft server";
- return -1;
- }
-
- std::vector<std::string> peer_ips;
- StringUtils::split(peer_ips_string, peer_ips, ",");
- std::string peers = StringUtils::join(peer_ips, ":0,");
-
- if (replication_state.start(raft_port, 1000, 60, config.get_raft_dir(), peers) != 0) {
- LOG(ERR) << "Failed to start raft state";
- return -1;
- }
-
- LOG(INFO) << "Typesense raft service is running on " << raft_server.listen_address();
-
- // Wait until 'CTRL-C' is pressed. then Stop() and Join() the service
- size_t raft_counter = 0;
- while (!brpc::IsAskedToQuit()) {
- if(++raft_counter % 10 == 0) {
- // reset peer configuration periodically to identify change in cluster membership
- const Option<std::string> & refreshed_peers_op = fetch_file_contents(path_to_peers);
- if(!peers_op.ok()) {
- LOG(ERR) << "Error while refreshing peer configuration: " << refreshed_peers_op.error();
- continue;
- }
-
- const std::string & refreshed_peer_ips_string = refreshed_peers_op.get();
- replication_state.refresh_peers(refreshed_peer_ips_string);
- }
-
- sleep(1);
- }
-
- LOG(INFO) << "Typesense raft service is going to quit";
-
- // Stop counter before server
- replication_state.shutdown();
- raft_server.Stop(0);
-
- // Wait until all the processing tasks are over.
- replication_state.join();
- raft_server.Join();
- return 0;
- });
-
- raft_thread.detach();
-
int ret_code = server->run(&replication_state);
// we are out of the event loop here