Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Basic raft clustering is working.
kishorenceb0c66d778a6
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 2f2a1c80..65f5d0b3 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -40,6 +40,7 @@ FIND_PACKAGE(Snappy REQUIRED)FIND_PACKAGE(ZLIB REQUIRED)FIND_PACKAGE(CURL REQUIRED)FIND_PACKAGE(ICU REQUIRED)+FIND_PACKAGE(Protobuf REQUIRED)message("OpenSSL library: ${OPENSSL_LIBRARIES}")@@ -50,6 +51,8 @@ include(cmake/GoogleTest.cmake)include(cmake/TestResources.cmake)include(cmake/g3log.cmake)include(cmake/Iconv.cmake)+include(cmake/brpc.cmake)+include(cmake/braft.cmake)FILE(GLOB SRC_FILES src/*.cpp)FILE(GLOB TEST_FILES test/*.cpp)@@ -65,6 +68,8 @@ include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/src)include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build/include)include_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/include)+include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)+include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)# Write dependency include directories to a filefile(WRITE ${DEP_ROOT_DIR}/includes.txt "")@@ -73,6 +78,9 @@ file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/include")file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include")file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/src")file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/build/include")+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/include")+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/include")+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/include")link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)link_directories(${DEP_ROOT_DIR}/${FOR_NAME})@@ -80,6 +88,8 @@ link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})link_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build)link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)+link_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/lib)+link_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/lib)# Write dependency libraries to a filefile(WRITE ${DEP_ROOT_DIR}/libs.txt "")@@ -88,6 +98,8 @@ file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/build")file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}")file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/build")file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs")+file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/lib")+file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/lib")add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp)add_executable(search ${SRC_FILES} src/main/main.cpp)@@ -122,18 +134,18 @@ target_compile_definitions(TYPESENSE_VERSION="${TYPESENSE_VERSION}")+set(ROCKSDB_LIBS rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})+if (APPLE)set(STD_LIB -static-libstdc++) # https://stackoverflow.com/a/26543140/131050 (can't statically link libgcc on Mac)FIND_PACKAGE(ngHTTP2 REQUIRED)set(CURL_LIBRARIES ${NGHTTP2_LIBRARIES} ${CURL_LIBRARIES}) # nghttp2 was not being statically built+ set(SYSTEM_LIBS "-framework CoreFoundation" "-framework CoreGraphics" "-framework CoreData"+ "-framework CoreText" "-framework Security" "-framework Foundation")else()set(STD_LIB -static-libgcc -static-libstdc++)-endif()--set(ROCKSDB_LIBS rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})--if(NOT APPLE)+ set(SYSTEM_LIBS "")list(APPEND ROCKSDB_LIBS rt)endif()@@ -141,6 +153,8 @@ set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIEset(CORE_LIBS h2o-evloop iconv ${CURL_LIBRARIES} for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES}${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} pthread dl ${STD_LIB})+set(CORE_LIBS braft brpc leveldb profiler tcmalloc ${CORE_LIBS} gflags ${PROTOBUF_LIBRARIES} ${SYSTEM_LIBS})+target_link_libraries(typesense-core ${CORE_LIBS})target_link_libraries(typesense-server ${CORE_LIBS})target_link_libraries(search ${CORE_LIBS})diff --git a/cmake/braft.cmake b/cmake/braft.cmakenew file mode 100644index 00000000..082f95c9--- /dev/null+++ b/cmake/braft.cmake@@ -0,0 +1,13 @@+set(BRAFT_VERSION 0a9ec3f)+set(BRAFT_NAME braft-${BRAFT_VERSION})+set(BRAFT_TAR_PATH ${DEP_ROOT_DIR}/${BRAFT_NAME}.tar.gz)++if(NOT EXISTS ${BRAFT_TAR_PATH})+ message(STATUS "Downloading braft...")+ file(DOWNLOAD https://dl.typesense.org/deps/${BRAFT_NAME}-${CMAKE_SYSTEM_NAME}.tar.gz ${BRAFT_TAR_PATH})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${BRAFT_NAME})+ message(STATUS "Extracting braft...")+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${BRAFT_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})+endif()diff --git a/cmake/brpc.cmake b/cmake/brpc.cmakenew file mode 100644index 00000000..4c97adf1--- /dev/null+++ b/cmake/brpc.cmake@@ -0,0 +1,13 @@+set(BRPC_VERSION 23c66e3)+set(BRPC_NAME brpc-${BRPC_VERSION})+set(BRPC_TAR_PATH ${DEP_ROOT_DIR}/${BRPC_NAME}.tar.gz)++if(NOT EXISTS ${BRPC_TAR_PATH})+ message(STATUS "Downloading brpc...")+ file(DOWNLOAD https://dl.typesense.org/deps/${BRPC_NAME}-${CMAKE_SYSTEM_NAME}.tar.gz ${BRPC_TAR_PATH})+endif()++if(NOT EXISTS ${DEP_ROOT_DIR}/${BRPC_NAME})+ message(STATUS "Extracting brpc...")+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${BRPC_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})+endif()diff --git a/include/config.h b/include/config.hindex 7d87e34f..42d87f94 100644--- a/include/config.h+++ b/include/config.h@@ -16,6 +16,10 @@ private:std::string listen_address;uint32_t listen_port;+ std::string raft_dir;+ uint32_t raft_port;+ std::string raft_peers;+std::string master;std::string ssl_certificate;@@ -33,6 +37,7 @@ public:Config() {this->listen_address = "0.0.0.0";this->listen_port = 8108;+ this->raft_port = 8107;this->enable_cors = false;this->indices_per_collection = 4;}@@ -82,6 +87,18 @@ public:void set_indices_per_collection(size_t indices_per_collection) {this->indices_per_collection = indices_per_collection;}++ void set_raft_port(int v) {+ this->raft_port = raft_port;+ }++ void set_raft_dir(const std::string & raft_dir) {+ this->raft_dir = raft_dir;+ }++ void set_raft_peers(const std::string & raft_peers) {+ this->raft_peers = raft_peers;+ }// getters@@ -133,6 +150,18 @@ public:return indices_per_collection;}+ int get_raft_port() const {+ return this->raft_port;+ }++ std::string get_raft_dir() const {+ return this->raft_dir;+ }++ std::string get_raft_peers() const {+ return this->raft_peers;+ }+// loadersstd::string get_env(const char *name) {@@ -158,6 +187,13 @@ public:this->listen_port = std::stoi(get_env("TYPESENSE_LISTEN_PORT"));}+ if(!get_env("TYPESENSE_RAFT_PORT").empty()) {+ this->raft_port = std::stoi(get_env("TYPESENSE_RAFT_PORT"));+ }++ this->raft_dir = get_env("TYPESENSE_RAFT_DIR");+ this->raft_peers = get_env("TYPESENSE_RAFT_PEERS");+this->master = get_env("TYPESENSE_MASTER");this->ssl_certificate = get_env("TYPESENSE_SSL_CERTIFICATE");this->ssl_certificate_key = get_env("TYPESENSE_SSL_CERTIFICATE_KEY");@@ -225,6 +261,18 @@ public:if(reader.Exists("server", "enable-cors")) {this->enable_cors = reader.GetBoolean("server", "enable-cors", false);}++ if(reader.Exists("server", "raft-port")) {+ this->raft_port = reader.GetInteger("server", "raft-port", 8107);+ }++ if(reader.Exists("server", "raft-dir")) {+ this->raft_dir = reader.Get("server", "raft-dir", "");+ }++ if(reader.Exists("server", "raft-peers")) {+ this->raft_peers = reader.Get("server", "raft-peers", "");+ }}void load_config_cmd_args(cmdline::parser & options) {@@ -267,6 +315,18 @@ public:if(options.exist("enable-cors")) {this->enable_cors = options.exist("enable-cors");}++ if(options.exist("raft-port")) {+ this->raft_port = options.get<uint32_t>("raft-port");+ }++ if(options.exist("raft-dir")) {+ this->raft_dir = options.get<std::string>("raft-dir");+ }++ if(options.exist("raft-peers")) {+ this->raft_peers = options.get<std::string>("raft-peers");+ }}// validation@@ -284,6 +344,14 @@ public:return Option<bool>(500, "API key is not specified.");}+ if(!raft_dir.empty() && raft_peers.empty()) {+ return Option<bool>(500, "Argument --raft-peers is not specified.");+ }++ if(!raft_peers.empty() && raft_dir.empty()) {+ return Option<bool>(500, "Argument --raft-dir is not specified.");+ }+return Option<bool>(true);}};diff --git a/include/http_data.h b/include/http_data.hindex 5c3dbe40..9b1534a0 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -4,6 +4,7 @@#include <string>#include <map>#include <vector>+#include "json.hpp"#define H2O_USE_LIBUV 0extern "C" {@@ -76,8 +77,35 @@ struct http_res {struct http_req {h2o_req_t* _req;+ std::string path_without_query;std::map<std::string, std::string> params;std::string body;++ http_req() {}++ http_req(h2o_req_t* _req, const std::string & path_without_query, const std::map<std::string, std::string> & params,+ std::string body): _req(_req), path_without_query(path_without_query), params(params), body(body) {}++ void deserialize(const std::string& serialized_content) {+ nlohmann::json content = nlohmann::json::parse(serialized_content);+ path_without_query = content["path"];+ body = content["body"];++ for (nlohmann::json::iterator it = content["params"].begin(); it != content["params"].end(); ++it) {+ params.emplace(it.key(), it.value());+ }++ _req = nullptr;+ }++ std::string serialize() const {+ nlohmann::json content;+ content["path"] = path_without_query;+ content["params"] = params;+ content["body"] = body;++ return content.dump();+ }};struct route_path {diff --git a/include/raft_server.h b/include/raft_server.hnew file mode 100644index 00000000..bae7a347--- /dev/null+++ b/include/raft_server.h@@ -0,0 +1,167 @@+#pragma once++#include <brpc/controller.h> // brpc::Controller+#include <brpc/server.h> // brpc::Server+#include <braft/raft.h> // braft::Node braft::StateMachine+#include <braft/storage.h> // braft::SnapshotWriter+#include <braft/util.h> // braft::AsyncClosureGuard+#include <braft/protobuf_file.h> // braft::ProtoBufFile+#include <rocksdb/db.h>++#include "http_data.h"+#include "http_server.h"++class ReplicationState;++// Implements the callback for the state machine+class ReplicationClosure : public braft::Closure {+private:+ ReplicationState *searchStore;+ HttpServer* httpServer;+ const http_req* request;+ http_res* response;+ google::protobuf::Closure* done;++public:+ ReplicationClosure(ReplicationState *searchStore, HttpServer* httpServer, const http_req* request, http_res* response,+ google::protobuf::Closure* done):+ searchStore(searchStore), httpServer(httpServer),+ request(request), response(response), done(done) {++ }++ ~ReplicationClosure() {}++ const http_req* get_request() const {+ return request;+ }++ http_res* get_response() const {+ return response;+ }++ void Run();+};++// Closure that fires when refresh peers operation finishes+class RefreshPeersClosure : public braft::Closure {+public:++ RefreshPeersClosure() {}++ ~RefreshPeersClosure() {}++ void Run() {+ // Auto delete this after Run()+ std::unique_ptr<RefreshPeersClosure> self_guard(this);++ if(status().ok()) {+ LOG(INFO) << "Peer refresh succeeded!";+ } else {+ LOG(ERROR) << "Peer refresh failed, error: " << status().error_str();+ }+ }+};++// Implements braft::StateMachine.+class ReplicationState : public braft::StateMachine {+private:+ braft::Node* volatile _node;+ butil::atomic<int64_t> _value;+ butil::atomic<int64_t> _leader_term;+ rocksdb::DB* _db;+ HttpServer* _http_server;++public:+ ReplicationState(): _node(NULL), _value(0), _leader_term(-1) {++ }++ ~ReplicationState() {+ delete _node;+ }++ // Starts this node+ int start(int port, int election_timeout_ms, int snapshot_interval_s,+ const std::string & data_path, const std::string & peers);++ // Generic write method for synchronizing all writes+ void write(const http_req* request, http_res* response, google::protobuf::Closure* done);++ // Generic read method for consistent reads+ void read(http_res* response);++ // updates cluster membership+ void refresh_peers(const std::string & peers);++ bool is_leader() const {+ return _leader_term.load(butil::memory_order_acquire) > 0;+ }++ // Shut this node down.+ void shutdown() {+ if (_node) {+ _node->shutdown(NULL);+ }+ }++ // Blocking this thread until the node is eventually down.+ void join() {+ if (_node) {+ _node->join();+ }+ }++private:+ friend class ReplicationClosure;++ // redirecting request to leader+ void redirect(http_res* response);++ // actual application of writes onto the WAL+ void on_apply(braft::Iterator& iter);++ struct SnapshotArg {+ rocksdb::DB* db;+ braft::SnapshotWriter* writer;+ braft::Closure* done;+ };++ static void *save_snapshot(void* arg);++ void on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done);++ static bool copy_snapshot(const std::string& from_path, const std::string& to_path);++ int on_snapshot_load(braft::SnapshotReader* reader);++ void on_leader_start(int64_t term) {+ _leader_term.store(term, butil::memory_order_release);+ LOG(INFO) << "Node becomes leader, term: " << term;+ }++ void on_leader_stop(const butil::Status& status) {+ _leader_term.store(-1, butil::memory_order_release);+ LOG(INFO) << "Node stepped down : " << status;+ }++ void on_shutdown() {+ LOG(INFO) << "This node is down";+ }++ void on_error(const ::braft::Error& e) {+ LOG(ERROR) << "Met raft error " << e;+ }++ void on_configuration_committed(const ::braft::Configuration& conf) {+ LOG(INFO) << "Configuration of this group is " << conf;+ }++ void on_stop_following(const ::braft::LeaderChangeContext& ctx) {+ LOG(INFO) << "Node stops following " << ctx;+ }++ void on_start_following(const ::braft::LeaderChangeContext& ctx) {+ LOG(INFO) << "Node start following " << ctx;+ }+};diff --git a/src/http_server.cpp b/src/http_server.cppindex d8133826..ae6eb0cd 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -349,7 +349,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {}}- http_req* request = new http_req{req, query_map, req_body};+ http_req* request = new http_req(req, path_without_query, query_map, req_body);http_res* response = new http_res();response->server = self->http_server;(rpath.handler)(*request, *response);diff --git a/src/raft_server.cpp b/src/raft_server.cppnew file mode 100644index 00000000..d5b817e5--- /dev/null+++ b/src/raft_server.cpp@@ -0,0 +1,254 @@+#include "raft_server.h"+#include <butil/files/file_enumerator.h>+#include <thread>+#include "rocksdb/utilities/checkpoint.h"+++void ReplicationClosure::Run() {+ // Auto delete this after Run()+ std::unique_ptr<ReplicationClosure> self_guard(this);++ // Respond to this RPC.+ brpc::ClosureGuard done_guard(done);+ if (status().ok()) {+ return;+ }++ // Try redirect if this request failed.+ // TODO: searchStore->redirect(response);+}++// State machine implementation++int ReplicationState::start(int port, int election_timeout_ms, int snapshot_interval_s,+ const std::string & data_path, const std::string & peers) {+ butil::EndPoint addr(butil::my_ip(), port);+ braft::NodeOptions node_options;++ if(node_options.initial_conf.parse_from(peers) != 0) {+ LOG(ERROR) << "Fail to parse peer configuration `" << peers << "`";+ return -1;+ }++ node_options.election_timeout_ms = election_timeout_ms;+ node_options.fsm = this;+ node_options.node_owns_fsm = false;+ node_options.snapshot_interval_s = snapshot_interval_s;+ std::string prefix = "local://" + data_path;+ node_options.log_uri = prefix + "/log";+ node_options.raft_meta_uri = prefix + "/raft_meta";+ node_options.snapshot_uri = prefix + "/snapshot";+ node_options.disable_cli = true;++ braft::Node* node = new braft::Node("default_group", braft::PeerId(addr));+ if (node->init(node_options) != 0) {+ LOG(ERROR) << "Fail to init raft node";+ delete node;+ return -1;+ }++ _node = node;+ return 0;+}++void ReplicationState::write(const http_req* request, http_res* response, google::protobuf::Closure* done) {+ brpc::ClosureGuard done_guard(done);++ if (!is_leader()) {+ // TODO: return redirect(response);+ }++ // Serialize request to replicated WAL so that all the peers in the group receive it as well.+ // NOTE: actual write must be done only on the `on_apply` method to maintain consistency.++ butil::IOBufBuilder bufBuilder;+ bufBuilder << request->serialize();+ butil::IOBuf log = bufBuilder.buf();++ // Apply this log as a braft::Task+ braft::Task task;+ task.data = &log;+ // This callback would be invoked when the task actually executes or fails+ task.done = new ReplicationClosure(this, _http_server, request, response, done_guard.release());++ // To avoid ABA problem+ task.expected_term = _leader_term.load(butil::memory_order_relaxed);++ // Now the task is applied to the group, waiting for the result.+ return _node->apply(task);+}++void ReplicationState::read(http_res* response) {+ // For consistency, reads to follower must be rejected.+ if (!is_leader()) {+ // This node is a follower or it's not up-to-date. Redirect to+ // the leader if possible.+ return redirect(response);+ }++ // This is the leader and is up-to-date. It's safe to respond client+ // TODO:+ // response->set_value(_value.load(butil::memory_order_relaxed));+}++void ReplicationState::redirect(http_res* response) {+ if (_node) {+ braft::PeerId leader = _node->leader_id();+ if (!leader.is_empty()) {+ // TODO: response->set_redirect(leader.to_string());+ }+ }+}++void ReplicationState::on_apply(braft::Iterator& iter) {+ // A batch of tasks are committed, which must be processed through+ // |iter|+ for (; iter.valid(); iter.next()) {+ int64_t detal_value = 0;+ http_res* response = NULL;+ const http_req* request;+ std::unique_ptr<http_req> remote_request(new http_req);++ // Guard helps to invoke iter.done()->Run() asynchronously to avoid the callback blocking the StateMachine+ braft::AsyncClosureGuard closure_guard(iter.done());++ if (iter.done()) {+ // This task is applied by this node, get value from the closure to avoid additional parsing.+ ReplicationClosure* c = dynamic_cast<ReplicationClosure*>(iter.done());+ response = c->get_response();+ request = c->get_request();+ } else {+ // Parse request from the log+ remote_request->deserialize(iter.data().to_string());+ request = remote_request.get();+ }++ // Now that the log has been parsed, perform the actual operation+ // TODO: call http server thread for write++ if (response) {+ // TODO: leader, so send response back to the client+ }+ }+}++void* ReplicationState::save_snapshot(void* arg) {+ ReplicationState::SnapshotArg* sa = (ReplicationState::SnapshotArg*) arg;+ std::unique_ptr<ReplicationState::SnapshotArg> arg_guard(sa);++ brpc::ClosureGuard done_guard(sa->done);++ std::string snapshot_path = sa->writer->get_path() + "/rocksdb_snapshot";++ rocksdb::Checkpoint* checkpoint = NULL;+ rocksdb::Status status = rocksdb::Checkpoint::Create(sa->db, &checkpoint);++ if(!status.ok()) {+ LOG(WARNING) << "Checkpoint Create failed, msg:" << status.ToString();+ return NULL;+ }++ std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);+ status = checkpoint->CreateCheckpoint(snapshot_path);+ if(!status.ok()) {+ LOG(WARNING) << "Checkpoint CreateCheckpoint failed, msg:" << status.ToString();+ return NULL;+ }++ butil::FileEnumerator dir_enum(butil::FilePath(snapshot_path),false, butil::FileEnumerator::FILES);+ for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {+ std::string file_name = "rocksdb_snapshot/" + name.BaseName().value();+ sa->writer->add_file(file_name);++ if (sa->writer->add_file(file_name) != 0) {+ sa->done->status().set_error(EIO, "Fail to add file to writer.");+ return NULL;+ }+ }++ return NULL;+}++void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done) {+ // Start a new bthread to avoid blocking StateMachine since it could be slow to write data to disk+ SnapshotArg* arg = new SnapshotArg;+ arg->db = _db;+ arg->writer = writer;+ arg->done = done;+ bthread_t tid;+ bthread_start_urgent(&tid, NULL, save_snapshot, arg);+}++bool ReplicationState::copy_snapshot(const std::string& from_path, const std::string& to_path) {+ struct stat from_stat;++ if (stat(from_path.c_str(), &from_stat) < 0 || !S_ISDIR(from_stat.st_mode)) {+ LOG(WARNING) << "stat " << from_path << " failed";+ return false;+ }++ if (!butil::CreateDirectory(butil::FilePath(to_path))) {+ LOG(WARNING) << "CreateDirectory " << to_path << " failed";+ return false;+ }++ butil::FileEnumerator dir_enum(butil::FilePath(from_path),false, butil::FileEnumerator::FILES);+ for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {+ std::string src_file(from_path);+ std::string dst_file(to_path);+ butil::string_appendf(&src_file, "/%s", name.BaseName().value().c_str());+ butil::string_appendf(&dst_file, "/%s", name.BaseName().value().c_str());++ if (0 != link(src_file.c_str(), dst_file.c_str())) {+ if (!butil::CopyFile(butil::FilePath(src_file), butil::FilePath(dst_file))) {+ LOG(WARNING) << "copy " << src_file << " to " << dst_file << " failed";+ return false;+ }+ }+ }++ return true;+}++int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {+ // reset rocksdb handle+ if (_db != NULL) {+ delete _db;+ _db = NULL;+ }++ CHECK(!is_leader()) << "Leader is not supposed to load snapshot";++ // Load snapshot from reader, replacing the running StateMachine++ std::string snapshot_path = reader->get_path();+ snapshot_path.append("/rocksdb_snapshot");++ std::string db_path = "./data/rocksdb_data"; // TODO+ if (!butil::DeleteFile(butil::FilePath(db_path), true)) {+ LOG(WARNING) << "rm " << db_path << " failed";+ return -1;+ }++ LOG(TRACE) << "rm " << db_path << " success";+ //TODO: try use link instead of copy++ if (!copy_snapshot(snapshot_path, db_path)) {+ LOG(WARNING) << "copy snapshot " << snapshot_path << " to " << db_path << " failed";+ return -1;+ }++ LOG(TRACE) << "copy snapshot " << snapshot_path << " to " << db_path << " success";++ return 0;+}++void ReplicationState::refresh_peers(const std::string & peers) {+ if(_node && is_leader()) {+ LOG(INFO) << "Refreshing peer config";+ braft::Configuration conf;+ conf.parse_from(peers);+ RefreshPeersClosure* refresh_peers_done = new RefreshPeersClosure;+ _node->change_peers(conf, refresh_peers_done);+ }+}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 9aab8252..1daa052a 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -1,8 +1,19 @@-#include "core_api.h"-#include "typesense_server_utils.h"#include <curl/curl.h>#include <sys/stat.h>+#include <gflags/gflags.h>+#include <brpc/controller.h>+#include <brpc/server.h>+#include <braft/raft.h>+#include <braft/storage.h>+#include <braft/util.h>+#include <braft/protobuf_file.h>+#include <raft_server.h>+#include <fstream>++#include "core_api.h"+#include "typesense_server_utils.h"+HttpServer* server;void catch_interrupt(int sig) {@@ -16,6 +27,11 @@ bool directory_exists(const std::string & dir_path) {return stat(dir_path.c_str(), &info) == 0 && (info.st_mode & S_IFDIR);}+bool file_exists(const std::string & file_path) {+ struct stat info;+ return stat(file_path.c_str(), &info) == 0;+}+void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.set_program_name("./typesense-server");@@ -25,6 +41,11 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<std::string>("listen-address", 'h', "Address to which Typesense server binds.", false, "0.0.0.0");options.add<uint32_t>("listen-port", 'p', "Port on which Typesense server listens.", false, 8108);++ options.add<std::string>("raft-dir", '\0', "Directory where raft data will be stored.", false);+ options.add<uint32_t>("raft-port", '\0', "Port on which Typesense raft service listens.", false, 8107);+ options.add<std::string>("raft-peers", '\0', "Path to file having comma separated string of Raft node IPs.", false);+options.add<std::string>("master", 'm', "To start the server as read-only replica, ""provide the master's address in http(s)://<master_address>:<master_port> format.",false, "");@@ -137,11 +158,89 @@ int run_server(const Config & config, const std::string & version,replication_thread.detach();}+ std::thread raft_thread([&store, &config]() {+ std::string path_to_peers = config.get_raft_peers();++ if(path_to_peers.empty()) {+ return 0;+ }++ if(!file_exists(path_to_peers)) {+ LOG(ERR) << "Error reading file containing raft peers.";+ return -1;+ }++ std::string peer_ips_string;+ std::ifstream infile(path_to_peers);+ std::getline(infile, peer_ips_string);+ infile.close();++ if(peer_ips_string.empty()) {+ LOG(ERR) << "File containing raft peers is empty.";+ return -1;+ }++ // start raft server+ brpc::Server raft_server;+ ReplicationState replication_state;++ uint32_t raft_port = config.get_raft_port();++ if (braft::add_service(&raft_server, raft_port) != 0) {+ LOG(ERR) << "Failed to add raft service";+ return -1; // TODO: must quit parent process as well+ }++ if (raft_server.Start(raft_port, NULL) != 0) {+ LOG(ERR) << "Failed to start raft server";+ return -1;+ }++ std::vector<std::string> peer_ips;+ StringUtils::split(peer_ips_string, peer_ips, ",");+ std::string peers = StringUtils::join(peer_ips, ":0,");++ if (replication_state.start(raft_port, 1000, 3600, config.get_raft_dir(), peers) != 0) {+ LOG(ERR) << "Failed to start raft state";+ return -1;+ }++ LOG(INFO) << "Typesense raft service is running on " << raft_server.listen_address();++ // Wait until 'CTRL-C' is pressed. then Stop() and Join() the service+ size_t raft_counter = 0;+ while (!brpc::IsAskedToQuit()) {+ if(++raft_counter % 10 == 0) {+ // reset peer configuration periodically to identify change in cluster membership+ std::string new_peer_ips_string;+ std::ifstream peers_infile(path_to_peers);+ std::getline(peers_infile, new_peer_ips_string);+ infile.close();+ replication_state.refresh_peers(new_peer_ips_string);+ }+ sleep(1);+ }++ LOG(INFO) << "Typesense raft service is going to quit";++ // Stop counter before server+ replication_state.shutdown();+ raft_server.Stop(0);++ // Wait until all the processing tasks are over.+ replication_state.join();+ raft_server.Join();+ return 0;+ });++ raft_thread.detach();+int ret_code = server->run();+ // we are out of the event loop here+curl_global_cleanup();- // we are out of the event loop heredelete server;CollectionManager::get_instance().dispose();