Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Basic raft clustering is working.

kishorenceb0c66d778a6
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2f2a1c80..65f5d0b3 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -40,6 +40,7 @@ FIND_PACKAGE(Snappy REQUIRED)
FIND_PACKAGE(ZLIB REQUIRED)
FIND_PACKAGE(CURL REQUIRED)
FIND_PACKAGE(ICU REQUIRED)
+FIND_PACKAGE(Protobuf REQUIRED)
message("OpenSSL library: ${OPENSSL_LIBRARIES}")
@@ -50,6 +51,8 @@ include(cmake/GoogleTest.cmake)
include(cmake/TestResources.cmake)
include(cmake/g3log.cmake)
include(cmake/Iconv.cmake)
+include(cmake/brpc.cmake)
+include(cmake/braft.cmake)
FILE(GLOB SRC_FILES src/*.cpp)
FILE(GLOB TEST_FILES test/*.cpp)
@@ -65,6 +68,8 @@ include_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include)
include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/src)
include_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build/include)
include_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/include)
+include_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/include)
+include_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/include)
# Write dependency include directories to a file
file(WRITE ${DEP_ROOT_DIR}/includes.txt "")
@@ -73,6 +78,9 @@ file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/include")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}/include")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/src")
file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/build/include")
+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/include")
+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/include")
+file(APPEND ${DEP_ROOT_DIR}/includes.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/include")
link_directories(${DEP_ROOT_DIR}/${GTEST_NAME}/googletest/build)
link_directories(${DEP_ROOT_DIR}/${FOR_NAME})
@@ -80,6 +88,8 @@ link_directories(${DEP_ROOT_DIR}/${H2O_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ROCKSDB_NAME})
link_directories(${DEP_ROOT_DIR}/${G3LOG_NAME}/build)
link_directories(${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs)
+link_directories(${DEP_ROOT_DIR}/${BRPC_NAME}/lib)
+link_directories(${DEP_ROOT_DIR}/${BRAFT_NAME}/lib)
# Write dependency libraries to a file
file(WRITE ${DEP_ROOT_DIR}/libs.txt "")
@@ -88,6 +98,8 @@ file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${H2O_NAME}/build")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ROCKSDB_NAME}")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${G3LOG_NAME}/build")
file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${ICONV_NAME}/lib/.libs")
+file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRPC_NAME}/lib")
+file(APPEND ${DEP_ROOT_DIR}/libs.txt " ${DEP_ROOT_DIR}/${BRAFT_NAME}/lib")
add_executable(typesense-server ${SRC_FILES} src/main/typesense_server.cpp)
add_executable(search ${SRC_FILES} src/main/main.cpp)
@@ -122,18 +134,18 @@ target_compile_definitions(
TYPESENSE_VERSION="${TYPESENSE_VERSION}"
)
+set(ROCKSDB_LIBS rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})
+
if (APPLE)
set(STD_LIB -static-libstdc++) # https://stackoverflow.com/a/26543140/131050 (can't statically link libgcc on Mac)
FIND_PACKAGE(ngHTTP2 REQUIRED)
set(CURL_LIBRARIES ${NGHTTP2_LIBRARIES} ${CURL_LIBRARIES}) # nghttp2 was not being statically built
+ set(SYSTEM_LIBS "-framework CoreFoundation" "-framework CoreGraphics" "-framework CoreData"
+ "-framework CoreText" "-framework Security" "-framework Foundation")
else()
set(STD_LIB -static-libgcc -static-libstdc++)
-endif()
-
-set(ROCKSDB_LIBS rocksdb ${SNAPPY_LIBRARIES} ${ZLIB_LIBRARIES})
-
-if(NOT APPLE)
+ set(SYSTEM_LIBS "")
list(APPEND ROCKSDB_LIBS rt)
endif()
@@ -141,6 +153,8 @@ set(ICU_ALL_LIBRARIES ${ICU_I18N_LIBRARIES} ${ICU_LIBRARIES} ${ICU_DATA_LIBRARIE
set(CORE_LIBS h2o-evloop iconv ${CURL_LIBRARIES} for ${ICU_ALL_LIBRARIES} ${G3LOGGER_LIBRARIES}
${ROCKSDB_LIBS} ${OPENSSL_LIBRARIES} pthread dl ${STD_LIB})
+set(CORE_LIBS braft brpc leveldb profiler tcmalloc ${CORE_LIBS} gflags ${PROTOBUF_LIBRARIES} ${SYSTEM_LIBS})
+
target_link_libraries(typesense-core ${CORE_LIBS})
target_link_libraries(typesense-server ${CORE_LIBS})
target_link_libraries(search ${CORE_LIBS})
diff --git a/cmake/braft.cmake b/cmake/braft.cmake
new file mode 100644
index 00000000..082f95c9
--- /dev/null
+++ b/cmake/braft.cmake
@@ -0,0 +1,13 @@
+set(BRAFT_VERSION 0a9ec3f)
+set(BRAFT_NAME braft-${BRAFT_VERSION})
+set(BRAFT_TAR_PATH ${DEP_ROOT_DIR}/${BRAFT_NAME}.tar.gz)
+
+if(NOT EXISTS ${BRAFT_TAR_PATH})
+ message(STATUS "Downloading braft...")
+ file(DOWNLOAD https://dl.typesense.org/deps/${BRAFT_NAME}-${CMAKE_SYSTEM_NAME}.tar.gz ${BRAFT_TAR_PATH})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${BRAFT_NAME})
+ message(STATUS "Extracting braft...")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${BRAFT_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
+endif()
diff --git a/cmake/brpc.cmake b/cmake/brpc.cmake
new file mode 100644
index 00000000..4c97adf1
--- /dev/null
+++ b/cmake/brpc.cmake
@@ -0,0 +1,13 @@
+set(BRPC_VERSION 23c66e3)
+set(BRPC_NAME brpc-${BRPC_VERSION})
+set(BRPC_TAR_PATH ${DEP_ROOT_DIR}/${BRPC_NAME}.tar.gz)
+
+if(NOT EXISTS ${BRPC_TAR_PATH})
+ message(STATUS "Downloading brpc...")
+ file(DOWNLOAD https://dl.typesense.org/deps/${BRPC_NAME}-${CMAKE_SYSTEM_NAME}.tar.gz ${BRPC_TAR_PATH})
+endif()
+
+if(NOT EXISTS ${DEP_ROOT_DIR}/${BRPC_NAME})
+ message(STATUS "Extracting brpc...")
+ execute_process(COMMAND ${CMAKE_COMMAND} -E tar xzf ${BRPC_TAR_PATH} WORKING_DIRECTORY ${DEP_ROOT_DIR})
+endif()
diff --git a/include/config.h b/include/config.h
index 7d87e34f..42d87f94 100644
--- a/include/config.h
+++ b/include/config.h
@@ -16,6 +16,10 @@ private:
std::string listen_address;
uint32_t listen_port;
+ std::string raft_dir;
+ uint32_t raft_port;
+ std::string raft_peers;
+
std::string master;
std::string ssl_certificate;
@@ -33,6 +37,7 @@ public:
Config() {
this->listen_address = "0.0.0.0";
this->listen_port = 8108;
+ this->raft_port = 8107;
this->enable_cors = false;
this->indices_per_collection = 4;
}
@@ -82,6 +87,18 @@ public:
void set_indices_per_collection(size_t indices_per_collection) {
this->indices_per_collection = indices_per_collection;
}
+
+ void set_raft_port(int v) {
+ this->raft_port = raft_port;
+ }
+
+ void set_raft_dir(const std::string & raft_dir) {
+ this->raft_dir = raft_dir;
+ }
+
+ void set_raft_peers(const std::string & raft_peers) {
+ this->raft_peers = raft_peers;
+ }
// getters
@@ -133,6 +150,18 @@ public:
return indices_per_collection;
}
+ int get_raft_port() const {
+ return this->raft_port;
+ }
+
+ std::string get_raft_dir() const {
+ return this->raft_dir;
+ }
+
+ std::string get_raft_peers() const {
+ return this->raft_peers;
+ }
+
// loaders
std::string get_env(const char *name) {
@@ -158,6 +187,13 @@ public:
this->listen_port = std::stoi(get_env("TYPESENSE_LISTEN_PORT"));
}
+ if(!get_env("TYPESENSE_RAFT_PORT").empty()) {
+ this->raft_port = std::stoi(get_env("TYPESENSE_RAFT_PORT"));
+ }
+
+ this->raft_dir = get_env("TYPESENSE_RAFT_DIR");
+ this->raft_peers = get_env("TYPESENSE_RAFT_PEERS");
+
this->master = get_env("TYPESENSE_MASTER");
this->ssl_certificate = get_env("TYPESENSE_SSL_CERTIFICATE");
this->ssl_certificate_key = get_env("TYPESENSE_SSL_CERTIFICATE_KEY");
@@ -225,6 +261,18 @@ public:
if(reader.Exists("server", "enable-cors")) {
this->enable_cors = reader.GetBoolean("server", "enable-cors", false);
}
+
+ if(reader.Exists("server", "raft-port")) {
+ this->raft_port = reader.GetInteger("server", "raft-port", 8107);
+ }
+
+ if(reader.Exists("server", "raft-dir")) {
+ this->raft_dir = reader.Get("server", "raft-dir", "");
+ }
+
+ if(reader.Exists("server", "raft-peers")) {
+ this->raft_peers = reader.Get("server", "raft-peers", "");
+ }
}
void load_config_cmd_args(cmdline::parser & options) {
@@ -267,6 +315,18 @@ public:
if(options.exist("enable-cors")) {
this->enable_cors = options.exist("enable-cors");
}
+
+ if(options.exist("raft-port")) {
+ this->raft_port = options.get<uint32_t>("raft-port");
+ }
+
+ if(options.exist("raft-dir")) {
+ this->raft_dir = options.get<std::string>("raft-dir");
+ }
+
+ if(options.exist("raft-peers")) {
+ this->raft_peers = options.get<std::string>("raft-peers");
+ }
}
// validation
@@ -284,6 +344,14 @@ public:
return Option<bool>(500, "API key is not specified.");
}
+ if(!raft_dir.empty() && raft_peers.empty()) {
+ return Option<bool>(500, "Argument --raft-peers is not specified.");
+ }
+
+ if(!raft_peers.empty() && raft_dir.empty()) {
+ return Option<bool>(500, "Argument --raft-dir is not specified.");
+ }
+
return Option<bool>(true);
}
};
diff --git a/include/http_data.h b/include/http_data.h
index 5c3dbe40..9b1534a0 100644
--- a/include/http_data.h
+++ b/include/http_data.h
@@ -4,6 +4,7 @@
#include <string>
#include <map>
#include <vector>
+#include "json.hpp"
#define H2O_USE_LIBUV 0
extern "C" {
@@ -76,8 +77,35 @@ struct http_res {
struct http_req {
h2o_req_t* _req;
+ std::string path_without_query;
std::map<std::string, std::string> params;
std::string body;
+
+ http_req() {}
+
+ http_req(h2o_req_t* _req, const std::string & path_without_query, const std::map<std::string, std::string> & params,
+ std::string body): _req(_req), path_without_query(path_without_query), params(params), body(body) {}
+
+ void deserialize(const std::string& serialized_content) {
+ nlohmann::json content = nlohmann::json::parse(serialized_content);
+ path_without_query = content["path"];
+ body = content["body"];
+
+ for (nlohmann::json::iterator it = content["params"].begin(); it != content["params"].end(); ++it) {
+ params.emplace(it.key(), it.value());
+ }
+
+ _req = nullptr;
+ }
+
+ std::string serialize() const {
+ nlohmann::json content;
+ content["path"] = path_without_query;
+ content["params"] = params;
+ content["body"] = body;
+
+ return content.dump();
+ }
};
struct route_path {
diff --git a/include/raft_server.h b/include/raft_server.h
new file mode 100644
index 00000000..bae7a347
--- /dev/null
+++ b/include/raft_server.h
@@ -0,0 +1,167 @@
+#pragma once
+
+#include <brpc/controller.h> // brpc::Controller
+#include <brpc/server.h> // brpc::Server
+#include <braft/raft.h> // braft::Node braft::StateMachine
+#include <braft/storage.h> // braft::SnapshotWriter
+#include <braft/util.h> // braft::AsyncClosureGuard
+#include <braft/protobuf_file.h> // braft::ProtoBufFile
+#include <rocksdb/db.h>
+
+#include "http_data.h"
+#include "http_server.h"
+
+class ReplicationState;
+
+// Implements the callback for the state machine
+class ReplicationClosure : public braft::Closure {
+private:
+ ReplicationState *searchStore;
+ HttpServer* httpServer;
+ const http_req* request;
+ http_res* response;
+ google::protobuf::Closure* done;
+
+public:
+ ReplicationClosure(ReplicationState *searchStore, HttpServer* httpServer, const http_req* request, http_res* response,
+ google::protobuf::Closure* done):
+ searchStore(searchStore), httpServer(httpServer),
+ request(request), response(response), done(done) {
+
+ }
+
+ ~ReplicationClosure() {}
+
+ const http_req* get_request() const {
+ return request;
+ }
+
+ http_res* get_response() const {
+ return response;
+ }
+
+ void Run();
+};
+
+// Closure that fires when refresh peers operation finishes
+class RefreshPeersClosure : public braft::Closure {
+public:
+
+ RefreshPeersClosure() {}
+
+ ~RefreshPeersClosure() {}
+
+ void Run() {
+ // Auto delete this after Run()
+ std::unique_ptr<RefreshPeersClosure> self_guard(this);
+
+ if(status().ok()) {
+ LOG(INFO) << "Peer refresh succeeded!";
+ } else {
+ LOG(ERROR) << "Peer refresh failed, error: " << status().error_str();
+ }
+ }
+};
+
+// Implements braft::StateMachine.
+class ReplicationState : public braft::StateMachine {
+private:
+ braft::Node* volatile _node;
+ butil::atomic<int64_t> _value;
+ butil::atomic<int64_t> _leader_term;
+ rocksdb::DB* _db;
+ HttpServer* _http_server;
+
+public:
+ ReplicationState(): _node(NULL), _value(0), _leader_term(-1) {
+
+ }
+
+ ~ReplicationState() {
+ delete _node;
+ }
+
+ // Starts this node
+ int start(int port, int election_timeout_ms, int snapshot_interval_s,
+ const std::string & data_path, const std::string & peers);
+
+ // Generic write method for synchronizing all writes
+ void write(const http_req* request, http_res* response, google::protobuf::Closure* done);
+
+ // Generic read method for consistent reads
+ void read(http_res* response);
+
+ // updates cluster membership
+ void refresh_peers(const std::string & peers);
+
+ bool is_leader() const {
+ return _leader_term.load(butil::memory_order_acquire) > 0;
+ }
+
+ // Shut this node down.
+ void shutdown() {
+ if (_node) {
+ _node->shutdown(NULL);
+ }
+ }
+
+ // Blocking this thread until the node is eventually down.
+ void join() {
+ if (_node) {
+ _node->join();
+ }
+ }
+
+private:
+ friend class ReplicationClosure;
+
+ // redirecting request to leader
+ void redirect(http_res* response);
+
+ // actual application of writes onto the WAL
+ void on_apply(braft::Iterator& iter);
+
+ struct SnapshotArg {
+ rocksdb::DB* db;
+ braft::SnapshotWriter* writer;
+ braft::Closure* done;
+ };
+
+ static void *save_snapshot(void* arg);
+
+ void on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done);
+
+ static bool copy_snapshot(const std::string& from_path, const std::string& to_path);
+
+ int on_snapshot_load(braft::SnapshotReader* reader);
+
+ void on_leader_start(int64_t term) {
+ _leader_term.store(term, butil::memory_order_release);
+ LOG(INFO) << "Node becomes leader, term: " << term;
+ }
+
+ void on_leader_stop(const butil::Status& status) {
+ _leader_term.store(-1, butil::memory_order_release);
+ LOG(INFO) << "Node stepped down : " << status;
+ }
+
+ void on_shutdown() {
+ LOG(INFO) << "This node is down";
+ }
+
+ void on_error(const ::braft::Error& e) {
+ LOG(ERROR) << "Met raft error " << e;
+ }
+
+ void on_configuration_committed(const ::braft::Configuration& conf) {
+ LOG(INFO) << "Configuration of this group is " << conf;
+ }
+
+ void on_stop_following(const ::braft::LeaderChangeContext& ctx) {
+ LOG(INFO) << "Node stops following " << ctx;
+ }
+
+ void on_start_following(const ::braft::LeaderChangeContext& ctx) {
+ LOG(INFO) << "Node start following " << ctx;
+ }
+};
diff --git a/src/http_server.cpp b/src/http_server.cpp
index d8133826..ae6eb0cd 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -349,7 +349,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_self, h2o_req_t *req) {
}
}
- http_req* request = new http_req{req, query_map, req_body};
+ http_req* request = new http_req(req, path_without_query, query_map, req_body);
http_res* response = new http_res();
response->server = self->http_server;
(rpath.handler)(*request, *response);
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
new file mode 100644
index 00000000..d5b817e5
--- /dev/null
+++ b/src/raft_server.cpp
@@ -0,0 +1,254 @@
+#include "raft_server.h"
+#include <butil/files/file_enumerator.h>
+#include <thread>
+#include "rocksdb/utilities/checkpoint.h"
+
+
+void ReplicationClosure::Run() {
+ // Auto delete this after Run()
+ std::unique_ptr<ReplicationClosure> self_guard(this);
+
+ // Respond to this RPC.
+ brpc::ClosureGuard done_guard(done);
+ if (status().ok()) {
+ return;
+ }
+
+ // Try redirect if this request failed.
+ // TODO: searchStore->redirect(response);
+}
+
+// State machine implementation
+
+int ReplicationState::start(int port, int election_timeout_ms, int snapshot_interval_s,
+ const std::string & data_path, const std::string & peers) {
+ butil::EndPoint addr(butil::my_ip(), port);
+ braft::NodeOptions node_options;
+
+ if(node_options.initial_conf.parse_from(peers) != 0) {
+ LOG(ERROR) << "Fail to parse peer configuration `" << peers << "`";
+ return -1;
+ }
+
+ node_options.election_timeout_ms = election_timeout_ms;
+ node_options.fsm = this;
+ node_options.node_owns_fsm = false;
+ node_options.snapshot_interval_s = snapshot_interval_s;
+ std::string prefix = "local://" + data_path;
+ node_options.log_uri = prefix + "/log";
+ node_options.raft_meta_uri = prefix + "/raft_meta";
+ node_options.snapshot_uri = prefix + "/snapshot";
+ node_options.disable_cli = true;
+
+ braft::Node* node = new braft::Node("default_group", braft::PeerId(addr));
+ if (node->init(node_options) != 0) {
+ LOG(ERROR) << "Fail to init raft node";
+ delete node;
+ return -1;
+ }
+
+ _node = node;
+ return 0;
+}
+
+void ReplicationState::write(const http_req* request, http_res* response, google::protobuf::Closure* done) {
+ brpc::ClosureGuard done_guard(done);
+
+ if (!is_leader()) {
+ // TODO: return redirect(response);
+ }
+
+ // Serialize request to replicated WAL so that all the peers in the group receive it as well.
+ // NOTE: actual write must be done only on the `on_apply` method to maintain consistency.
+
+ butil::IOBufBuilder bufBuilder;
+ bufBuilder << request->serialize();
+ butil::IOBuf log = bufBuilder.buf();
+
+ // Apply this log as a braft::Task
+ braft::Task task;
+ task.data = &log;
+ // This callback would be invoked when the task actually executes or fails
+ task.done = new ReplicationClosure(this, _http_server, request, response, done_guard.release());
+
+ // To avoid ABA problem
+ task.expected_term = _leader_term.load(butil::memory_order_relaxed);
+
+ // Now the task is applied to the group, waiting for the result.
+ return _node->apply(task);
+}
+
+void ReplicationState::read(http_res* response) {
+ // For consistency, reads to follower must be rejected.
+ if (!is_leader()) {
+ // This node is a follower or it's not up-to-date. Redirect to
+ // the leader if possible.
+ return redirect(response);
+ }
+
+ // This is the leader and is up-to-date. It's safe to respond client
+ // TODO:
+ // response->set_value(_value.load(butil::memory_order_relaxed));
+}
+
+void ReplicationState::redirect(http_res* response) {
+ if (_node) {
+ braft::PeerId leader = _node->leader_id();
+ if (!leader.is_empty()) {
+ // TODO: response->set_redirect(leader.to_string());
+ }
+ }
+}
+
+void ReplicationState::on_apply(braft::Iterator& iter) {
+ // A batch of tasks are committed, which must be processed through
+ // |iter|
+ for (; iter.valid(); iter.next()) {
+ int64_t detal_value = 0;
+ http_res* response = NULL;
+ const http_req* request;
+ std::unique_ptr<http_req> remote_request(new http_req);
+
+ // Guard helps to invoke iter.done()->Run() asynchronously to avoid the callback blocking the StateMachine
+ braft::AsyncClosureGuard closure_guard(iter.done());
+
+ if (iter.done()) {
+ // This task is applied by this node, get value from the closure to avoid additional parsing.
+ ReplicationClosure* c = dynamic_cast<ReplicationClosure*>(iter.done());
+ response = c->get_response();
+ request = c->get_request();
+ } else {
+ // Parse request from the log
+ remote_request->deserialize(iter.data().to_string());
+ request = remote_request.get();
+ }
+
+ // Now that the log has been parsed, perform the actual operation
+ // TODO: call http server thread for write
+
+ if (response) {
+ // TODO: leader, so send response back to the client
+ }
+ }
+}
+
+void* ReplicationState::save_snapshot(void* arg) {
+ ReplicationState::SnapshotArg* sa = (ReplicationState::SnapshotArg*) arg;
+ std::unique_ptr<ReplicationState::SnapshotArg> arg_guard(sa);
+
+ brpc::ClosureGuard done_guard(sa->done);
+
+ std::string snapshot_path = sa->writer->get_path() + "/rocksdb_snapshot";
+
+ rocksdb::Checkpoint* checkpoint = NULL;
+ rocksdb::Status status = rocksdb::Checkpoint::Create(sa->db, &checkpoint);
+
+ if(!status.ok()) {
+ LOG(WARNING) << "Checkpoint Create failed, msg:" << status.ToString();
+ return NULL;
+ }
+
+ std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);
+ status = checkpoint->CreateCheckpoint(snapshot_path);
+ if(!status.ok()) {
+ LOG(WARNING) << "Checkpoint CreateCheckpoint failed, msg:" << status.ToString();
+ return NULL;
+ }
+
+ butil::FileEnumerator dir_enum(butil::FilePath(snapshot_path),false, butil::FileEnumerator::FILES);
+ for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {
+ std::string file_name = "rocksdb_snapshot/" + name.BaseName().value();
+ sa->writer->add_file(file_name);
+
+ if (sa->writer->add_file(file_name) != 0) {
+ sa->done->status().set_error(EIO, "Fail to add file to writer.");
+ return NULL;
+ }
+ }
+
+ return NULL;
+}
+
+void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done) {
+ // Start a new bthread to avoid blocking StateMachine since it could be slow to write data to disk
+ SnapshotArg* arg = new SnapshotArg;
+ arg->db = _db;
+ arg->writer = writer;
+ arg->done = done;
+ bthread_t tid;
+ bthread_start_urgent(&tid, NULL, save_snapshot, arg);
+}
+
+bool ReplicationState::copy_snapshot(const std::string& from_path, const std::string& to_path) {
+ struct stat from_stat;
+
+ if (stat(from_path.c_str(), &from_stat) < 0 || !S_ISDIR(from_stat.st_mode)) {
+ LOG(WARNING) << "stat " << from_path << " failed";
+ return false;
+ }
+
+ if (!butil::CreateDirectory(butil::FilePath(to_path))) {
+ LOG(WARNING) << "CreateDirectory " << to_path << " failed";
+ return false;
+ }
+
+ butil::FileEnumerator dir_enum(butil::FilePath(from_path),false, butil::FileEnumerator::FILES);
+ for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {
+ std::string src_file(from_path);
+ std::string dst_file(to_path);
+ butil::string_appendf(&src_file, "/%s", name.BaseName().value().c_str());
+ butil::string_appendf(&dst_file, "/%s", name.BaseName().value().c_str());
+
+ if (0 != link(src_file.c_str(), dst_file.c_str())) {
+ if (!butil::CopyFile(butil::FilePath(src_file), butil::FilePath(dst_file))) {
+ LOG(WARNING) << "copy " << src_file << " to " << dst_file << " failed";
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
+ // reset rocksdb handle
+ if (_db != NULL) {
+ delete _db;
+ _db = NULL;
+ }
+
+ CHECK(!is_leader()) << "Leader is not supposed to load snapshot";
+
+ // Load snapshot from reader, replacing the running StateMachine
+
+ std::string snapshot_path = reader->get_path();
+ snapshot_path.append("/rocksdb_snapshot");
+
+ std::string db_path = "./data/rocksdb_data"; // TODO
+ if (!butil::DeleteFile(butil::FilePath(db_path), true)) {
+ LOG(WARNING) << "rm " << db_path << " failed";
+ return -1;
+ }
+
+ LOG(TRACE) << "rm " << db_path << " success";
+ //TODO: try use link instead of copy
+
+ if (!copy_snapshot(snapshot_path, db_path)) {
+ LOG(WARNING) << "copy snapshot " << snapshot_path << " to " << db_path << " failed";
+ return -1;
+ }
+
+ LOG(TRACE) << "copy snapshot " << snapshot_path << " to " << db_path << " success";
+
+ return 0;
+}
+
+void ReplicationState::refresh_peers(const std::string & peers) {
+ if(_node && is_leader()) {
+ LOG(INFO) << "Refreshing peer config";
+ braft::Configuration conf;
+ conf.parse_from(peers);
+ RefreshPeersClosure* refresh_peers_done = new RefreshPeersClosure;
+ _node->change_peers(conf, refresh_peers_done);
+ }
+}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 9aab8252..1daa052a 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -1,8 +1,19 @@
-#include "core_api.h"
-#include "typesense_server_utils.h"
#include <curl/curl.h>
#include <sys/stat.h>
+#include <gflags/gflags.h>
+#include <brpc/controller.h>
+#include <brpc/server.h>
+#include <braft/raft.h>
+#include <braft/storage.h>
+#include <braft/util.h>
+#include <braft/protobuf_file.h>
+#include <raft_server.h>
+#include <fstream>
+
+#include "core_api.h"
+#include "typesense_server_utils.h"
+
HttpServer* server;
void catch_interrupt(int sig) {
@@ -16,6 +27,11 @@ bool directory_exists(const std::string & dir_path) {
return stat(dir_path.c_str(), &info) == 0 && (info.st_mode & S_IFDIR);
}
+bool file_exists(const std::string & file_path) {
+ struct stat info;
+ return stat(file_path.c_str(), &info) == 0;
+}
+
void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.set_program_name("./typesense-server");
@@ -25,6 +41,11 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("listen-address", 'h', "Address to which Typesense server binds.", false, "0.0.0.0");
options.add<uint32_t>("listen-port", 'p', "Port on which Typesense server listens.", false, 8108);
+
+ options.add<std::string>("raft-dir", '\0', "Directory where raft data will be stored.", false);
+ options.add<uint32_t>("raft-port", '\0', "Port on which Typesense raft service listens.", false, 8107);
+ options.add<std::string>("raft-peers", '\0', "Path to file having comma separated string of Raft node IPs.", false);
+
options.add<std::string>("master", 'm', "To start the server as read-only replica, "
"provide the master's address in http(s)://<master_address>:<master_port> format.",
false, "");
@@ -137,11 +158,89 @@ int run_server(const Config & config, const std::string & version,
replication_thread.detach();
}
+ std::thread raft_thread([&store, &config]() {
+ std::string path_to_peers = config.get_raft_peers();
+
+ if(path_to_peers.empty()) {
+ return 0;
+ }
+
+ if(!file_exists(path_to_peers)) {
+ LOG(ERR) << "Error reading file containing raft peers.";
+ return -1;
+ }
+
+ std::string peer_ips_string;
+ std::ifstream infile(path_to_peers);
+ std::getline(infile, peer_ips_string);
+ infile.close();
+
+ if(peer_ips_string.empty()) {
+ LOG(ERR) << "File containing raft peers is empty.";
+ return -1;
+ }
+
+ // start raft server
+ brpc::Server raft_server;
+ ReplicationState replication_state;
+
+ uint32_t raft_port = config.get_raft_port();
+
+ if (braft::add_service(&raft_server, raft_port) != 0) {
+ LOG(ERR) << "Failed to add raft service";
+ return -1; // TODO: must quit parent process as well
+ }
+
+ if (raft_server.Start(raft_port, NULL) != 0) {
+ LOG(ERR) << "Failed to start raft server";
+ return -1;
+ }
+
+ std::vector<std::string> peer_ips;
+ StringUtils::split(peer_ips_string, peer_ips, ",");
+ std::string peers = StringUtils::join(peer_ips, ":0,");
+
+ if (replication_state.start(raft_port, 1000, 3600, config.get_raft_dir(), peers) != 0) {
+ LOG(ERR) << "Failed to start raft state";
+ return -1;
+ }
+
+ LOG(INFO) << "Typesense raft service is running on " << raft_server.listen_address();
+
+ // Wait until 'CTRL-C' is pressed. then Stop() and Join() the service
+ size_t raft_counter = 0;
+ while (!brpc::IsAskedToQuit()) {
+ if(++raft_counter % 10 == 0) {
+ // reset peer configuration periodically to identify change in cluster membership
+ std::string new_peer_ips_string;
+ std::ifstream peers_infile(path_to_peers);
+ std::getline(peers_infile, new_peer_ips_string);
+ infile.close();
+ replication_state.refresh_peers(new_peer_ips_string);
+ }
+ sleep(1);
+ }
+
+ LOG(INFO) << "Typesense raft service is going to quit";
+
+ // Stop counter before server
+ replication_state.shutdown();
+ raft_server.Stop(0);
+
+ // Wait until all the processing tasks are over.
+ replication_state.join();
+ raft_server.Join();
+ return 0;
+ });
+
+ raft_thread.detach();
+
int ret_code = server->run();
+ // we are out of the event loop here
+
curl_global_cleanup();
- // we are out of the event loop here
delete server;
CollectionManager::get_instance().dispose();