Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Handle membership changes more gracefully.

kishorencd9b08d4a906c
diff --git a/include/raft_server.h b/include/raft_server.h
index 1866af3b..d1500beb 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -80,6 +80,7 @@ private:
braft::Node* volatile node;
butil::atomic<int64_t> leader_term;
+ std::set<braft::PeerId> peers;
Store *store;
ThreadPool* thread_pool;
@@ -148,6 +149,9 @@ public:
size_t get_init_readiness_count() const;
+ static std::string to_nodes_config(const butil::EndPoint &peering_endpoint, const int api_port,
+ const std::string &nodes_config);
+
static constexpr const char* REPLICATION_MSG = "raft_replication";
private:
@@ -199,6 +203,7 @@ private:
void on_configuration_committed(const ::braft::Configuration& conf) {
LOG(INFO) << "Configuration of this group is " << conf;
+ conf.list_peers(&peers);
}
void on_start_following(const ::braft::LeaderChangeContext& ctx) {
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 151bdc82..b543cafb 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -2,6 +2,7 @@
#include "raft_server.h"
#include <butil/files/file_enumerator.h>
#include <thread>
+#include <algorithm>
#include <string_utils.h>
#include <file_utils.h>
#include <collection_manager.h>
@@ -22,16 +23,10 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int
const std::string & raft_dir, const std::string & nodes) {
braft::NodeOptions node_options;
+ std::string actual_nodes_config = to_nodes_config(peering_endpoint, api_port, nodes);
- std::string actual_nodes = nodes;
-
- if(actual_nodes.empty()) {
- std::string ip_str = butil::ip2str(peering_endpoint.ip).c_str();
- actual_nodes = ip_str + ":" + std::to_string(peering_endpoint.port) + ":" + std::to_string(api_port);
- }
-
- if(node_options.initial_conf.parse_from(actual_nodes) != 0) {
- LOG(ERROR) << "Failed to parse peer configuration `" << nodes << "`";
+ if(node_options.initial_conf.parse_from(actual_nodes_config) != 0) {
+ LOG(ERROR) << "Failed to parse nodes configuration `" << nodes << "`";
return -1;
}
@@ -77,18 +72,24 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int
}
std::vector<std::string> peer_vec;
- StringUtils::split(actual_nodes, peer_vec, ",");
-
- if(peer_vec.size() == 1) {
- // NOTE: `reset_peers` is NOT safe to run on a cluster of nodes, but okay for standalone
- // This is handy to reset local state if the instance is started on a different IP
- node->reset_peers(node_options.initial_conf);
- }
+ StringUtils::split(actual_nodes_config, peer_vec, ",");
this->node = node;
return 0;
}
+std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_endpoint, const int api_port,
+ const std::string& nodes_config) {
+ std::string actual_nodes_config = nodes_config;
+
+ if(nodes_config.empty()) {
+ std::string ip_str = butil::ip2str(peering_endpoint.ip).c_str();
+ actual_nodes_config = ip_str + ":" + std::to_string(peering_endpoint.port) + ":" + std::to_string(api_port);
+ }
+
+ return actual_nodes_config;
+}
+
void ReplicationState::write(http_req* request, http_res* response) {
if (!is_leader()) {
if(node->leader_id().is_empty()) {
@@ -317,17 +318,29 @@ void ReplicationState::refresh_nodes(const std::string & nodes) {
return ;
}
- braft::Configuration conf;
- conf.parse_from(nodes);
+ braft::Configuration new_conf;
+ new_conf.parse_from(nodes);
if(is_leader()) {
RefreshNodesClosure* refresh_nodes_done = new RefreshNodesClosure;
- node->change_peers(conf, refresh_nodes_done);
- } else {
- // if the node is not a leader and is also not able to find a leader, we have to forcefully reset the peers
- if(node->leader_id().is_empty()) {
- LOG(WARNING) << "No leader: resetting peers.";
- node->reset_peers(conf);
+ node->change_peers(new_conf, refresh_nodes_done);
+ } else if(node->leader_id().is_empty()) {
+ // Reset the peers of a node that:
+ // a) is NOT a leader
+ // b) does NOT have a leader
+ // c) every node except self has changed
+
+ std::set<braft::PeerId> new_peers;
+ new_conf.list_peers(&new_peers);
+
+ std::set<braft::PeerId> intersect;
+ std::set_intersection(peers.begin(), peers.end(), new_peers.begin(), new_peers.end(),
+ std::inserter(intersect, intersect.end()));
+
+ if(intersect.empty() ||
+ (intersect.size() == 1 && (*intersect.begin()).to_string() == node->node_id().peer_id.to_string())) {
+ LOG(WARNING) << "Quorum change: resetting peers.";
+ node->reset_peers(new_conf);
}
}
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index be62396c..858937f6 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -83,7 +83,7 @@ void stream_response(bool (*handler)(http_req* req, http_res* res, void* data),
http_req & request, http_res & response, void* data) {
h2o_req_t* req = request._req;
h2o_custom_generator_t* custom_generator = new h2o_custom_generator_t {
- h2o_generator_t {response_proceed, response_stop}, handler, &request, &response, data
+ h2o_generator_t {response_proceed, response_stop}, handler, &request, &response, data
};
req->res.status = response.status_code;
@@ -173,30 +173,41 @@ bool on_send_response(void *data) {
return true;
}
-int start_raft_server(ReplicationState& replication_state, const std::string& state_dir, const std::string& path_to_nodes,
- const std::string& peering_address, uint32_t peering_port, uint32_t api_port) {
-
- std::string peer_ips;
+Option<std::string> fetch_nodes_config(const std::string& path_to_nodes) {
+ std::string nodes_config;
- if(path_to_nodes.empty()) {
- LOG(INFO) << "Since no --nodes argument is provided, starting a single node Typesense cluster.";
- } else {
+ if(!path_to_nodes.empty()) {
const Option<std::string> & nodes_op = fetch_file_contents(path_to_nodes);
if(!nodes_op.ok()) {
- LOG(ERROR) << "Error reading file containing nodes configuration: " << nodes_op.error();
- exit(-1);
+ return Option<std::string>(500, "Error reading file containing nodes configuration: " + nodes_op.error());
} else {
- peer_ips = nodes_op.get();
- if(peer_ips.empty()) {
- LOG(ERROR) << "File containing nodes configuration is empty.";
- exit(-1);
+ nodes_config = nodes_op.get();
+ if(nodes_config.empty()) {
+ return Option<std::string>(500, "File containing nodes configuration is empty.");
} else {
- peer_ips = nodes_op.get();
+ nodes_config = nodes_op.get();
}
}
}
+ return Option<std::string>(nodes_config);
+}
+
+int start_raft_server(ReplicationState& replication_state, const std::string& state_dir, const std::string& path_to_nodes,
+ const std::string& peering_address, uint32_t peering_port, uint32_t api_port) {
+
+ if(path_to_nodes.empty()) {
+ LOG(INFO) << "Since no --nodes argument is provided, starting a single node Typesense cluster.";
+ }
+
+ const Option<std::string>& nodes_config_op = fetch_nodes_config(path_to_nodes);
+
+ if(!nodes_config_op.ok()) {
+ LOG(ERROR) << nodes_config_op.error();
+ exit(-1);
+ }
+
butil::ip_t peering_ip;
if(!peering_address.empty()) {
int ip_conv_status = butil::str2ip(peering_address.c_str(), &peering_ip);
@@ -223,7 +234,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
exit(-1);
}
- if (replication_state.start(peering_endpoint, api_port, 1000, 600, state_dir, peer_ips) != 0) {
+ if (replication_state.start(peering_endpoint, api_port, 1000, 600, state_dir, nodes_config_op.get()) != 0) {
LOG(ERROR) << "Failed to start peering state";
exit(-1);
}
@@ -233,16 +244,17 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
// Wait until 'CTRL-C' is pressed. then Stop() and Join() the service
size_t raft_counter = 0;
while (!brpc::IsAskedToQuit() && !quit_raft_service.load()) {
- if(++raft_counter % 10 == 0 && !path_to_nodes.empty()) {
+ // pre-increment to ensure that we don't refresh right away on a fresh boot
+ if(++raft_counter % 10 == 0) {
// reset peer configuration periodically to identify change in cluster membership
- const Option<std::string> & refreshed_node_op = fetch_file_contents(path_to_nodes);
- if(!refreshed_node_op.ok()) {
- LOG(ERROR) << "Error while refreshing peer configuration: " << refreshed_node_op.error();
+ const Option<std::string> & refreshed_nodes_op = fetch_nodes_config(path_to_nodes);
+ if(!refreshed_nodes_op.ok()) {
+ LOG(ERROR) << "Error while refreshing peer configuration: " << refreshed_nodes_op.error();
continue;
}
-
- const std::string & refreshed_node_ips_string = refreshed_node_op.get();
- replication_state.refresh_nodes(refreshed_node_ips_string);
+ const std::string& nodes_config = ReplicationState::to_nodes_config(peering_endpoint, api_port,
+ refreshed_nodes_op.get());
+ replication_state.refresh_nodes(nodes_config);
}
sleep(1);