Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Retry parsing of nodes config: helps DNS resolution lag.

Kishore Nallan8e4fc412b129
diff --git a/include/raft_server.h b/include/raft_server.h
index 1581d80f..00f9adb4 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -143,7 +143,8 @@ public:
// Starts this node
int start(const butil::EndPoint & peering_endpoint, int api_port,
int election_timeout_ms, int snapshot_interval_s,
- const std::string & raft_dir, const std::string & nodes);
+ const std::string & raft_dir, const std::string & nodes,
+ const std::atomic<bool>& quit_abruptly);
// Generic write method for synchronizing all writes
void write(const std::shared_ptr<http_req>& request, const std::shared_ptr<http_res>& response);
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index c1fe2016..9728edd9 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -28,20 +28,42 @@ void ReplicationClosure::Run() {
int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int api_port,
int election_timeout_ms, int snapshot_interval_s,
- const std::string & raft_dir, const std::string & nodes) {
+ const std::string & raft_dir, const std::string & nodes,
+ const std::atomic<bool>& quit_abruptly) {
this->election_timeout_interval_ms = election_timeout_ms;
this->raft_dir_path = raft_dir;
braft::NodeOptions node_options;
- std::string actual_nodes_config = to_nodes_config(peering_endpoint, api_port, nodes);
- if(node_options.initial_conf.parse_from(actual_nodes_config) != 0) {
- LOG(ERROR) << "Failed to parse nodes configuration `" << nodes << "`";
- return -1;
- }
+ size_t max_tries = 3;
- LOG(INFO) << "Nodes config: " << actual_nodes_config;
+ while(true) {
+ std::string actual_nodes_config = to_nodes_config(peering_endpoint, api_port, nodes);
+
+ if(node_options.initial_conf.parse_from(actual_nodes_config) != 0) {
+ if(--max_tries == 0) {
+ LOG(ERROR) << "Giving up parsing nodes configuration: `" << nodes << "`";
+ return -1;
+ }
+
+ LOG(ERROR) << "Failed to parse nodes configuration: `" << nodes << "` -- " << " will retry shortly...";
+
+ size_t i = 0;
+ while(i++ < 30) {
+ std::this_thread::sleep_for(std::chrono::seconds(1));
+ if(quit_abruptly) {
+ // enables quitting of server during retries
+ return -1;
+ }
+ }
+
+ continue;
+ }
+
+ LOG(INFO) << "Nodes configuration: " << actual_nodes_config;
+ break;
+ }
this->read_caught_up = false;
this->write_caught_up = false;
@@ -150,11 +172,13 @@ string ReplicationState::resolve_node_hosts(const string& nodes_config) {
butil::ip_t ip;
int status = butil::hostname2ip(node_parts[0].c_str(), &ip);
+
if(status == 0) {
final_nodes_vec.push_back(
std::string(butil::ip2str(ip).c_str()) + ":" + node_parts[1] + ":" + node_parts[2]
);
} else {
+ LOG(ERROR) << "Unable to resolve host: " << node_parts[0];
final_nodes_vec.push_back(node_str);
}
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 93b84eef..924f2a58 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -252,7 +252,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
size_t election_timeout_ms = 5000;
if (replication_state.start(peering_endpoint, api_port, election_timeout_ms, snapshot_interval_seconds, state_dir,
- nodes_config_op.get()) != 0) {
+ nodes_config_op.get(), quit_raft_service) != 0) {
LOG(ERROR) << "Failed to start peering state";
exit(-1);
}