Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Allow peers to be reset on error/on-demand.

Kishore Nallan56184aff26a5
diff --git a/include/core_api.h b/include/core_api.h
index fd5a4140..f435ccc4 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -119,6 +119,8 @@ bool post_clear_cache(const std::shared_ptr<http_req>& req, const std::shared_pt
bool post_compact_db(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+bool post_reset_peers(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+
// Rate Limiting
bool get_rate_limits(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
diff --git a/include/http_server.h b/include/http_server.h
index 55390ab1..d6be3291 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -326,6 +326,8 @@ public:
bool trigger_vote();
+ bool reset_peers();
+
void persist_applying_index();
int64_t get_num_queued_writes();
diff --git a/include/raft_server.h b/include/raft_server.h
index 5ae7cf27..40a09de2 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -139,6 +139,8 @@ private:
const uint64_t snapshot_interval_s; // frequency of actual snapshotting
uint64_t last_snapshot_ts; // when last snapshot ran
+ butil::EndPoint peering_endpoint;
+
public:
static constexpr const char* log_dir_name = "log";
@@ -163,12 +165,15 @@ public:
void read(const std::shared_ptr<http_res>& response);
// updates cluster membership
- void refresh_nodes(const std::string & nodes);
+ void refresh_nodes(const std::string & nodes, const size_t raft_counter,
+ const std::atomic<bool>& reset_peers_on_error);
void refresh_catchup_status(bool log_msg);
bool trigger_vote();
+ bool reset_peers();
+
bool has_leader_term() const {
return leader_term.load(butil::memory_order_acquire) > 0;
}
diff --git a/include/tsconfig.h b/include/tsconfig.h
index 0c02a188..57796930 100644
--- a/include/tsconfig.h
+++ b/include/tsconfig.h
@@ -62,6 +62,8 @@ private:
std::atomic<int> log_slow_searches_time_ms;
+ std::atomic<bool> reset_peers_on_error;
+
protected:
Config() {
@@ -84,6 +86,7 @@ protected:
this->memory_used_max_percentage = 100;
this->skip_writes = false;
this->log_slow_searches_time_ms = 30 * 1000;
+ this->reset_peers_on_error = false;
}
Config(Config const&) {
@@ -166,6 +169,10 @@ public:
this->skip_writes = skip_writes;
}
+ void set_reset_peers_on_error(bool reset_peers_on_error) {
+ this->reset_peers_on_error = reset_peers_on_error;
+ }
+
// getters
std::string get_data_dir() const {
@@ -265,6 +272,10 @@ public:
return this->log_slow_searches_time_ms;
}
+ const std::atomic<bool>& get_reset_peers_on_error() const {
+ return reset_peers_on_error;
+ }
+
size_t get_num_collections_parallel_load() const {
return this->num_collections_parallel_load;
}
@@ -419,6 +430,7 @@ public:
}
this->skip_writes = ("TRUE" == get_env("TYPESENSE_SKIP_WRITES"));
+ this->reset_peers_on_error = ("TRUE" == get_env("TYPESENSE_RESET_PEERS_ON_ERROR"));
}
void load_config_file(cmdline::parser & options) {
@@ -575,6 +587,11 @@ public:
this->skip_writes = (skip_writes_str == "true");
}
+ if(reader.Exists("server", "reset-peers-on-error")) {
+ auto reset_peers_on_error_str = reader.Get("server", "reset-peers-on-error", "false");
+ this->reset_peers_on_error = (reset_peers_on_error_str == "true");
+ }
+
if(reader.Exists("server", "model-dir")) {
this->model_dir = reader.Get("server", "model-dir", "");
}
@@ -715,6 +732,10 @@ public:
if(options.exist("skip-writes")) {
this->skip_writes = options.get<bool>("skip-writes");
}
+
+ if(options.exist("reset-peers-on-error")) {
+ this->reset_peers_on_error = options.get<bool>("reset-peers-on-error");
+ }
}
void set_cors_domains(std::string& cors_domains_value) {
@@ -743,4 +764,8 @@ public:
}
Option<bool> update_config(const nlohmann::json& req_json);
+
+ static Option<std::string> fetch_file_contents(const std::string & file_path);
+
+ static Option<std::string> fetch_nodes_config(const std::string& path_to_nodes);
};
diff --git a/src/core_api.cpp b/src/core_api.cpp
index c03bf8c5..6732d0aa 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1598,6 +1598,17 @@ bool post_compact_db(const std::shared_ptr<http_req>& req, const std::shared_ptr
return true;
}
+bool post_reset_peers(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ res->status_code = 200;
+ res->content_type_header = "application/json";
+
+ nlohmann::json response;
+ response["success"] = server->reset_peers();
+ res->body = response.dump();
+
+ return true;
+}
+
bool get_synonyms(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
auto collection = collectionManager.get_collection(req->params["collection"]);
diff --git a/src/http_server.cpp b/src/http_server.cpp
index b6cabeb9..5eb7dbb7 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -1034,6 +1034,10 @@ bool HttpServer::trigger_vote() {
return replication_state->trigger_vote();
}
+bool HttpServer::reset_peers() {
+ return replication_state->reset_peers();
+}
+
ThreadPool* HttpServer::get_thread_pool() const {
return thread_pool;
}
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index cb632f2a..87902022 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -77,6 +77,7 @@ void master_server_routes() {
server->post("/operations/vote", post_vote, false, false);
server->post("/operations/cache/clear", post_clear_cache, false, false);
server->post("/operations/db/compact", post_compact_db, false, false);
+ server->post("/operations/reset_peers", post_reset_peers, false, false);
server->get("/limits", get_rate_limits);
server->get("/limits/active", get_active_throttles);
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 1a1a45a5..6c8228fd 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -35,6 +35,7 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int
this->election_timeout_interval_ms = election_timeout_ms;
this->raft_dir_path = raft_dir;
+ this->peering_endpoint = peering_endpoint;
braft::NodeOptions node_options;
@@ -532,7 +533,8 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
return init_db_status;
}
-void ReplicationState::refresh_nodes(const std::string & nodes) {
+void ReplicationState::refresh_nodes(const std::string & nodes, const size_t raft_counter,
+ const std::atomic<bool>& reset_peers_on_error) {
std::shared_lock lock(node_mutex);
if(!node) {
@@ -569,8 +571,8 @@ void ReplicationState::refresh_nodes(const std::string & nodes) {
std::vector<braft::PeerId> latest_nodes;
new_conf.list_peers(&latest_nodes);
- if(latest_nodes.size() == 1) {
- LOG(WARNING) << "Single-node with no leader. Resetting peers.";
+ if(latest_nodes.size() == 1 || (raft_counter > 0 && reset_peers_on_error)) {
+ LOG(WARNING) << "Node with no leader. Resetting peers of size: " << latest_nodes.size();
node->reset_peers(new_conf);
} else {
LOG(WARNING) << "Multi-node with no leader: refusing to reset peers.";
@@ -773,6 +775,35 @@ bool ReplicationState::trigger_vote() {
return false;
}
+bool ReplicationState::reset_peers() {
+ std::shared_lock lock(node_mutex);
+
+ if(node) {
+ const Option<std::string> & refreshed_nodes_op = Config::fetch_nodes_config(config->get_nodes());
+ if(!refreshed_nodes_op.ok()) {
+ LOG(WARNING) << "Error while fetching peer configuration: " << refreshed_nodes_op.error();
+ return false;
+ }
+
+ const std::string& nodes_config = ReplicationState::to_nodes_config(peering_endpoint,
+ Config::get_instance().get_api_port(),
+ refreshed_nodes_op.get());
+
+ braft::Configuration peer_config;
+ peer_config.parse_from(nodes_config);
+
+ std::vector<braft::PeerId> peers;
+ peer_config.list_peers(&peers);
+
+ auto status = node->reset_peers(peer_config);
+ LOG(INFO) << "Reset peers. Ok? " << status.ok() << ", status: " << status;
+ LOG(INFO) << "New peer config is: " << peer_config;
+ return status.ok();
+ }
+
+ return false;
+}
+
http_message_dispatcher* ReplicationState::get_message_dispatcher() const {
return message_dispatcher;
}
diff --git a/src/tsconfig.cpp b/src/tsconfig.cpp
index a50664c6..ee7af9e1 100644
--- a/src/tsconfig.cpp
+++ b/src/tsconfig.cpp
@@ -1,6 +1,8 @@
#include "option.h"
#include "json.hpp"
#include "tsconfig.h"
+#include "file_utils.h"
+#include <fstream>
Option<bool> Config::update_config(const nlohmann::json& req_json) {
bool found_config = false;
@@ -54,3 +56,36 @@ Option<bool> Config::update_config(const nlohmann::json& req_json) {
return Option<bool>(true);
}
+
+Option<std::string> Config::fetch_file_contents(const std::string & file_path) {
+ if(!file_exists(file_path)) {
+ return Option<std::string>(404, std::string("File does not exist at: ") + file_path);
+ }
+
+ std::ifstream infile(file_path);
+ std::string content((std::istreambuf_iterator<char>(infile)), (std::istreambuf_iterator<char>()));
+ infile.close();
+
+ return Option<std::string>(content);
+}
+
+Option<std::string> Config::fetch_nodes_config(const std::string& path_to_nodes) {
+ std::string nodes_config;
+
+ if(!path_to_nodes.empty()) {
+ const Option<std::string> & nodes_op = fetch_file_contents(path_to_nodes);
+
+ if(!nodes_op.ok()) {
+ return Option<std::string>(500, "Error reading file containing nodes configuration: " + nodes_op.error());
+ } else {
+ nodes_config = nodes_op.get();
+ if(nodes_config.empty()) {
+ return Option<std::string>(500, "File containing nodes configuration is empty.");
+ } else {
+ nodes_config = nodes_op.get();
+ }
+ }
+ }
+
+ return Option<std::string>(nodes_config);
+}
\ No newline at end of file
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index cd9b4952..1ccf2e49 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -51,18 +51,6 @@ void catch_interrupt(int sig) {
quit_raft_service = true;
}
-Option<std::string> fetch_file_contents(const std::string & file_path) {
- if(!file_exists(file_path)) {
- return Option<std::string>(404, std::string("File does not exist at: ") + file_path);
- }
-
- std::ifstream infile(file_path);
- std::string content((std::istreambuf_iterator<char>(infile)), (std::istreambuf_iterator<char>()));
- infile.close();
-
- return Option<std::string>(content);
-}
-
void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.set_program_name("./typesense-server");
@@ -106,6 +94,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<int>("disk-used-max-percentage", '\0', "Reject writes when used disk space exceeds this percentage. Default: 100 (never reject).", false, 100);
options.add<int>("memory-used-max-percentage", '\0', "Reject writes when memory usage exceeds this percentage. Default: 100 (never reject).", false, 100);
options.add<bool>("skip-writes", '\0', "Skip all writes except config changes. Default: false.", false, false);
+ options.add<bool>("reset-peers-on-error", '\0', "Reset node's peers on clustering error. Default: false.", false, false);
options.add<int>("log-slow-searches-time-ms", '\0', "When >= 0, searches that take longer than this duration are logged.", false, 30*1000);
@@ -155,27 +144,6 @@ int init_root_logger(Config & config, const std::string & server_version) {
return 0;
}
-Option<std::string> fetch_nodes_config(const std::string& path_to_nodes) {
- std::string nodes_config;
-
- if(!path_to_nodes.empty()) {
- const Option<std::string> & nodes_op = fetch_file_contents(path_to_nodes);
-
- if(!nodes_op.ok()) {
- return Option<std::string>(500, "Error reading file containing nodes configuration: " + nodes_op.error());
- } else {
- nodes_config = nodes_op.get();
- if(nodes_config.empty()) {
- return Option<std::string>(500, "File containing nodes configuration is empty.");
- } else {
- nodes_config = nodes_op.get();
- }
- }
- }
-
- return Option<std::string>(nodes_config);
-}
-
bool is_private_ip(uint32_t ip) {
uint8_t b1, b2;
b1 = (uint8_t) (ip >> 24);
@@ -251,13 +219,14 @@ const char* get_internal_ip(const std::string& subnet_cidr) {
int start_raft_server(ReplicationState& replication_state, const std::string& state_dir, const std::string& path_to_nodes,
const std::string& peering_address, uint32_t peering_port, const std::string& peering_subnet,
- uint32_t api_port, int snapshot_interval_seconds, int snapshot_max_byte_count_per_rpc) {
+ uint32_t api_port, int snapshot_interval_seconds, int snapshot_max_byte_count_per_rpc,
+ const std::atomic<bool>& reset_peers_on_error) {
if(path_to_nodes.empty()) {
LOG(INFO) << "Since no --nodes argument is provided, starting a single node Typesense cluster.";
}
- const Option<std::string>& nodes_config_op = fetch_nodes_config(path_to_nodes);
+ const Option<std::string>& nodes_config_op = Config::fetch_nodes_config(path_to_nodes);
if(!nodes_config_op.ok()) {
LOG(ERROR) << nodes_config_op.error();
@@ -311,7 +280,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
while (!brpc::IsAskedToQuit() && !quit_raft_service.load()) {
if(raft_counter % 10 == 0) {
// reset peer configuration periodically to identify change in cluster membership
- const Option<std::string> & refreshed_nodes_op = fetch_nodes_config(path_to_nodes);
+ const Option<std::string> & refreshed_nodes_op = Config::fetch_nodes_config(path_to_nodes);
if(!refreshed_nodes_op.ok()) {
LOG(WARNING) << "Error while refreshing peer configuration: " << refreshed_nodes_op.error();
continue;
@@ -319,7 +288,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
const std::string& nodes_config = ReplicationState::to_nodes_config(peering_endpoint, api_port,
refreshed_nodes_op.get());
- replication_state.refresh_nodes(nodes_config);
+ replication_state.refresh_nodes(nodes_config, raft_counter, reset_peers_on_error);
if(raft_counter % 60 == 0) {
replication_state.do_snapshot(nodes_config);
@@ -459,6 +428,7 @@ int run_server(const Config & config, const std::string & version, void (*master
TextEmbedderManager::model_dir = config.get_model_dir();
TextEmbedderManager::download_default_model();
}
+
// first we start the peering service
ReplicationState replication_state(server, batch_indexer, &store,
@@ -482,7 +452,8 @@ int run_server(const Config & config, const std::string & version, void (*master
config.get_peering_subnet(),
config.get_api_port(),
config.get_snapshot_interval_seconds(),
- config.get_snapshot_max_byte_count_per_rpc());
+ config.get_snapshot_max_byte_count_per_rpc(),
+ config.get_reset_peers_on_error());
LOG(INFO) << "Shutting down batch indexer...";
batch_indexer->stop();