Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow peers to be reset on error/on-demand.
Kishore Nallan56184aff26a5
diff --git a/include/core_api.h b/include/core_api.hindex fd5a4140..f435ccc4 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -119,6 +119,8 @@ bool post_clear_cache(const std::shared_ptr<http_req>& req, const std::shared_ptbool post_compact_db(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool post_reset_peers(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+// Rate Limitingbool get_rate_limits(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);diff --git a/include/http_server.h b/include/http_server.hindex 55390ab1..d6be3291 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -326,6 +326,8 @@ public:bool trigger_vote();+ bool reset_peers();+void persist_applying_index();int64_t get_num_queued_writes();diff --git a/include/raft_server.h b/include/raft_server.hindex 5ae7cf27..40a09de2 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -139,6 +139,8 @@ private:const uint64_t snapshot_interval_s; // frequency of actual snapshottinguint64_t last_snapshot_ts; // when last snapshot ran+ butil::EndPoint peering_endpoint;+public:static constexpr const char* log_dir_name = "log";@@ -163,12 +165,15 @@ public:void read(const std::shared_ptr<http_res>& response);// updates cluster membership- void refresh_nodes(const std::string & nodes);+ void refresh_nodes(const std::string & nodes, const size_t raft_counter,+ const std::atomic<bool>& reset_peers_on_error);void refresh_catchup_status(bool log_msg);bool trigger_vote();+ bool reset_peers();+bool has_leader_term() const {return leader_term.load(butil::memory_order_acquire) > 0;}diff --git a/include/tsconfig.h b/include/tsconfig.hindex 0c02a188..57796930 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -62,6 +62,8 @@ private:std::atomic<int> log_slow_searches_time_ms;+ std::atomic<bool> reset_peers_on_error;+protected:Config() {@@ -84,6 +86,7 @@ protected:this->memory_used_max_percentage = 100;this->skip_writes = false;this->log_slow_searches_time_ms = 30 * 1000;+ this->reset_peers_on_error = false;}Config(Config const&) {@@ -166,6 +169,10 @@ public:this->skip_writes = skip_writes;}+ void set_reset_peers_on_error(bool reset_peers_on_error) {+ this->reset_peers_on_error = reset_peers_on_error;+ }+// gettersstd::string get_data_dir() const {@@ -265,6 +272,10 @@ public:return this->log_slow_searches_time_ms;}+ const std::atomic<bool>& get_reset_peers_on_error() const {+ return reset_peers_on_error;+ }+size_t get_num_collections_parallel_load() const {return this->num_collections_parallel_load;}@@ -419,6 +430,7 @@ public:}this->skip_writes = ("TRUE" == get_env("TYPESENSE_SKIP_WRITES"));+ this->reset_peers_on_error = ("TRUE" == get_env("TYPESENSE_RESET_PEERS_ON_ERROR"));}void load_config_file(cmdline::parser & options) {@@ -575,6 +587,11 @@ public:this->skip_writes = (skip_writes_str == "true");}+ if(reader.Exists("server", "reset-peers-on-error")) {+ auto reset_peers_on_error_str = reader.Get("server", "reset-peers-on-error", "false");+ this->reset_peers_on_error = (reset_peers_on_error_str == "true");+ }+if(reader.Exists("server", "model-dir")) {this->model_dir = reader.Get("server", "model-dir", "");}@@ -715,6 +732,10 @@ public:if(options.exist("skip-writes")) {this->skip_writes = options.get<bool>("skip-writes");}++ if(options.exist("reset-peers-on-error")) {+ this->reset_peers_on_error = options.get<bool>("reset-peers-on-error");+ }}void set_cors_domains(std::string& cors_domains_value) {@@ -743,4 +764,8 @@ public:}Option<bool> update_config(const nlohmann::json& req_json);++ static Option<std::string> fetch_file_contents(const std::string & file_path);++ static Option<std::string> fetch_nodes_config(const std::string& path_to_nodes);};diff --git a/src/core_api.cpp b/src/core_api.cppindex c03bf8c5..6732d0aa 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1598,6 +1598,17 @@ bool post_compact_db(const std::shared_ptr<http_req>& req, const std::shared_ptrreturn true;}+bool post_reset_peers(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ res->status_code = 200;+ res->content_type_header = "application/json";++ nlohmann::json response;+ response["success"] = server->reset_peers();+ res->body = response.dump();++ return true;+}+bool get_synonyms(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {CollectionManager & collectionManager = CollectionManager::get_instance();auto collection = collectionManager.get_collection(req->params["collection"]);diff --git a/src/http_server.cpp b/src/http_server.cppindex b6cabeb9..5eb7dbb7 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -1034,6 +1034,10 @@ bool HttpServer::trigger_vote() {return replication_state->trigger_vote();}+bool HttpServer::reset_peers() {+ return replication_state->reset_peers();+}+ThreadPool* HttpServer::get_thread_pool() const {return thread_pool;}diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex cb632f2a..87902022 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -77,6 +77,7 @@ void master_server_routes() {server->post("/operations/vote", post_vote, false, false);server->post("/operations/cache/clear", post_clear_cache, false, false);server->post("/operations/db/compact", post_compact_db, false, false);+ server->post("/operations/reset_peers", post_reset_peers, false, false);server->get("/limits", get_rate_limits);server->get("/limits/active", get_active_throttles);diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 1a1a45a5..6c8228fd 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -35,6 +35,7 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const intthis->election_timeout_interval_ms = election_timeout_ms;this->raft_dir_path = raft_dir;+ this->peering_endpoint = peering_endpoint;braft::NodeOptions node_options;@@ -532,7 +533,8 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {return init_db_status;}-void ReplicationState::refresh_nodes(const std::string & nodes) {+void ReplicationState::refresh_nodes(const std::string & nodes, const size_t raft_counter,+ const std::atomic<bool>& reset_peers_on_error) {std::shared_lock lock(node_mutex);if(!node) {@@ -569,8 +571,8 @@ void ReplicationState::refresh_nodes(const std::string & nodes) {std::vector<braft::PeerId> latest_nodes;new_conf.list_peers(&latest_nodes);- if(latest_nodes.size() == 1) {- LOG(WARNING) << "Single-node with no leader. Resetting peers.";+ if(latest_nodes.size() == 1 || (raft_counter > 0 && reset_peers_on_error)) {+ LOG(WARNING) << "Node with no leader. Resetting peers of size: " << latest_nodes.size();node->reset_peers(new_conf);} else {LOG(WARNING) << "Multi-node with no leader: refusing to reset peers.";@@ -773,6 +775,35 @@ bool ReplicationState::trigger_vote() {return false;}+bool ReplicationState::reset_peers() {+ std::shared_lock lock(node_mutex);++ if(node) {+ const Option<std::string> & refreshed_nodes_op = Config::fetch_nodes_config(config->get_nodes());+ if(!refreshed_nodes_op.ok()) {+ LOG(WARNING) << "Error while fetching peer configuration: " << refreshed_nodes_op.error();+ return false;+ }++ const std::string& nodes_config = ReplicationState::to_nodes_config(peering_endpoint,+ Config::get_instance().get_api_port(),+ refreshed_nodes_op.get());++ braft::Configuration peer_config;+ peer_config.parse_from(nodes_config);++ std::vector<braft::PeerId> peers;+ peer_config.list_peers(&peers);++ auto status = node->reset_peers(peer_config);+ LOG(INFO) << "Reset peers. Ok? " << status.ok() << ", status: " << status;+ LOG(INFO) << "New peer config is: " << peer_config;+ return status.ok();+ }++ return false;+}+http_message_dispatcher* ReplicationState::get_message_dispatcher() const {return message_dispatcher;}diff --git a/src/tsconfig.cpp b/src/tsconfig.cppindex a50664c6..ee7af9e1 100644--- a/src/tsconfig.cpp+++ b/src/tsconfig.cpp@@ -1,6 +1,8 @@#include "option.h"#include "json.hpp"#include "tsconfig.h"+#include "file_utils.h"+#include <fstream>Option<bool> Config::update_config(const nlohmann::json& req_json) {bool found_config = false;@@ -54,3 +56,36 @@ Option<bool> Config::update_config(const nlohmann::json& req_json) {return Option<bool>(true);}++Option<std::string> Config::fetch_file_contents(const std::string & file_path) {+ if(!file_exists(file_path)) {+ return Option<std::string>(404, std::string("File does not exist at: ") + file_path);+ }++ std::ifstream infile(file_path);+ std::string content((std::istreambuf_iterator<char>(infile)), (std::istreambuf_iterator<char>()));+ infile.close();++ return Option<std::string>(content);+}++Option<std::string> Config::fetch_nodes_config(const std::string& path_to_nodes) {+ std::string nodes_config;++ if(!path_to_nodes.empty()) {+ const Option<std::string> & nodes_op = fetch_file_contents(path_to_nodes);++ if(!nodes_op.ok()) {+ return Option<std::string>(500, "Error reading file containing nodes configuration: " + nodes_op.error());+ } else {+ nodes_config = nodes_op.get();+ if(nodes_config.empty()) {+ return Option<std::string>(500, "File containing nodes configuration is empty.");+ } else {+ nodes_config = nodes_op.get();+ }+ }+ }++ return Option<std::string>(nodes_config);+}\ No newline at end of filediff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex cd9b4952..1ccf2e49 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -51,18 +51,6 @@ void catch_interrupt(int sig) {quit_raft_service = true;}-Option<std::string> fetch_file_contents(const std::string & file_path) {- if(!file_exists(file_path)) {- return Option<std::string>(404, std::string("File does not exist at: ") + file_path);- }-- std::ifstream infile(file_path);- std::string content((std::istreambuf_iterator<char>(infile)), (std::istreambuf_iterator<char>()));- infile.close();-- return Option<std::string>(content);-}-void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.set_program_name("./typesense-server");@@ -106,6 +94,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<int>("disk-used-max-percentage", '\0', "Reject writes when used disk space exceeds this percentage. Default: 100 (never reject).", false, 100);options.add<int>("memory-used-max-percentage", '\0', "Reject writes when memory usage exceeds this percentage. Default: 100 (never reject).", false, 100);options.add<bool>("skip-writes", '\0', "Skip all writes except config changes. Default: false.", false, false);+ options.add<bool>("reset-peers-on-error", '\0', "Reset node's peers on clustering error. Default: false.", false, false);options.add<int>("log-slow-searches-time-ms", '\0', "When >= 0, searches that take longer than this duration are logged.", false, 30*1000);@@ -155,27 +144,6 @@ int init_root_logger(Config & config, const std::string & server_version) {return 0;}-Option<std::string> fetch_nodes_config(const std::string& path_to_nodes) {- std::string nodes_config;-- if(!path_to_nodes.empty()) {- const Option<std::string> & nodes_op = fetch_file_contents(path_to_nodes);-- if(!nodes_op.ok()) {- return Option<std::string>(500, "Error reading file containing nodes configuration: " + nodes_op.error());- } else {- nodes_config = nodes_op.get();- if(nodes_config.empty()) {- return Option<std::string>(500, "File containing nodes configuration is empty.");- } else {- nodes_config = nodes_op.get();- }- }- }-- return Option<std::string>(nodes_config);-}-bool is_private_ip(uint32_t ip) {uint8_t b1, b2;b1 = (uint8_t) (ip >> 24);@@ -251,13 +219,14 @@ const char* get_internal_ip(const std::string& subnet_cidr) {int start_raft_server(ReplicationState& replication_state, const std::string& state_dir, const std::string& path_to_nodes,const std::string& peering_address, uint32_t peering_port, const std::string& peering_subnet,- uint32_t api_port, int snapshot_interval_seconds, int snapshot_max_byte_count_per_rpc) {+ uint32_t api_port, int snapshot_interval_seconds, int snapshot_max_byte_count_per_rpc,+ const std::atomic<bool>& reset_peers_on_error) {if(path_to_nodes.empty()) {LOG(INFO) << "Since no --nodes argument is provided, starting a single node Typesense cluster.";}- const Option<std::string>& nodes_config_op = fetch_nodes_config(path_to_nodes);+ const Option<std::string>& nodes_config_op = Config::fetch_nodes_config(path_to_nodes);if(!nodes_config_op.ok()) {LOG(ERROR) << nodes_config_op.error();@@ -311,7 +280,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& stwhile (!brpc::IsAskedToQuit() && !quit_raft_service.load()) {if(raft_counter % 10 == 0) {// reset peer configuration periodically to identify change in cluster membership- const Option<std::string> & refreshed_nodes_op = fetch_nodes_config(path_to_nodes);+ const Option<std::string> & refreshed_nodes_op = Config::fetch_nodes_config(path_to_nodes);if(!refreshed_nodes_op.ok()) {LOG(WARNING) << "Error while refreshing peer configuration: " << refreshed_nodes_op.error();continue;@@ -319,7 +288,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& stconst std::string& nodes_config = ReplicationState::to_nodes_config(peering_endpoint, api_port,refreshed_nodes_op.get());- replication_state.refresh_nodes(nodes_config);+ replication_state.refresh_nodes(nodes_config, raft_counter, reset_peers_on_error);if(raft_counter % 60 == 0) {replication_state.do_snapshot(nodes_config);@@ -459,6 +428,7 @@ int run_server(const Config & config, const std::string & version, void (*masterTextEmbedderManager::model_dir = config.get_model_dir();TextEmbedderManager::download_default_model();}+// first we start the peering serviceReplicationState replication_state(server, batch_indexer, &store,@@ -482,7 +452,8 @@ int run_server(const Config & config, const std::string & version, void (*masterconfig.get_peering_subnet(),config.get_api_port(),config.get_snapshot_interval_seconds(),- config.get_snapshot_max_byte_count_per_rpc());+ config.get_snapshot_max_byte_count_per_rpc(),+ config.get_reset_peers_on_error());LOG(INFO) << "Shutting down batch indexer...";batch_indexer->stop();