Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Protect API until follower has caught up with leader.
kishorenca2b421209cc3
diff --git a/include/config.h b/include/config.hindex d601f350..61cd508d 100644--- a/include/config.h+++ b/include/config.h@@ -32,6 +32,8 @@ private:float max_memory_ratio;int snapshot_interval_seconds;+ size_t catch_up_threshold_percentage;+std::string config_file;int config_file_validity;@@ -44,6 +46,7 @@ public:this->enable_cors = false;this->max_memory_ratio = 1.0f;this->snapshot_interval_seconds = 3600;+ this->catch_up_threshold_percentage = 95;}// setters@@ -156,6 +159,10 @@ public:return this->snapshot_interval_seconds;}+ int get_catch_up_threshold_percentage() const {+ return this->catch_up_threshold_percentage;+ }+// loadersstd::string get_env(const char *name) {@@ -215,6 +222,10 @@ public:if(!get_env("TYPESENSE_SNAPSHOT_INTERVAL_SECONDS").empty()) {this->snapshot_interval_seconds = std::stoi(get_env("TYPESENSE_SNAPSHOT_INTERVAL_SECONDS"));}++ if(!get_env("TYPESENSE_CATCH_UP_THRESHOLD_PERCENTAGE").empty()) {+ this->catch_up_threshold_percentage = std::stoi(get_env("TYPESENSE_CATCH_UP_THRESHOLD_PERCENTAGE"));+ }}void load_config_file(cmdline::parser & options) {@@ -304,6 +315,10 @@ public:if(reader.Exists("server", "snapshot-interval-seconds")) {this->snapshot_interval_seconds = (int) reader.GetInteger("server", "snapshot-interval-seconds", 3600);}++ if(reader.Exists("server", "catch-up-threshold-percentage")) {+ this->catch_up_threshold_percentage = (int) reader.GetInteger("server", "catch-up-threshold-percentage", 95);+ }}void load_config_cmd_args(cmdline::parser & options) {@@ -375,6 +390,10 @@ public:if(options.exist("snapshot-interval-seconds")) {this->snapshot_interval_seconds = options.get<int>("snapshot-interval-seconds");}++ if(options.exist("catch-up-threshold-percentage")) {+ this->catch_up_threshold_percentage = options.get<int>("catch-up-threshold-percentage");+ }}// validationdiff --git a/include/core_api.h b/include/core_api.hindex 6f29771b..f4536644 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -67,6 +67,8 @@ bool get_health(http_req& req, http_res& res);bool get_metrics_json(http_req& req, http_res& res);+bool get_log_sequence(http_req& req, http_res& res);+// Misc helpersbool raft_write_send_response(void *data);diff --git a/include/http_server.h b/include/http_server.hindex 0a30e4b5..bf209b32 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -58,6 +58,8 @@ private:bool exit_loop = false;+ bool ssl_enabled = false;+private:std::string version;@@ -171,4 +173,8 @@ public:static void on_deferred_process_request(h2o_timer_t *entry);void defer_processing(http_req& req, http_res& res, size_t timeout_ms);++ bool is_ssl_enabled() {+ return ssl_enabled;+ }};diff --git a/include/raft_server.h b/include/raft_server.hindex f5bc9354..f793e938 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -86,7 +86,10 @@ private:ThreadPool* thread_pool;http_message_dispatcher* message_dispatcher;- std::atomic<size_t> init_readiness_count;+ const size_t catch_up_threshold_percentage;+ std::atomic<bool> caught_up;++ const bool api_uses_ssl;bool create_init_db_snapshot;@@ -99,6 +102,7 @@ public:static constexpr const char* snapshot_dir_name = "snapshot";ReplicationState(Store* store, ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,+ bool api_uses_ssl, size_t catch_up_threshold_percentage,bool create_init_db_snapshot, std::atomic<bool>& quit_service);// Starts this node@@ -120,7 +124,7 @@ public:}bool is_ready() const {- return init_readiness_count >= 2;+ return caught_up;}bool is_alive() const;@@ -149,8 +153,6 @@ public:void reset_db();- size_t get_init_readiness_count() const;-static std::string to_nodes_config(const butil::EndPoint &peering_endpoint, const int api_port,const std::string &nodes_config);@@ -185,8 +187,6 @@ private:write(request, response);}- init_readiness_count++;-LOG(INFO) << "Node becomes leader, term: " << term;}@@ -210,7 +210,6 @@ private:void on_start_following(const ::braft::LeaderChangeContext& ctx) {LOG(INFO) << "Node starts following " << ctx;- init_readiness_count++;}void on_stop_following(const ::braft::LeaderChangeContext& ctx) {diff --git a/src/core_api.cpp b/src/core_api.cppindex 905d48db..34347a90 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -213,6 +213,15 @@ bool get_metrics_json(http_req &req, http_res &res) {return true;}++bool get_log_sequence(http_req &req, http_res &res) {+ CollectionManager & collectionManager = CollectionManager::get_instance();+ const uint64_t seq_num = collectionManager.get_store()->get_latest_seq_number();+ res.content_type_header = "text/plain; charset=utf8";+ res.set_body(200, std::to_string(seq_num));+ return true;+}+bool get_search(http_req & req, http_res & res) {auto begin = std::chrono::high_resolution_clock::now();diff --git a/src/http_server.cpp b/src/http_server.cppindex f0fa153f..590a7511 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -112,6 +112,7 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {}h2o_ssl_register_alpn_protocols(accept_ctx->ssl_ctx, h2o_http2_alpn_protocols);+ ssl_enabled = true;return 0;}@@ -326,7 +327,8 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {// Except for health check, wait for replicating state to be ready before allowing requests// Follower or leader must have started AND data must also have been loaded- if(path_without_query != "/health" && !h2o_handler->http_server->get_replication_state()->is_ready()) {+ if(path_without_query != "/health" && path_without_query != "/sequence" &&+ !h2o_handler->http_server->get_replication_state()->is_ready()) {std::string message = "{ \"message\": \"Not Ready\"}";return send_response(req, 503, message);}diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 9e91b7d0..4164110e 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -48,21 +48,7 @@ void master_server_routes() {server->get("/metrics.json", get_metrics_json);server->get("/debug", get_debug);server->get("/health", get_health);-}--void replica_server_routes() {- // collection management- server->get("/collections", get_collections);- server->get("/collections/:collection", get_collection_summary);-- // document management - `/documents/:id` end-points must be placed last in the list- server->get("/collections/:collection/documents/search", get_search);- server->get("/collections/:collection/documents/export", get_export_documents, true);- server->get("/collections/:collection/documents/:id", get_fetch_document);-- // meta- server->get("/debug", get_debug);- server->get("/health", get_health);+ server->get("/sequence", get_log_sequence);}int main(int argc, char **argv) {diff --git a/src/raft_server.cpp b/src/raft_server.cppindex b8118306..f5308752 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -33,6 +33,8 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const intreturn -1;}+ this->caught_up = false;+// do snapshot only when the gap between applied index and last snapshot index is >= this numberbraft::FLAGS_raft_do_snapshot_min_index_gap = 1;@@ -361,8 +363,6 @@ int ReplicationState::init_db() {return 1;}- init_readiness_count++;-return 0;}@@ -407,30 +407,77 @@ void ReplicationState::refresh_nodes(const std::string & nodes) {if(node->is_leader()) {RefreshNodesClosure* refresh_nodes_done = new RefreshNodesClosure;node->change_peers(new_conf, refresh_nodes_done);- } else if(node->leader_id().is_empty()) {- // When node is not a leader, does not have a leader and is also a single-node cluster,- // we forcefully reset its peers.- // NOTE: `reset_peers()` is not a safe call to make as we give up on consistency and consensus guarantees.- // We are doing this solely to handle single node cluster whose IP changes.- // Examples: Docker container IP change, local DHCP leased IP change etc.-- std::vector<braft::PeerId> latest_nodes;- new_conf.list_peers(&latest_nodes);-- if(latest_nodes.size() == 1) {- LOG(WARNING) << "Single-node with no leader. Resetting peers.";- node->reset_peers(new_conf);- } else {- LOG(WARNING) << "Multi-node with no leader: refusing to reset peers.";+ this->caught_up = true;+ } else {+ if(node->leader_id().is_empty()) {+ // When node is not a leader, does not have a leader and is also a single-node cluster,+ // we forcefully reset its peers.+ // NOTE: `reset_peers()` is not a safe call to make as we give up on consistency and consensus guarantees.+ // We are doing this solely to handle single node cluster whose IP changes.+ // Examples: Docker container IP change, local DHCP leased IP change etc.++ std::vector<braft::PeerId> latest_nodes;+ new_conf.list_peers(&latest_nodes);++ if(latest_nodes.size() == 1) {+ LOG(WARNING) << "Single-node with no leader. Resetting peers.";+ node->reset_peers(new_conf);+ } else {+ LOG(WARNING) << "Multi-node with no leader: refusing to reset peers.";+ }++ this->caught_up = false;+ return ;}++ // update catch up status+ thread_pool->enqueue([this]() {+ auto seq_num = this->store->get_latest_seq_number();+ const std::string & leader_addr = node->leader_id().to_string();++ std::vector<std::string> addr_parts;+ StringUtils::split(leader_addr, addr_parts, ":");+ std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];+ std::string protocol = this->api_uses_ssl ? "https" : "http";+ std::string seq_url = protocol + "://" + leader_host_port + "/sequence";++ std::string api_res;+ std::map<std::string, std::string> res_headers;+ long status = HttpClient::get_response(seq_url, api_res, res_headers);++ if(status != 500) {+ if(!StringUtils::is_uint64_t(api_res)) {+ LOG(ERROR) << "Invalid API response when fetching sequence number: " << api_res;+ this->caught_up = false;+ return ;+ }++ uint64_t leader_seq = std::atoll(api_res.c_str());+ if(leader_seq < seq_num) {+ LOG(ERROR) << "Leader sequence " << leader_seq << " is less than local sequence " << seq_num;+ this->caught_up = false;+ return ;+ }++ if(leader_seq == 0) {+ this->caught_up = true;+ return ;+ }++ float seq_progress = (float(seq_num) / leader_seq) * 100;+ LOG(INFO) << "Follower progress percentage: " << seq_progress;+ this->caught_up = (seq_progress >= catch_up_threshold_percentage);+ }+ });}}ReplicationState::ReplicationState(Store *store, ThreadPool* thread_pool, http_message_dispatcher *message_dispatcher,+ bool api_uses_ssl, size_t catch_up_threshold_percentage,bool create_init_db_snapshot, std::atomic<bool>& quit_service):node(nullptr), leader_term(-1), store(store), thread_pool(thread_pool),- message_dispatcher(message_dispatcher), init_readiness_count(0),- create_init_db_snapshot(create_init_db_snapshot), shut_down(quit_service) {+ message_dispatcher(message_dispatcher), catch_up_threshold_percentage(catch_up_threshold_percentage),+ api_uses_ssl(api_uses_ssl), create_init_db_snapshot(create_init_db_snapshot), shut_down(quit_service) {}@@ -438,10 +485,6 @@ void ReplicationState::reset_db() {store->close();}-size_t ReplicationState::get_init_readiness_count() const {- return init_readiness_count.load();-}-bool ReplicationState::is_alive() const {if(node == nullptr || !is_ready()) {return false;diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 7228ae53..1a0b6bca 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -88,6 +88,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<float>("max-memory-ratio", '\0', "Maximum fraction of system memory to be used.", false, 1.0f);options.add<int>("snapshot-interval-seconds", '\0', "Frequency of replication log snapshots.", false, 3600);+ options.add<int>("catch-up-threshold-percentage", '\0', "The threshold at which a follower is deemed to have caught up with leader.", false, 95);options.add<std::string>("log-dir", '\0', "Path to the log directory.", false, "");@@ -236,8 +237,8 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st// Wait until 'CTRL-C' is pressed. then Stop() and Join() the servicesize_t raft_counter = 0;while (!brpc::IsAskedToQuit() && !quit_raft_service.load()) {- // pre-increment to ensure that we don't refresh right away on a fresh boot- if(++raft_counter % 10 == 0) {+ // post-increment to ensure that we refresh right away on a fresh boot+ if(raft_counter++ % 10 == 0) {// reset peer configuration periodically to identify change in cluster membershipconst Option<std::string> & refreshed_nodes_op = fetch_nodes_config(path_to_nodes);if(!refreshed_nodes_op.ok()) {@@ -360,6 +361,7 @@ int run_server(const Config & config, const std::string & version, void (*masterThreadPool thread_pool(32);ReplicationState replication_state(&store, &thread_pool, server->get_message_dispatcher(),+ server->is_ssl_enabled(), config.get_catch_up_threshold_percentage(),create_init_db_snapshot, quit_raft_service);std::thread raft_thread([&replication_state, &config, &state_dir]() {