Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Protect API until follower has caught up with leader.

kishorenca2b421209cc3
diff --git a/include/config.h b/include/config.h
index d601f350..61cd508d 100644
--- a/include/config.h
+++ b/include/config.h
@@ -32,6 +32,8 @@ private:
float max_memory_ratio;
int snapshot_interval_seconds;
+ size_t catch_up_threshold_percentage;
+
std::string config_file;
int config_file_validity;
@@ -44,6 +46,7 @@ public:
this->enable_cors = false;
this->max_memory_ratio = 1.0f;
this->snapshot_interval_seconds = 3600;
+ this->catch_up_threshold_percentage = 95;
}
// setters
@@ -156,6 +159,10 @@ public:
return this->snapshot_interval_seconds;
}
+ int get_catch_up_threshold_percentage() const {
+ return this->catch_up_threshold_percentage;
+ }
+
// loaders
std::string get_env(const char *name) {
@@ -215,6 +222,10 @@ public:
if(!get_env("TYPESENSE_SNAPSHOT_INTERVAL_SECONDS").empty()) {
this->snapshot_interval_seconds = std::stoi(get_env("TYPESENSE_SNAPSHOT_INTERVAL_SECONDS"));
}
+
+ if(!get_env("TYPESENSE_CATCH_UP_THRESHOLD_PERCENTAGE").empty()) {
+ this->catch_up_threshold_percentage = std::stoi(get_env("TYPESENSE_CATCH_UP_THRESHOLD_PERCENTAGE"));
+ }
}
void load_config_file(cmdline::parser & options) {
@@ -304,6 +315,10 @@ public:
if(reader.Exists("server", "snapshot-interval-seconds")) {
this->snapshot_interval_seconds = (int) reader.GetInteger("server", "snapshot-interval-seconds", 3600);
}
+
+ if(reader.Exists("server", "catch-up-threshold-percentage")) {
+ this->catch_up_threshold_percentage = (int) reader.GetInteger("server", "catch-up-threshold-percentage", 95);
+ }
}
void load_config_cmd_args(cmdline::parser & options) {
@@ -375,6 +390,10 @@ public:
if(options.exist("snapshot-interval-seconds")) {
this->snapshot_interval_seconds = options.get<int>("snapshot-interval-seconds");
}
+
+ if(options.exist("catch-up-threshold-percentage")) {
+ this->catch_up_threshold_percentage = options.get<int>("catch-up-threshold-percentage");
+ }
}
// validation
diff --git a/include/core_api.h b/include/core_api.h
index 6f29771b..f4536644 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -67,6 +67,8 @@ bool get_health(http_req& req, http_res& res);
bool get_metrics_json(http_req& req, http_res& res);
+bool get_log_sequence(http_req& req, http_res& res);
+
// Misc helpers
bool raft_write_send_response(void *data);
diff --git a/include/http_server.h b/include/http_server.h
index 0a30e4b5..bf209b32 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -58,6 +58,8 @@ private:
bool exit_loop = false;
+ bool ssl_enabled = false;
+
private:
std::string version;
@@ -171,4 +173,8 @@ public:
static void on_deferred_process_request(h2o_timer_t *entry);
void defer_processing(http_req& req, http_res& res, size_t timeout_ms);
+
+ bool is_ssl_enabled() {
+ return ssl_enabled;
+ }
};
diff --git a/include/raft_server.h b/include/raft_server.h
index f5bc9354..f793e938 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -86,7 +86,10 @@ private:
ThreadPool* thread_pool;
http_message_dispatcher* message_dispatcher;
- std::atomic<size_t> init_readiness_count;
+ const size_t catch_up_threshold_percentage;
+ std::atomic<bool> caught_up;
+
+ const bool api_uses_ssl;
bool create_init_db_snapshot;
@@ -99,6 +102,7 @@ public:
static constexpr const char* snapshot_dir_name = "snapshot";
ReplicationState(Store* store, ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,
+ bool api_uses_ssl, size_t catch_up_threshold_percentage,
bool create_init_db_snapshot, std::atomic<bool>& quit_service);
// Starts this node
@@ -120,7 +124,7 @@ public:
}
bool is_ready() const {
- return init_readiness_count >= 2;
+ return caught_up;
}
bool is_alive() const;
@@ -149,8 +153,6 @@ public:
void reset_db();
- size_t get_init_readiness_count() const;
-
static std::string to_nodes_config(const butil::EndPoint &peering_endpoint, const int api_port,
const std::string &nodes_config);
@@ -185,8 +187,6 @@ private:
write(request, response);
}
- init_readiness_count++;
-
LOG(INFO) << "Node becomes leader, term: " << term;
}
@@ -210,7 +210,6 @@ private:
void on_start_following(const ::braft::LeaderChangeContext& ctx) {
LOG(INFO) << "Node starts following " << ctx;
- init_readiness_count++;
}
void on_stop_following(const ::braft::LeaderChangeContext& ctx) {
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 905d48db..34347a90 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -213,6 +213,15 @@ bool get_metrics_json(http_req &req, http_res &res) {
return true;
}
+
+bool get_log_sequence(http_req &req, http_res &res) {
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ const uint64_t seq_num = collectionManager.get_store()->get_latest_seq_number();
+ res.content_type_header = "text/plain; charset=utf8";
+ res.set_body(200, std::to_string(seq_num));
+ return true;
+}
+
bool get_search(http_req & req, http_res & res) {
auto begin = std::chrono::high_resolution_clock::now();
diff --git a/src/http_server.cpp b/src/http_server.cpp
index f0fa153f..590a7511 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -112,6 +112,7 @@ int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {
}
h2o_ssl_register_alpn_protocols(accept_ctx->ssl_ctx, h2o_http2_alpn_protocols);
+ ssl_enabled = true;
return 0;
}
@@ -326,7 +327,8 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {
// Except for health check, wait for replicating state to be ready before allowing requests
// Follower or leader must have started AND data must also have been loaded
- if(path_without_query != "/health" && !h2o_handler->http_server->get_replication_state()->is_ready()) {
+ if(path_without_query != "/health" && path_without_query != "/sequence" &&
+ !h2o_handler->http_server->get_replication_state()->is_ready()) {
std::string message = "{ \"message\": \"Not Ready\"}";
return send_response(req, 503, message);
}
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 9e91b7d0..4164110e 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -48,21 +48,7 @@ void master_server_routes() {
server->get("/metrics.json", get_metrics_json);
server->get("/debug", get_debug);
server->get("/health", get_health);
-}
-
-void replica_server_routes() {
- // collection management
- server->get("/collections", get_collections);
- server->get("/collections/:collection", get_collection_summary);
-
- // document management - `/documents/:id` end-points must be placed last in the list
- server->get("/collections/:collection/documents/search", get_search);
- server->get("/collections/:collection/documents/export", get_export_documents, true);
- server->get("/collections/:collection/documents/:id", get_fetch_document);
-
- // meta
- server->get("/debug", get_debug);
- server->get("/health", get_health);
+ server->get("/sequence", get_log_sequence);
}
int main(int argc, char **argv) {
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index b8118306..f5308752 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -33,6 +33,8 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int
return -1;
}
+ this->caught_up = false;
+
// do snapshot only when the gap between applied index and last snapshot index is >= this number
braft::FLAGS_raft_do_snapshot_min_index_gap = 1;
@@ -361,8 +363,6 @@ int ReplicationState::init_db() {
return 1;
}
- init_readiness_count++;
-
return 0;
}
@@ -407,30 +407,77 @@ void ReplicationState::refresh_nodes(const std::string & nodes) {
if(node->is_leader()) {
RefreshNodesClosure* refresh_nodes_done = new RefreshNodesClosure;
node->change_peers(new_conf, refresh_nodes_done);
- } else if(node->leader_id().is_empty()) {
- // When node is not a leader, does not have a leader and is also a single-node cluster,
- // we forcefully reset its peers.
- // NOTE: `reset_peers()` is not a safe call to make as we give up on consistency and consensus guarantees.
- // We are doing this solely to handle single node cluster whose IP changes.
- // Examples: Docker container IP change, local DHCP leased IP change etc.
-
- std::vector<braft::PeerId> latest_nodes;
- new_conf.list_peers(&latest_nodes);
-
- if(latest_nodes.size() == 1) {
- LOG(WARNING) << "Single-node with no leader. Resetting peers.";
- node->reset_peers(new_conf);
- } else {
- LOG(WARNING) << "Multi-node with no leader: refusing to reset peers.";
+ this->caught_up = true;
+ } else {
+ if(node->leader_id().is_empty()) {
+ // When node is not a leader, does not have a leader and is also a single-node cluster,
+ // we forcefully reset its peers.
+ // NOTE: `reset_peers()` is not a safe call to make as we give up on consistency and consensus guarantees.
+ // We are doing this solely to handle single node cluster whose IP changes.
+ // Examples: Docker container IP change, local DHCP leased IP change etc.
+
+ std::vector<braft::PeerId> latest_nodes;
+ new_conf.list_peers(&latest_nodes);
+
+ if(latest_nodes.size() == 1) {
+ LOG(WARNING) << "Single-node with no leader. Resetting peers.";
+ node->reset_peers(new_conf);
+ } else {
+ LOG(WARNING) << "Multi-node with no leader: refusing to reset peers.";
+ }
+
+ this->caught_up = false;
+ return ;
}
+
+ // update catch up status
+ thread_pool->enqueue([this]() {
+ auto seq_num = this->store->get_latest_seq_number();
+ const std::string & leader_addr = node->leader_id().to_string();
+
+ std::vector<std::string> addr_parts;
+ StringUtils::split(leader_addr, addr_parts, ":");
+ std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];
+ std::string protocol = this->api_uses_ssl ? "https" : "http";
+ std::string seq_url = protocol + "://" + leader_host_port + "/sequence";
+
+ std::string api_res;
+ std::map<std::string, std::string> res_headers;
+ long status = HttpClient::get_response(seq_url, api_res, res_headers);
+
+ if(status != 500) {
+ if(!StringUtils::is_uint64_t(api_res)) {
+ LOG(ERROR) << "Invalid API response when fetching sequence number: " << api_res;
+ this->caught_up = false;
+ return ;
+ }
+
+ uint64_t leader_seq = std::atoll(api_res.c_str());
+ if(leader_seq < seq_num) {
+ LOG(ERROR) << "Leader sequence " << leader_seq << " is less than local sequence " << seq_num;
+ this->caught_up = false;
+ return ;
+ }
+
+ if(leader_seq == 0) {
+ this->caught_up = true;
+ return ;
+ }
+
+ float seq_progress = (float(seq_num) / leader_seq) * 100;
+ LOG(INFO) << "Follower progress percentage: " << seq_progress;
+ this->caught_up = (seq_progress >= catch_up_threshold_percentage);
+ }
+ });
}
}
ReplicationState::ReplicationState(Store *store, ThreadPool* thread_pool, http_message_dispatcher *message_dispatcher,
+ bool api_uses_ssl, size_t catch_up_threshold_percentage,
bool create_init_db_snapshot, std::atomic<bool>& quit_service):
node(nullptr), leader_term(-1), store(store), thread_pool(thread_pool),
- message_dispatcher(message_dispatcher), init_readiness_count(0),
- create_init_db_snapshot(create_init_db_snapshot), shut_down(quit_service) {
+ message_dispatcher(message_dispatcher), catch_up_threshold_percentage(catch_up_threshold_percentage),
+ api_uses_ssl(api_uses_ssl), create_init_db_snapshot(create_init_db_snapshot), shut_down(quit_service) {
}
@@ -438,10 +485,6 @@ void ReplicationState::reset_db() {
store->close();
}
-size_t ReplicationState::get_init_readiness_count() const {
- return init_readiness_count.load();
-}
-
bool ReplicationState::is_alive() const {
if(node == nullptr || !is_ready()) {
return false;
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 7228ae53..1a0b6bca 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -88,6 +88,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<float>("max-memory-ratio", '\0', "Maximum fraction of system memory to be used.", false, 1.0f);
options.add<int>("snapshot-interval-seconds", '\0', "Frequency of replication log snapshots.", false, 3600);
+ options.add<int>("catch-up-threshold-percentage", '\0', "The threshold at which a follower is deemed to have caught up with leader.", false, 95);
options.add<std::string>("log-dir", '\0', "Path to the log directory.", false, "");
@@ -236,8 +237,8 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
// Wait until 'CTRL-C' is pressed. then Stop() and Join() the service
size_t raft_counter = 0;
while (!brpc::IsAskedToQuit() && !quit_raft_service.load()) {
- // pre-increment to ensure that we don't refresh right away on a fresh boot
- if(++raft_counter % 10 == 0) {
+ // post-increment to ensure that we refresh right away on a fresh boot
+ if(raft_counter++ % 10 == 0) {
// reset peer configuration periodically to identify change in cluster membership
const Option<std::string> & refreshed_nodes_op = fetch_nodes_config(path_to_nodes);
if(!refreshed_nodes_op.ok()) {
@@ -360,6 +361,7 @@ int run_server(const Config & config, const std::string & version, void (*master
ThreadPool thread_pool(32);
ReplicationState replication_state(&store, &thread_pool, server->get_message_dispatcher(),
+ server->is_ssl_enabled(), config.get_catch_up_threshold_percentage(),
create_init_db_snapshot, quit_raft_service);
std::thread raft_thread([&replication_state, &config, &state_dir]() {