Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Implement rate limiting to protect from write lag.
Kishore Nallan0ae663ab026b
diff --git a/include/config.h b/include/config.hindex db71aa98..4d475e0c 100644--- a/include/config.h+++ b/include/config.h@@ -32,8 +32,8 @@ private:float max_memory_ratio;int snapshot_interval_seconds;- size_t catch_up_min_sequence_diff;- size_t catch_up_threshold_percentage;+ size_t read_max_lag;+ size_t write_max_lag;std::string config_file;int config_file_validity;@@ -54,8 +54,8 @@ protected:this->enable_cors = false;this->max_memory_ratio = 1.0f;this->snapshot_interval_seconds = 3600;- this->catch_up_min_sequence_diff = 3000;- this->catch_up_threshold_percentage = 95;+ this->read_max_lag = 1000;+ this->write_max_lag = 100;this->log_slow_requests_time_ms = -1;this->num_collections_parallel_load = 0; // will be set dynamically if not overriddenthis->num_documents_parallel_load = 1000;@@ -189,12 +189,12 @@ public:return this->snapshot_interval_seconds;}- int get_catch_up_min_sequence_diff() const {- return this->catch_up_min_sequence_diff;+ int get_read_max_lag() const {+ return this->read_max_lag;}- int get_catch_up_threshold_percentage() const {- return this->catch_up_threshold_percentage;+ int get_write_max_lag() const {+ return this->write_max_lag;}int get_log_slow_requests_time_ms() const {@@ -273,12 +273,12 @@ public:this->snapshot_interval_seconds = std::stoi(get_env("TYPESENSE_SNAPSHOT_INTERVAL_SECONDS"));}- if(!get_env("TYPESENSE_CATCH_UP_MIN_SEQUENCE_DIFF").empty()) {- this->catch_up_min_sequence_diff = std::stoi(get_env("TYPESENSE_CATCH_UP_MIN_SEQUENCE_DIFF"));+ if(!get_env("TYPESENSE_READ_MAX_LAG").empty()) {+ this->read_max_lag = std::stoi(get_env("TYPESENSE_READ_MAX_LAG"));}- if(!get_env("TYPESENSE_CATCH_UP_THRESHOLD_PERCENTAGE").empty()) {- this->catch_up_threshold_percentage = std::stoi(get_env("TYPESENSE_CATCH_UP_THRESHOLD_PERCENTAGE"));+ if(!get_env("TYPESENSE_WRITE_MAX_LAG").empty()) {+ this->write_max_lag = std::stoi(get_env("TYPESENSE_WRITE_MAX_LAG"));}if(!get_env("TYPESENSE_LOG_SLOW_REQUESTS_TIME_MS").empty()) {@@ -386,12 +386,12 @@ public:this->snapshot_interval_seconds = (int) reader.GetInteger("server", "snapshot-interval-seconds", 3600);}- if(reader.Exists("server", "catch-up-min-sequence-diff")) {- this->catch_up_min_sequence_diff = (int) reader.GetInteger("server", "catch-up-min-sequence-diff", 3000);+ if(reader.Exists("server", "read-max-lag")) {+ this->read_max_lag = (int) reader.GetInteger("server", "read-max-lag", 1000);}- if(reader.Exists("server", "catch-up-threshold-percentage")) {- this->catch_up_threshold_percentage = (int) reader.GetInteger("server", "catch-up-threshold-percentage", 95);+ if(reader.Exists("server", "write-max-lag")) {+ this->write_max_lag = (int) reader.GetInteger("server", "write-max-lag", 100);}if(reader.Exists("server", "log-slow-requests-time-ms")) {@@ -481,12 +481,12 @@ public:this->snapshot_interval_seconds = options.get<int>("snapshot-interval-seconds");}- if(options.exist("catch-up-min-sequence-diff")) {- this->catch_up_min_sequence_diff = options.get<int>("catch-up-min-sequence-diff");+ if(options.exist("read-max-lag")) {+ this->read_max_lag = options.get<int>("read-max-lag");}- if(options.exist("catch-up-threshold-percentage")) {- this->catch_up_threshold_percentage = options.get<int>("catch-up-threshold-percentage");+ if(options.exist("write-max-lag")) {+ this->write_max_lag = options.get<int>("write-max-lag");}if(options.exist("log-slow-requests-time-ms")) {diff --git a/include/http_data.h b/include/http_data.hindex 4de27590..7eda382e 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -109,6 +109,7 @@ struct http_res {case 405: return "Not Allowed";case 409: return "Conflict";case 422: return "Unprocessable Entity";+ case 429: return "Too Many Requests";case 500: return "Internal Server Error";default: return "";}diff --git a/include/raft_server.h b/include/raft_server.hindex 9bd3874a..aa3ef0d5 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -86,7 +86,7 @@ class ReplicationState : public braft::StateMachine {private:static constexpr const char* db_snapshot_name = "db_snapshot";- mutable std::shared_mutex mutex;+ mutable std::shared_mutex node_mutex;braft::Node* volatile node;butil::atomic<int64_t> leader_term;@@ -97,13 +97,14 @@ private:ThreadPool* thread_pool;http_message_dispatcher* message_dispatcher;- const size_t catchup_min_sequence_diff;- const size_t catch_up_threshold_percentage;+ const size_t read_max_lag;+ const size_t write_max_lag;const size_t num_collections_parallel_load;const size_t num_documents_parallel_load;- std::atomic<bool> caught_up;+ std::atomic<bool> read_caught_up;+ std::atomic<bool> write_caught_up;const bool api_uses_ssl;@@ -127,7 +128,7 @@ public:static constexpr const char* snapshot_dir_name = "snapshot";ReplicationState(HttpServer* server, Store* store, ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,- bool api_uses_ssl, size_t catchup_min_sequence_diff, size_t catch_up_threshold_percentage,+ bool api_uses_ssl, size_t read_max_lag, size_t write_max_lag,size_t num_collections_parallel_load, size_t num_documents_parallel_load);// Starts this node@@ -152,8 +153,12 @@ public:return leader_term.load(butil::memory_order_acquire) > 0;}- bool is_ready() const {- return caught_up;+ bool is_read_caught_up() const {+ return read_caught_up;+ }++ bool is_write_caught_up() const {+ return write_caught_up;}bool is_alive() const;diff --git a/src/core_api.cpp b/src/core_api.cppindex c032b61c..cfc07a52 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -342,8 +342,9 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share}bool post_import_documents(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {- //LOG(INFO) << "post_import_documents";+ //LOG(INFO) << "Import, req->body_index=" << req->body_index << ", body size: " << req->body.size();//LOG(INFO) << "req->first_chunk=" << req->first_chunk_aggregate << ", last_chunk=" << req->last_chunk_aggregate;+const char *BATCH_SIZE = "batch_size";const char *ACTION = "action";const char *DIRTY_VALUES = "dirty_values";diff --git a/src/http_server.cpp b/src/http_server.cppindex 08e642a4..7b7395b9 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -303,10 +303,26 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {// Except for health check, wait for replicating state to be ready before allowing requests// Follower or leader must have started AND data must also have been loaded- if(path_without_query != "/health" && path_without_query != "/debug" && path_without_query != "/sequence" &&- !h2o_handler->http_server->get_replication_state()->is_ready()) {- std::string message = "{ \"message\": \"Not Ready\"}";- return send_response(req, 503, message);+ bool needs_readiness_check = !(+ path_without_query == "/health" || path_without_query == "/debug" || path_without_query == "/sequence"+ );++ if(needs_readiness_check) {+ bool is_read_op = !(+ http_method == "POST" || http_method == "PUT" || http_method == "DELETE" || http_method == "PATCH"+ );++ bool write_op = !is_read_op;++ if(is_read_op && !h2o_handler->http_server->get_replication_state()->is_read_caught_up()) {+ std::string message = "{ \"message\": \"Not Ready\"}";+ return send_response(req, 503, message);+ }++ if(write_op && !h2o_handler->http_server->get_replication_state()->is_write_caught_up()) {+ std::string message = "{ \"message\": \"Too Many Writes\"}";+ return send_response(req, 429, message);+ }}std::vector<std::string> path_parts;diff --git a/src/raft_server.cpp b/src/raft_server.cppindex d6bff1b8..a44ac86d 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -41,7 +41,8 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const intreturn -1;}- this->caught_up = false;+ this->read_caught_up = false;+ this->write_caught_up = false;// do snapshot only when the gap between applied index and last snapshot index is >= this numberbraft::FLAGS_raft_do_snapshot_min_index_gap = 1;@@ -54,6 +55,7 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int// flag controls snapshot download size of each RPCbraft::FLAGS_raft_max_byte_count_per_rpc = 4 * 1024 * 1024; // 4 MB+ node_options.catchup_margin = read_max_lag;node_options.election_timeout_ms = election_timeout_ms;node_options.fsm = this;node_options.node_owns_fsm = false;@@ -123,7 +125,7 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const stdreturn ;}- std::shared_lock lock(mutex);+ std::shared_lock lock(node_mutex);if(!node) {return ;@@ -154,10 +156,11 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const stdtask.expected_term = leader_term.load(butil::memory_order_relaxed);//LOG(INFO) << ":::" << "body size before apply: " << request->body.size();- pending_writes++;- // Now the task is applied to the group, waiting for the result.- return node->apply(task);+ // Now the task is applied to the group+ node->apply(task);++ pending_writes++;}void ReplicationState::write_to_leader(const std::shared_ptr<http_req>& request, const std::shared_ptr<http_res>& response) {@@ -285,8 +288,6 @@ void ReplicationState::on_apply(braft::Iterator& iter) {}// Now that the log has been parsed, perform the actual operation- // Call http server thread for write and response back to client (if `response` is NOT null)- // We use a future to block current thread until the async flow finishesbool async_res = false;@@ -425,14 +426,15 @@ int ReplicationState::init_db() {}int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {- std::shared_lock lock(mutex);+ std::shared_lock lock(node_mutex);CHECK(!node || !node->is_leader()) << "Leader is not supposed to load snapshot";lock.unlock();LOG(INFO) << "on_snapshot_load";- // ensures that reads are rejected, as `store->reload()` unique locks the DB handle- caught_up = false;+ // ensures that reads and writes are rejected, as `store->reload()` unique locks the DB handle+ read_caught_up = false;+ write_caught_up = false;// Load snapshot from leader, replacing the running StateMachinestd::string snapshot_path = reader->get_path();@@ -443,11 +445,14 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {return reload_store;}- return init_db();+ bool init_db_status = init_db();++ read_caught_up = write_caught_up = (init_db_status == 0);+ return init_db_status;}void ReplicationState::refresh_nodes(const std::string & nodes) {- std::shared_lock lock(mutex);+ std::shared_lock lock(node_mutex);if(!node) {LOG(WARNING) << "Node state is not initialized: unable to refresh nodes.";@@ -497,85 +502,49 @@ void ReplicationState::refresh_nodes(const std::string & nodes) {}void ReplicationState::refresh_catchup_status(bool log_msg) {- std::shared_lock lock(mutex);+ std::shared_lock lock(node_mutex);if (!node) {LOG_IF(WARNING, log_msg) << "Node state is not initialized: unable to refresh nodes.";return;}- if(node->is_leader()) {- this->caught_up = true;- } else {-+ if(!node->is_leader() && node->leader_id().is_empty()) {// follower does not have a leader!- if(node->leader_id().is_empty()) {- this->caught_up = false;- return ;- }-- lock.unlock();-- // update catch up status- thread_pool->enqueue([this, log_msg]() {- auto seq_num = this->store->get_latest_seq_number();- std::shared_lock lock(this->mutex);- const std::string & leader_addr = node->leader_id().to_string();- lock.unlock();-- std::vector<std::string> addr_parts;- StringUtils::split(leader_addr, addr_parts, ":");- std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];- std::string protocol = this->api_uses_ssl ? "https" : "http";- std::string seq_url = protocol + "://" + leader_host_port + "/sequence";-- std::string api_res;- std::map<std::string, std::string> res_headers;- long status = HttpClient::get_response(seq_url, api_res, res_headers);-- if(status != 500) {- if(!StringUtils::is_uint64_t(api_res)) {- LOG_IF(ERROR, log_msg) << "Invalid API response when fetching sequence number: " << api_res;- this->caught_up = false;- return ;- }-- uint64_t leader_seq = std::atoll(api_res.c_str());+ this->read_caught_up = false;+ this->write_caught_up = false;+ return ;+ }- // Since leader waits for writes on followers to finish, follower's storage offset could be- // momentarily ahead of the leader's. So we will use std::abs() for checking the difference.- const uint64_t seq_diff = std::abs(int64_t(leader_seq) - int64_t(seq_num));+ braft::NodeStatus n_status;+ node->get_status(&n_status);+ lock.unlock();- if(seq_diff < catchup_min_sequence_diff) {- this->caught_up = true;- return ;- }+ if (n_status.applying_index == 0) {+ this->read_caught_up = true;+ this->write_caught_up = true;+ return ;+ }- // However, if the difference is large, then something could be wrong- if(leader_seq < seq_num) {- LOG_IF(ERROR, log_msg) << "Leader sequence " << leader_seq << " is less than local sequence "- << seq_num << ", catchup_min_sequence_diff: " << catchup_min_sequence_diff;- this->caught_up = false;- return ;- }+ size_t apply_lag = size_t(n_status.last_index - n_status.known_applied_index);- float seq_progress = (float(seq_num) / leader_seq) * 100;- LOG_IF(INFO, log_msg) << "Follower progress percentage: " << seq_progress;+ if (apply_lag > read_max_lag) {+ LOG(ERROR) << apply_lag << " lagging entries > read max lag of " + std::to_string(read_max_lag);+ this->read_caught_up = false;+ }- this->caught_up = (seq_progress >= catch_up_threshold_percentage);- }- });+ if (apply_lag > write_max_lag) {+ LOG(ERROR) << apply_lag << " lagging entries > write max lag of " + std::to_string(write_max_lag);+ this->write_caught_up = false;}}ReplicationState::ReplicationState(HttpServer* server, Store *store, ThreadPool* thread_pool,http_message_dispatcher *message_dispatcher,- bool api_uses_ssl, size_t catchup_min_sequence_diff,- size_t catch_up_threshold_percentage,+ bool api_uses_ssl, size_t read_max_lag, size_t write_max_lag,size_t num_collections_parallel_load, size_t num_documents_parallel_load):node(nullptr), leader_term(-1), server(server), store(store), thread_pool(thread_pool),- message_dispatcher(message_dispatcher),- catchup_min_sequence_diff(catchup_min_sequence_diff), catch_up_threshold_percentage(catch_up_threshold_percentage),+ message_dispatcher(message_dispatcher), read_max_lag(read_max_lag), write_max_lag(write_max_lag),num_collections_parallel_load(num_collections_parallel_load),num_documents_parallel_load(num_documents_parallel_load),api_uses_ssl(api_uses_ssl),@@ -584,18 +553,23 @@ ReplicationState::ReplicationState(HttpServer* server, Store *store, ThreadPool*}bool ReplicationState::is_alive() const {- std::shared_lock lock(mutex);+ std::shared_lock lock(node_mutex);++ if(node == nullptr ) {+ return false;+ }- if(node == nullptr || !is_ready()) {+ bool leader_or_follower = (node->is_leader() || !node->leader_id().is_empty());+ if(!leader_or_follower) {return false;}- // node should either be a leader or have a leader- return (node->is_leader() || !node->leader_id().is_empty());+ // for general health check we will only care about the `read_caught_up` threshold+ return read_caught_up;}uint64_t ReplicationState::node_state() const {- std::shared_lock lock(mutex);+ std::shared_lock lock(node_mutex);if(node == nullptr) {return 0;@@ -614,7 +588,7 @@ void ReplicationState::do_snapshot(const std::string& snapshot_path, const std::thread_pool->enqueue([&snapshot_path, req, res, this]() {OnDemandSnapshotClosure* snapshot_closure = new OnDemandSnapshotClosure(this, req, res);ext_snapshot_path = snapshot_path;- std::shared_lock lock(this->mutex);+ std::shared_lock lock(this->node_mutex);node->snapshot(snapshot_closure);});}@@ -628,7 +602,7 @@ const std::string &ReplicationState::get_ext_snapshot_path() const {}void ReplicationState::do_dummy_write() {- std::shared_lock lock(mutex);+ std::shared_lock lock(node_mutex);if(node->leader_id().is_empty()) {LOG(ERROR) << "Could not do a dummy write, as node does not have a leader";@@ -649,7 +623,7 @@ void ReplicationState::do_dummy_write() {}bool ReplicationState::trigger_vote() {- std::shared_lock lock(mutex);+ std::shared_lock lock(node_mutex);if(node) {auto status = node->vote(election_timeout_interval_ms);@@ -680,7 +654,7 @@ void ReplicationState::shutdown() {}LOG(INFO) << "Replication state shutdown, store sequence: " << store->get_latest_seq_number();- std::unique_lock lock(mutex);+ std::unique_lock lock(node_mutex);if (node) {LOG(INFO) << "node->shutdown";diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex d94afa86..6adc338f 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -78,8 +78,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<float>("max-memory-ratio", '\0', "Maximum fraction of system memory to be used.", false, 1.0f);options.add<int>("snapshot-interval-seconds", '\0', "Frequency of replication log snapshots.", false, 3600);- options.add<int>("catch-up-min-sequence-diff", '\0', "The absolute storage sequence difference within which a follower is deemed to have caught up with leader.", false, 3000);- options.add<int>("catch-up-threshold-percentage", '\0', "The threshold at which a follower is deemed to have caught up with leader.", false, 95);+ options.add<int>("read-max-lag", '\0', "Reads are rejected if the updates lag behind this threshold.", false, 1000);+ options.add<int>("write-max-lag", '\0', "Writes are rejected if the updates lag behind this threshold.", false, 100);options.add<int>("log-slow-requests-time-ms", '\0', "When > 0, requests that take longer than this duration are logged.", false, -1);options.add<uint32_t>("num-collections-parallel-load", '\0', "Number of collections that are loaded in parallel during start up.", false, 4);@@ -385,8 +385,9 @@ int run_server(const Config & config, const std::string & version, void (*master// first we start the peering serviceReplicationState replication_state(server, &store, &app_thread_pool, server->get_message_dispatcher(),- ssl_enabled, config.get_catch_up_min_sequence_diff(),- config.get_catch_up_threshold_percentage(),+ ssl_enabled,+ config.get_read_max_lag(),+ config.get_write_max_lag(),num_collections_parallel_load,config.get_num_documents_parallel_load());