Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Implement rate limiting to protect from write lag.

Kishore Nallan0ae663ab026b
diff --git a/include/config.h b/include/config.h
index db71aa98..4d475e0c 100644
--- a/include/config.h
+++ b/include/config.h
@@ -32,8 +32,8 @@ private:
float max_memory_ratio;
int snapshot_interval_seconds;
- size_t catch_up_min_sequence_diff;
- size_t catch_up_threshold_percentage;
+ size_t read_max_lag;
+ size_t write_max_lag;
std::string config_file;
int config_file_validity;
@@ -54,8 +54,8 @@ protected:
this->enable_cors = false;
this->max_memory_ratio = 1.0f;
this->snapshot_interval_seconds = 3600;
- this->catch_up_min_sequence_diff = 3000;
- this->catch_up_threshold_percentage = 95;
+ this->read_max_lag = 1000;
+ this->write_max_lag = 100;
this->log_slow_requests_time_ms = -1;
this->num_collections_parallel_load = 0; // will be set dynamically if not overridden
this->num_documents_parallel_load = 1000;
@@ -189,12 +189,12 @@ public:
return this->snapshot_interval_seconds;
}
- int get_catch_up_min_sequence_diff() const {
- return this->catch_up_min_sequence_diff;
+ int get_read_max_lag() const {
+ return this->read_max_lag;
}
- int get_catch_up_threshold_percentage() const {
- return this->catch_up_threshold_percentage;
+ int get_write_max_lag() const {
+ return this->write_max_lag;
}
int get_log_slow_requests_time_ms() const {
@@ -273,12 +273,12 @@ public:
this->snapshot_interval_seconds = std::stoi(get_env("TYPESENSE_SNAPSHOT_INTERVAL_SECONDS"));
}
- if(!get_env("TYPESENSE_CATCH_UP_MIN_SEQUENCE_DIFF").empty()) {
- this->catch_up_min_sequence_diff = std::stoi(get_env("TYPESENSE_CATCH_UP_MIN_SEQUENCE_DIFF"));
+ if(!get_env("TYPESENSE_READ_MAX_LAG").empty()) {
+ this->read_max_lag = std::stoi(get_env("TYPESENSE_READ_MAX_LAG"));
}
- if(!get_env("TYPESENSE_CATCH_UP_THRESHOLD_PERCENTAGE").empty()) {
- this->catch_up_threshold_percentage = std::stoi(get_env("TYPESENSE_CATCH_UP_THRESHOLD_PERCENTAGE"));
+ if(!get_env("TYPESENSE_WRITE_MAX_LAG").empty()) {
+ this->write_max_lag = std::stoi(get_env("TYPESENSE_WRITE_MAX_LAG"));
}
if(!get_env("TYPESENSE_LOG_SLOW_REQUESTS_TIME_MS").empty()) {
@@ -386,12 +386,12 @@ public:
this->snapshot_interval_seconds = (int) reader.GetInteger("server", "snapshot-interval-seconds", 3600);
}
- if(reader.Exists("server", "catch-up-min-sequence-diff")) {
- this->catch_up_min_sequence_diff = (int) reader.GetInteger("server", "catch-up-min-sequence-diff", 3000);
+ if(reader.Exists("server", "read-max-lag")) {
+ this->read_max_lag = (int) reader.GetInteger("server", "read-max-lag", 1000);
}
- if(reader.Exists("server", "catch-up-threshold-percentage")) {
- this->catch_up_threshold_percentage = (int) reader.GetInteger("server", "catch-up-threshold-percentage", 95);
+ if(reader.Exists("server", "write-max-lag")) {
+ this->write_max_lag = (int) reader.GetInteger("server", "write-max-lag", 100);
}
if(reader.Exists("server", "log-slow-requests-time-ms")) {
@@ -481,12 +481,12 @@ public:
this->snapshot_interval_seconds = options.get<int>("snapshot-interval-seconds");
}
- if(options.exist("catch-up-min-sequence-diff")) {
- this->catch_up_min_sequence_diff = options.get<int>("catch-up-min-sequence-diff");
+ if(options.exist("read-max-lag")) {
+ this->read_max_lag = options.get<int>("read-max-lag");
}
- if(options.exist("catch-up-threshold-percentage")) {
- this->catch_up_threshold_percentage = options.get<int>("catch-up-threshold-percentage");
+ if(options.exist("write-max-lag")) {
+ this->write_max_lag = options.get<int>("write-max-lag");
}
if(options.exist("log-slow-requests-time-ms")) {
diff --git a/include/http_data.h b/include/http_data.h
index 4de27590..7eda382e 100644
--- a/include/http_data.h
+++ b/include/http_data.h
@@ -109,6 +109,7 @@ struct http_res {
case 405: return "Not Allowed";
case 409: return "Conflict";
case 422: return "Unprocessable Entity";
+ case 429: return "Too Many Requests";
case 500: return "Internal Server Error";
default: return "";
}
diff --git a/include/raft_server.h b/include/raft_server.h
index 9bd3874a..aa3ef0d5 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -86,7 +86,7 @@ class ReplicationState : public braft::StateMachine {
private:
static constexpr const char* db_snapshot_name = "db_snapshot";
- mutable std::shared_mutex mutex;
+ mutable std::shared_mutex node_mutex;
braft::Node* volatile node;
butil::atomic<int64_t> leader_term;
@@ -97,13 +97,14 @@ private:
ThreadPool* thread_pool;
http_message_dispatcher* message_dispatcher;
- const size_t catchup_min_sequence_diff;
- const size_t catch_up_threshold_percentage;
+ const size_t read_max_lag;
+ const size_t write_max_lag;
const size_t num_collections_parallel_load;
const size_t num_documents_parallel_load;
- std::atomic<bool> caught_up;
+ std::atomic<bool> read_caught_up;
+ std::atomic<bool> write_caught_up;
const bool api_uses_ssl;
@@ -127,7 +128,7 @@ public:
static constexpr const char* snapshot_dir_name = "snapshot";
ReplicationState(HttpServer* server, Store* store, ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,
- bool api_uses_ssl, size_t catchup_min_sequence_diff, size_t catch_up_threshold_percentage,
+ bool api_uses_ssl, size_t read_max_lag, size_t write_max_lag,
size_t num_collections_parallel_load, size_t num_documents_parallel_load);
// Starts this node
@@ -152,8 +153,12 @@ public:
return leader_term.load(butil::memory_order_acquire) > 0;
}
- bool is_ready() const {
- return caught_up;
+ bool is_read_caught_up() const {
+ return read_caught_up;
+ }
+
+ bool is_write_caught_up() const {
+ return write_caught_up;
}
bool is_alive() const;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index c032b61c..cfc07a52 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -342,8 +342,9 @@ bool get_export_documents(const std::shared_ptr<http_req>& req, const std::share
}
bool post_import_documents(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
- //LOG(INFO) << "post_import_documents";
+ //LOG(INFO) << "Import, req->body_index=" << req->body_index << ", body size: " << req->body.size();
//LOG(INFO) << "req->first_chunk=" << req->first_chunk_aggregate << ", last_chunk=" << req->last_chunk_aggregate;
+
const char *BATCH_SIZE = "batch_size";
const char *ACTION = "action";
const char *DIRTY_VALUES = "dirty_values";
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 08e642a4..7b7395b9 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -303,10 +303,26 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {
// Except for health check, wait for replicating state to be ready before allowing requests
// Follower or leader must have started AND data must also have been loaded
- if(path_without_query != "/health" && path_without_query != "/debug" && path_without_query != "/sequence" &&
- !h2o_handler->http_server->get_replication_state()->is_ready()) {
- std::string message = "{ \"message\": \"Not Ready\"}";
- return send_response(req, 503, message);
+ bool needs_readiness_check = !(
+ path_without_query == "/health" || path_without_query == "/debug" || path_without_query == "/sequence"
+ );
+
+ if(needs_readiness_check) {
+ bool is_read_op = !(
+ http_method == "POST" || http_method == "PUT" || http_method == "DELETE" || http_method == "PATCH"
+ );
+
+ bool write_op = !is_read_op;
+
+ if(is_read_op && !h2o_handler->http_server->get_replication_state()->is_read_caught_up()) {
+ std::string message = "{ \"message\": \"Not Ready\"}";
+ return send_response(req, 503, message);
+ }
+
+ if(write_op && !h2o_handler->http_server->get_replication_state()->is_write_caught_up()) {
+ std::string message = "{ \"message\": \"Too Many Writes\"}";
+ return send_response(req, 429, message);
+ }
}
std::vector<std::string> path_parts;
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index d6bff1b8..a44ac86d 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -41,7 +41,8 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int
return -1;
}
- this->caught_up = false;
+ this->read_caught_up = false;
+ this->write_caught_up = false;
// do snapshot only when the gap between applied index and last snapshot index is >= this number
braft::FLAGS_raft_do_snapshot_min_index_gap = 1;
@@ -54,6 +55,7 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int
// flag controls snapshot download size of each RPC
braft::FLAGS_raft_max_byte_count_per_rpc = 4 * 1024 * 1024; // 4 MB
+ node_options.catchup_margin = read_max_lag;
node_options.election_timeout_ms = election_timeout_ms;
node_options.fsm = this;
node_options.node_owns_fsm = false;
@@ -123,7 +125,7 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const std
return ;
}
- std::shared_lock lock(mutex);
+ std::shared_lock lock(node_mutex);
if(!node) {
return ;
@@ -154,10 +156,11 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const std
task.expected_term = leader_term.load(butil::memory_order_relaxed);
//LOG(INFO) << ":::" << "body size before apply: " << request->body.size();
- pending_writes++;
- // Now the task is applied to the group, waiting for the result.
- return node->apply(task);
+ // Now the task is applied to the group
+ node->apply(task);
+
+ pending_writes++;
}
void ReplicationState::write_to_leader(const std::shared_ptr<http_req>& request, const std::shared_ptr<http_res>& response) {
@@ -285,8 +288,6 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
}
// Now that the log has been parsed, perform the actual operation
- // Call http server thread for write and response back to client (if `response` is NOT null)
- // We use a future to block current thread until the async flow finishes
bool async_res = false;
@@ -425,14 +426,15 @@ int ReplicationState::init_db() {
}
int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
- std::shared_lock lock(mutex);
+ std::shared_lock lock(node_mutex);
CHECK(!node || !node->is_leader()) << "Leader is not supposed to load snapshot";
lock.unlock();
LOG(INFO) << "on_snapshot_load";
- // ensures that reads are rejected, as `store->reload()` unique locks the DB handle
- caught_up = false;
+ // ensures that reads and writes are rejected, as `store->reload()` unique locks the DB handle
+ read_caught_up = false;
+ write_caught_up = false;
// Load snapshot from leader, replacing the running StateMachine
std::string snapshot_path = reader->get_path();
@@ -443,11 +445,14 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
return reload_store;
}
- return init_db();
+ bool init_db_status = init_db();
+
+ read_caught_up = write_caught_up = (init_db_status == 0);
+ return init_db_status;
}
void ReplicationState::refresh_nodes(const std::string & nodes) {
- std::shared_lock lock(mutex);
+ std::shared_lock lock(node_mutex);
if(!node) {
LOG(WARNING) << "Node state is not initialized: unable to refresh nodes.";
@@ -497,85 +502,49 @@ void ReplicationState::refresh_nodes(const std::string & nodes) {
}
void ReplicationState::refresh_catchup_status(bool log_msg) {
- std::shared_lock lock(mutex);
+ std::shared_lock lock(node_mutex);
if (!node) {
LOG_IF(WARNING, log_msg) << "Node state is not initialized: unable to refresh nodes.";
return;
}
- if(node->is_leader()) {
- this->caught_up = true;
- } else {
-
+ if(!node->is_leader() && node->leader_id().is_empty()) {
// follower does not have a leader!
- if(node->leader_id().is_empty()) {
- this->caught_up = false;
- return ;
- }
-
- lock.unlock();
-
- // update catch up status
- thread_pool->enqueue([this, log_msg]() {
- auto seq_num = this->store->get_latest_seq_number();
- std::shared_lock lock(this->mutex);
- const std::string & leader_addr = node->leader_id().to_string();
- lock.unlock();
-
- std::vector<std::string> addr_parts;
- StringUtils::split(leader_addr, addr_parts, ":");
- std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];
- std::string protocol = this->api_uses_ssl ? "https" : "http";
- std::string seq_url = protocol + "://" + leader_host_port + "/sequence";
-
- std::string api_res;
- std::map<std::string, std::string> res_headers;
- long status = HttpClient::get_response(seq_url, api_res, res_headers);
-
- if(status != 500) {
- if(!StringUtils::is_uint64_t(api_res)) {
- LOG_IF(ERROR, log_msg) << "Invalid API response when fetching sequence number: " << api_res;
- this->caught_up = false;
- return ;
- }
-
- uint64_t leader_seq = std::atoll(api_res.c_str());
+ this->read_caught_up = false;
+ this->write_caught_up = false;
+ return ;
+ }
- // Since leader waits for writes on followers to finish, follower's storage offset could be
- // momentarily ahead of the leader's. So we will use std::abs() for checking the difference.
- const uint64_t seq_diff = std::abs(int64_t(leader_seq) - int64_t(seq_num));
+ braft::NodeStatus n_status;
+ node->get_status(&n_status);
+ lock.unlock();
- if(seq_diff < catchup_min_sequence_diff) {
- this->caught_up = true;
- return ;
- }
+ if (n_status.applying_index == 0) {
+ this->read_caught_up = true;
+ this->write_caught_up = true;
+ return ;
+ }
- // However, if the difference is large, then something could be wrong
- if(leader_seq < seq_num) {
- LOG_IF(ERROR, log_msg) << "Leader sequence " << leader_seq << " is less than local sequence "
- << seq_num << ", catchup_min_sequence_diff: " << catchup_min_sequence_diff;
- this->caught_up = false;
- return ;
- }
+ size_t apply_lag = size_t(n_status.last_index - n_status.known_applied_index);
- float seq_progress = (float(seq_num) / leader_seq) * 100;
- LOG_IF(INFO, log_msg) << "Follower progress percentage: " << seq_progress;
+ if (apply_lag > read_max_lag) {
+ LOG(ERROR) << apply_lag << " lagging entries > read max lag of " + std::to_string(read_max_lag);
+ this->read_caught_up = false;
+ }
- this->caught_up = (seq_progress >= catch_up_threshold_percentage);
- }
- });
+ if (apply_lag > write_max_lag) {
+ LOG(ERROR) << apply_lag << " lagging entries > write max lag of " + std::to_string(write_max_lag);
+ this->write_caught_up = false;
}
}
ReplicationState::ReplicationState(HttpServer* server, Store *store, ThreadPool* thread_pool,
http_message_dispatcher *message_dispatcher,
- bool api_uses_ssl, size_t catchup_min_sequence_diff,
- size_t catch_up_threshold_percentage,
+ bool api_uses_ssl, size_t read_max_lag, size_t write_max_lag,
size_t num_collections_parallel_load, size_t num_documents_parallel_load):
node(nullptr), leader_term(-1), server(server), store(store), thread_pool(thread_pool),
- message_dispatcher(message_dispatcher),
- catchup_min_sequence_diff(catchup_min_sequence_diff), catch_up_threshold_percentage(catch_up_threshold_percentage),
+ message_dispatcher(message_dispatcher), read_max_lag(read_max_lag), write_max_lag(write_max_lag),
num_collections_parallel_load(num_collections_parallel_load),
num_documents_parallel_load(num_documents_parallel_load),
api_uses_ssl(api_uses_ssl),
@@ -584,18 +553,23 @@ ReplicationState::ReplicationState(HttpServer* server, Store *store, ThreadPool*
}
bool ReplicationState::is_alive() const {
- std::shared_lock lock(mutex);
+ std::shared_lock lock(node_mutex);
+
+ if(node == nullptr ) {
+ return false;
+ }
- if(node == nullptr || !is_ready()) {
+ bool leader_or_follower = (node->is_leader() || !node->leader_id().is_empty());
+ if(!leader_or_follower) {
return false;
}
- // node should either be a leader or have a leader
- return (node->is_leader() || !node->leader_id().is_empty());
+ // for general health check we will only care about the `read_caught_up` threshold
+ return read_caught_up;
}
uint64_t ReplicationState::node_state() const {
- std::shared_lock lock(mutex);
+ std::shared_lock lock(node_mutex);
if(node == nullptr) {
return 0;
@@ -614,7 +588,7 @@ void ReplicationState::do_snapshot(const std::string& snapshot_path, const std::
thread_pool->enqueue([&snapshot_path, req, res, this]() {
OnDemandSnapshotClosure* snapshot_closure = new OnDemandSnapshotClosure(this, req, res);
ext_snapshot_path = snapshot_path;
- std::shared_lock lock(this->mutex);
+ std::shared_lock lock(this->node_mutex);
node->snapshot(snapshot_closure);
});
}
@@ -628,7 +602,7 @@ const std::string &ReplicationState::get_ext_snapshot_path() const {
}
void ReplicationState::do_dummy_write() {
- std::shared_lock lock(mutex);
+ std::shared_lock lock(node_mutex);
if(node->leader_id().is_empty()) {
LOG(ERROR) << "Could not do a dummy write, as node does not have a leader";
@@ -649,7 +623,7 @@ void ReplicationState::do_dummy_write() {
}
bool ReplicationState::trigger_vote() {
- std::shared_lock lock(mutex);
+ std::shared_lock lock(node_mutex);
if(node) {
auto status = node->vote(election_timeout_interval_ms);
@@ -680,7 +654,7 @@ void ReplicationState::shutdown() {
}
LOG(INFO) << "Replication state shutdown, store sequence: " << store->get_latest_seq_number();
- std::unique_lock lock(mutex);
+ std::unique_lock lock(node_mutex);
if (node) {
LOG(INFO) << "node->shutdown";
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index d94afa86..6adc338f 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -78,8 +78,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<float>("max-memory-ratio", '\0', "Maximum fraction of system memory to be used.", false, 1.0f);
options.add<int>("snapshot-interval-seconds", '\0', "Frequency of replication log snapshots.", false, 3600);
- options.add<int>("catch-up-min-sequence-diff", '\0', "The absolute storage sequence difference within which a follower is deemed to have caught up with leader.", false, 3000);
- options.add<int>("catch-up-threshold-percentage", '\0', "The threshold at which a follower is deemed to have caught up with leader.", false, 95);
+ options.add<int>("read-max-lag", '\0', "Reads are rejected if the updates lag behind this threshold.", false, 1000);
+ options.add<int>("write-max-lag", '\0', "Writes are rejected if the updates lag behind this threshold.", false, 100);
options.add<int>("log-slow-requests-time-ms", '\0', "When > 0, requests that take longer than this duration are logged.", false, -1);
options.add<uint32_t>("num-collections-parallel-load", '\0', "Number of collections that are loaded in parallel during start up.", false, 4);
@@ -385,8 +385,9 @@ int run_server(const Config & config, const std::string & version, void (*master
// first we start the peering service
ReplicationState replication_state(server, &store, &app_thread_pool, server->get_message_dispatcher(),
- ssl_enabled, config.get_catch_up_min_sequence_diff(),
- config.get_catch_up_threshold_percentage(),
+ ssl_enabled,
+ config.get_read_max_lag(),
+ config.get_write_max_lag(),
num_collections_parallel_load,
config.get_num_documents_parallel_load());