Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow read/write lags to be updated via config API.
Kishore Nallanb24df3c26a7d
diff --git a/include/config.h b/include/config.hindex 4e0118c1..715de85c 100644--- a/include/config.h+++ b/include/config.h@@ -1,5 +1,6 @@#pragma once+#include <atomic>#include <cmdline.h>#include "option.h"#include "string_utils.h"@@ -33,13 +34,13 @@ private:float max_memory_ratio;int snapshot_interval_seconds;- size_t healthy_read_lag;- size_t healthy_write_lag;+ std::atomic<size_t> healthy_read_lag;+ std::atomic<size_t> healthy_write_lag;std::string config_file;int config_file_validity;- int log_slow_requests_time_ms;+ std::atomic<int> log_slow_requests_time_ms;uint32_t num_collections_parallel_load;uint32_t num_documents_parallel_load;@@ -124,6 +125,14 @@ public:this->log_slow_requests_time_ms = log_slow_requests_time_ms;}+ void set_healthy_read_lag(size_t healthy_read_lag) {+ this->healthy_read_lag = healthy_read_lag;+ }++ void set_healthy_write_lag(size_t healthy_write_lag) {+ this->healthy_write_lag = healthy_write_lag;+ }+// gettersstd::string get_data_dir() const {@@ -191,11 +200,11 @@ public:return this->snapshot_interval_seconds;}- int get_healthy_read_lag() const {+ size_t get_healthy_read_lag() const {return this->healthy_read_lag;}- int get_healthy_write_lag() const {+ size_t get_healthy_write_lag() const {return this->healthy_write_lag;}@@ -397,11 +406,11 @@ public:}if(reader.Exists("server", "healthy-read-lag")) {- this->healthy_read_lag = (int) reader.GetInteger("server", "healthy-read-lag", 1000);+ this->healthy_read_lag = (size_t) reader.GetInteger("server", "healthy-read-lag", 1000);}if(reader.Exists("server", "healthy-write-lag")) {- this->healthy_write_lag = (int) reader.GetInteger("server", "healthy-write-lag", 100);+ this->healthy_write_lag = (size_t) reader.GetInteger("server", "healthy-write-lag", 100);}if(reader.Exists("server", "log-slow-requests-time-ms")) {@@ -496,11 +505,11 @@ public:}if(options.exist("healthy-read-lag")) {- this->healthy_read_lag = options.get<int>("healthy-read-lag");+ this->healthy_read_lag = options.get<size_t>("healthy-read-lag");}if(options.exist("healthy-write-lag")) {- this->healthy_write_lag = options.get<int>("healthy-write-lag");+ this->healthy_write_lag = options.get<size_t>("healthy-write-lag");}if(options.exist("log-slow-requests-time-ms")) {diff --git a/include/raft_server.h b/include/raft_server.hindex 28147f66..101f7e9d 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -109,8 +109,7 @@ private:const bool api_uses_ssl;- const int64_t healthy_read_lag;- const int64_t healthy_write_lag;+ const Config* config;const size_t num_collections_parallel_load;const size_t num_documents_parallel_load;@@ -139,7 +138,7 @@ public:ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer, Store* store, Store* meta_store,ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,- bool api_uses_ssl, int64_t healthy_read_lag, int64_t healthy_write_lag,+ bool api_uses_ssl, const Config* config,size_t num_collections_parallel_load, size_t num_documents_parallel_load);// Starts this nodediff --git a/src/core_api.cpp b/src/core_api.cppindex e0934131..bc85cbcb 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1231,6 +1231,8 @@ bool post_config(const std::shared_ptr<http_req>& req, const std::shared_ptr<httreturn false;}+ bool found_config = false;+if(req_json.count("log-slow-requests-time-ms") != 0) {if(!req_json["log-slow-requests-time-ms"].is_number_integer()) {res->set_400("Configuration `log-slow-requests-time-ms` must be an integer.");@@ -1238,12 +1240,47 @@ bool post_config(const std::shared_ptr<http_req>& req, const std::shared_ptr<htt}Config::get_instance().set_log_slow_requests_time_ms(req_json["log-slow-requests-time-ms"].get<int>());+ found_config = true;+ }++ if(req_json.count("healthy-read-lag") != 0) {+ if(!req_json["healthy-read-lag"].is_number_integer()) {+ res->set_400("Configuration `healthy-read-lag` must be a positive integer.");+ return false;+ }++ size_t read_lag = req_json["healthy-read-lag"].get<int>();+ if(read_lag <= 0) {+ res->set_400("Configuration `healthy-read-lag` must be a positive integer.");+ return false;+ }++ Config::get_instance().set_healthy_read_lag(read_lag);+ found_config = true;+ }++ if(req_json.count("healthy-write-lag") != 0) {+ if(!req_json["healthy-write-lag"].is_number_integer()) {+ res->set_400("Configuration `healthy-write-lag` must be an integer.");+ return false;+ }++ size_t write_lag = req_json["healthy-write-lag"].get<int>();+ if(write_lag <= 0) {+ res->set_400("Configuration `healthy-write-lag` must be a positive integer.");+ return false;+ }+ Config::get_instance().set_healthy_write_lag(req_json["healthy-write-lag"].get<int>());+ found_config = true;+ }++ if(!found_config) {+ res->set_400("Invalid configuration.");+ } else {nlohmann::json response;response["success"] = true;res->set_201(response.dump());- } else {- res->set_400("Invalid configuration.");}return true;diff --git a/src/http_server.cpp b/src/http_server.cppindex 61fe1039..453f2bae 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -355,7 +355,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {!(root_resource == "health" || root_resource == "debug" ||root_resource == "stats.json" || root_resource == "metrics.json" ||- root_resource == "sequence" || root_resource == "operations"+ root_resource == "sequence" || root_resource == "operations" || root_resource == "config");if(needs_readiness_check) {diff --git a/src/raft_server.cpp b/src/raft_server.cppindex ff3cf605..bea91fdb 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -79,7 +79,7 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int// flag controls snapshot download size of each RPCbraft::FLAGS_raft_max_byte_count_per_rpc = 4 * 1024 * 1024; // 4 MB- node_options.catchup_margin = healthy_read_lag;+ node_options.catchup_margin = config->get_healthy_read_lag();node_options.election_timeout_ms = election_timeout_ms;node_options.fsm = this;node_options.node_owns_fsm = false;@@ -607,6 +607,9 @@ void ReplicationState::refresh_catchup_status(bool log_msg) {//LOG(INFO) << "last_index: " << n_status.applying_index << ", known_applied_index: " << n_status.known_applied_index;//LOG(INFO) << "apply_lag: " << apply_lag;+ int healthy_read_lag = config->get_healthy_read_lag();+ int healthy_write_lag = config->get_healthy_write_lag();+if (apply_lag > healthy_read_lag) {LOG_IF(ERROR, log_msg) << apply_lag << " lagging entries > healthy read lag of " << healthy_read_lag;this->read_caught_up = false;@@ -635,13 +638,12 @@ void ReplicationState::refresh_catchup_status(bool log_msg) {ReplicationState::ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer,Store *store, Store* meta_store, ThreadPool* thread_pool,http_message_dispatcher *message_dispatcher,- bool api_uses_ssl,- int64_t healthy_read_lag, int64_t healthy_write_lag,+ bool api_uses_ssl, const Config* config,size_t num_collections_parallel_load, size_t num_documents_parallel_load):node(nullptr), leader_term(-1), server(server), batched_indexer(batched_indexer),store(store), meta_store(meta_store),thread_pool(thread_pool), message_dispatcher(message_dispatcher), api_uses_ssl(api_uses_ssl),- healthy_read_lag(healthy_read_lag), healthy_write_lag(healthy_write_lag),+ config(config),num_collections_parallel_load(num_collections_parallel_load),num_documents_parallel_load(num_documents_parallel_load),ready(false), shutting_down(false), pending_writes(0) {diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 6b4b214e..832c8d14 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -398,8 +398,7 @@ int run_server(const Config & config, const std::string & version, void (*masterReplicationState replication_state(server, &batch_indexer, &store, &meta_store,&app_thread_pool, server->get_message_dispatcher(),ssl_enabled,- config.get_healthy_read_lag(),- config.get_healthy_write_lag(),+ &config,num_collections_parallel_load,config.get_num_documents_parallel_load());