Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Flag for skipping writes to avoid bad writes.
Kishore Nallan936270b4a5a4
diff --git a/include/batched_indexer.h b/include/batched_indexer.hindex e75a0cba..3d132eac 100644--- a/include/batched_indexer.h+++ b/include/batched_indexer.h@@ -67,6 +67,9 @@ private:rocksdb::Iterator* skip_index_iter = nullptr;static constexpr const char* SKIP_INDICES_PREFIX = "$XP";+ // When set, all writes (both live and log serialized) are skipped with 422 response+ const std::atomic<bool>& skip_writes;+static const size_t GC_INTERVAL_SECONDS = 60;static const size_t GC_PRUNE_MAX_SECONDS = 3600;@@ -78,7 +81,8 @@ public:static const constexpr char* RAFT_REQ_LOG_PREFIX = "$RL_";- BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, size_t num_threads);+ BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, size_t num_threads,+ const std::atomic<bool>& skip_writes);~BatchedIndexer();diff --git a/include/config.h b/include/config.hindex 495aeb8d..aa4a4d30 100644--- a/include/config.h+++ b/include/config.h@@ -5,6 +5,7 @@#include "option.h"#include "string_utils.h"#include "INIReader.h"+#include "json.hpp"class Config {private:@@ -55,6 +56,8 @@ private:int disk_used_max_percentage;+ std::atomic<bool> skip_writes;+protected:Config() {@@ -74,6 +77,7 @@ protected:this->ssl_refresh_interval_seconds = 8 * 60 * 60;this->enable_access_logging = false;this->disk_used_max_percentage = 100;+ this->skip_writes = false;}Config(Config const&) {@@ -144,6 +148,10 @@ public:this->healthy_write_lag = healthy_write_lag;}+ void set_skip_writes(bool skip_writes) {+ this->skip_writes = skip_writes;+ }+// gettersstd::string get_data_dir() const {@@ -267,6 +275,10 @@ public:return this->log_dir + "/typesense-access.log";}+ const std::atomic<bool>& get_skip_writes() const {+ return skip_writes;+ }+// loadersstd::string get_env(const char *name) {@@ -371,6 +383,8 @@ public:if(!get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE").empty()) {this->disk_used_max_percentage = std::stoi(get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE"));}++ this->skip_writes = ("TRUE" == get_env("TYPESENSE_SKIP_WRITES"));}void load_config_file(cmdline::parser & options) {@@ -392,7 +406,7 @@ public:}config_file_validity = 1;-+if(reader.Exists("server", "data-dir")) {this->data_dir = reader.Get("server", "data-dir", "");}@@ -513,6 +527,11 @@ public:if(reader.Exists("server", "disk-used-max-percentage")) {this->disk_used_max_percentage = (int) reader.GetInteger("server", "disk-used-max-percentage", 100);}++ if(reader.Exists("server", "skip-writes")) {+ auto skip_writes_str = reader.Get("server", "skip-writes", "false");+ this->skip_writes = (skip_writes_str == "true");+ }}void load_config_cmd_args(cmdline::parser & options) {@@ -633,6 +652,10 @@ public:if(options.exist("disk-used-max-percentage")) {this->disk_used_max_percentage = options.get<int>("disk-used-max-percentage");}++ if(options.exist("skip-writes")) {+ this->skip_writes = options.get<bool>("skip-writes");+ }}void set_cors_domains(std::string& cors_domains_value) {@@ -659,4 +682,6 @@ public:return Option<bool>(true);}++ Option<bool> update_config(const nlohmann::json& req_json);};diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cppindex 99e27a71..248c31b5 100644--- a/src/batched_indexer.cpp+++ b/src/batched_indexer.cpp@@ -2,9 +2,11 @@#include "core_api.h"#include "thread_local_vars.h"-BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, const size_t num_threads):+BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, const size_t num_threads,+ const std::atomic<bool>& skip_writes):server(server), store(store), meta_store(meta_store), num_threads(num_threads),- last_gc_run(std::chrono::high_resolution_clock::now()), quit(false) {+ last_gc_run(std::chrono::high_resolution_clock::now()), quit(false),+ skip_writes(skip_writes) {queues.resize(num_threads);qmutuxes = new await_t[num_threads];}@@ -194,6 +196,13 @@ void BatchedIndexer::run() {//LOG(INFO) << "index req " << req_id << ", chunk index: " << orig_req_res.next_chunk_index;if(route_found) {+ if(skip_writes && found_rpath->handler != post_config) {+ orig_res->set(422, "Skipping write.");+ async_req_res_t* async_req_res = new async_req_res_t(orig_req, orig_res, true);+ server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, async_req_res);+ break;+ }+async_res = found_rpath->async_res;try {found_rpath->handler(orig_req, orig_res);diff --git a/src/config.cpp b/src/config.cppnew file mode 100644index 00000000..de599c71--- /dev/null+++ b/src/config.cpp@@ -0,0 +1,56 @@+#include "config.h"+#include "option.h"+#include "json.hpp"++Option<bool> Config::update_config(const nlohmann::json& req_json) {+ bool found_config = false;++ if(req_json.count("log-slow-requests-time-ms") != 0) {+ if(!req_json["log-slow-requests-time-ms"].is_number_integer()) {+ return Option<bool>(400, "Configuration `log-slow-requests-time-ms` must be an integer.");+ }++ set_log_slow_requests_time_ms(req_json["log-slow-requests-time-ms"].get<int>());+ found_config = true;+ }++ if(req_json.count("healthy-read-lag") != 0) {+ if(!req_json["healthy-read-lag"].is_number_integer()) {+ return Option<bool>(400, "Configuration `healthy-read-lag` must be a positive integer.");+ }++ int read_lag = req_json["healthy-read-lag"].get<int>();+ if(read_lag <= 0) {+ return Option<bool>(400, "Configuration `healthy-read-lag` must be a positive integer.");+ }++ set_healthy_read_lag(read_lag);+ found_config = true;+ }++ if(req_json.count("healthy-write-lag") != 0) {+ if(!req_json["healthy-write-lag"].is_number_integer()) {+ return Option<bool>(400, "Configuration `healthy-write-lag` must be an integer.");+ }++ size_t write_lag = req_json["healthy-write-lag"].get<int>();+ if(write_lag <= 0) {+ return Option<bool>(400, "Configuration `healthy-write-lag` must be a positive integer.");+ }++ set_healthy_write_lag(write_lag);+ found_config = true;+ }++ if(req_json.count("skip-writes") != 0) {+ if(!req_json["skip-writes"].is_boolean()) {+ return Option<bool>(400, ("Configuration `skip-writes` must be a boolean."));+ }++ bool skip_writes = req_json["skip-writes"].get<bool>();+ set_skip_writes(skip_writes);+ found_config = true;+ }++ return Option<bool>(true);+}diff --git a/src/core_api.cpp b/src/core_api.cppindex 3b19c152..4baf0edd 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1449,52 +1449,10 @@ bool post_config(const std::shared_ptr<http_req>& req, const std::shared_ptr<httreturn false;}- bool found_config = false;+ auto config_update_op = Config::get_instance().update_config(req_json);- if(req_json.count("log-slow-requests-time-ms") != 0) {- if(!req_json["log-slow-requests-time-ms"].is_number_integer()) {- res->set_400("Configuration `log-slow-requests-time-ms` must be an integer.");- return false;- }-- Config::get_instance().set_log_slow_requests_time_ms(req_json["log-slow-requests-time-ms"].get<int>());- found_config = true;- }-- if(req_json.count("healthy-read-lag") != 0) {- if(!req_json["healthy-read-lag"].is_number_integer()) {- res->set_400("Configuration `healthy-read-lag` must be a positive integer.");- return false;- }-- size_t read_lag = req_json["healthy-read-lag"].get<int>();- if(read_lag <= 0) {- res->set_400("Configuration `healthy-read-lag` must be a positive integer.");- return false;- }-- Config::get_instance().set_healthy_read_lag(read_lag);- found_config = true;- }-- if(req_json.count("healthy-write-lag") != 0) {- if(!req_json["healthy-write-lag"].is_number_integer()) {- res->set_400("Configuration `healthy-write-lag` must be an integer.");- return false;- }-- size_t write_lag = req_json["healthy-write-lag"].get<int>();- if(write_lag <= 0) {- res->set_400("Configuration `healthy-write-lag` must be a positive integer.");- return false;- }-- Config::get_instance().set_healthy_write_lag(req_json["healthy-write-lag"].get<int>());- found_config = true;- }-- if(!found_config) {- res->set_400("Invalid configuration.");+ if(!config_update_op.ok()) {+ res->set(config_update_op.code(), config_update_op.error());} else {nlohmann::json response;response["success"] = true;diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 3df93670..a4e5398e 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -192,6 +192,12 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const stdreturn message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);}+ if(config->get_skip_writes() && request->path_without_query != "/config") {+ response->set_422("Skipping writes.");+ auto req_res = new async_req_res_t(request, response, true);+ return message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);+ }+std::shared_lock lock(node_mutex);if(!node) {diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 9e56a8f3..3e9e51ad 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -101,6 +101,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<bool>("enable-access-logging", '\0', "Enable access logging.", false, false);options.add<int>("disk-used-max-percentage", '\0', "Reject writes when used disk space exceeds this percentage. Default: 100 (never reject).", false, 100);+ options.add<bool>("skip-writes", '\0', "Skip all writes except config changes. Default: false.", false, false);// DEPRECATEDoptions.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");@@ -435,7 +436,8 @@ int run_server(const Config & config, const std::string & version, void (*masterbool ssl_enabled = (!config.get_ssl_cert().empty() && !config.get_ssl_cert_key().empty());- BatchedIndexer* batch_indexer = new BatchedIndexer(server, &store, &meta_store, num_threads);+ BatchedIndexer* batch_indexer = new BatchedIndexer(server, &store, &meta_store, num_threads,+ config.get_skip_writes());CollectionManager & collectionManager = CollectionManager::get_instance();collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),