Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Flag for skipping writes to avoid bad writes.

Kishore Nallan936270b4a5a4
diff --git a/include/batched_indexer.h b/include/batched_indexer.h
index e75a0cba..3d132eac 100644
--- a/include/batched_indexer.h
+++ b/include/batched_indexer.h
@@ -67,6 +67,9 @@ private:
rocksdb::Iterator* skip_index_iter = nullptr;
static constexpr const char* SKIP_INDICES_PREFIX = "$XP";
+ // When set, all writes (both live and log serialized) are skipped with 422 response
+ const std::atomic<bool>& skip_writes;
+
static const size_t GC_INTERVAL_SECONDS = 60;
static const size_t GC_PRUNE_MAX_SECONDS = 3600;
@@ -78,7 +81,8 @@ public:
static const constexpr char* RAFT_REQ_LOG_PREFIX = "$RL_";
- BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, size_t num_threads);
+ BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, size_t num_threads,
+ const std::atomic<bool>& skip_writes);
~BatchedIndexer();
diff --git a/include/config.h b/include/config.h
index 495aeb8d..aa4a4d30 100644
--- a/include/config.h
+++ b/include/config.h
@@ -5,6 +5,7 @@
#include "option.h"
#include "string_utils.h"
#include "INIReader.h"
+#include "json.hpp"
class Config {
private:
@@ -55,6 +56,8 @@ private:
int disk_used_max_percentage;
+ std::atomic<bool> skip_writes;
+
protected:
Config() {
@@ -74,6 +77,7 @@ protected:
this->ssl_refresh_interval_seconds = 8 * 60 * 60;
this->enable_access_logging = false;
this->disk_used_max_percentage = 100;
+ this->skip_writes = false;
}
Config(Config const&) {
@@ -144,6 +148,10 @@ public:
this->healthy_write_lag = healthy_write_lag;
}
+ void set_skip_writes(bool skip_writes) {
+ this->skip_writes = skip_writes;
+ }
+
// getters
std::string get_data_dir() const {
@@ -267,6 +275,10 @@ public:
return this->log_dir + "/typesense-access.log";
}
+ const std::atomic<bool>& get_skip_writes() const {
+ return skip_writes;
+ }
+
// loaders
std::string get_env(const char *name) {
@@ -371,6 +383,8 @@ public:
if(!get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE").empty()) {
this->disk_used_max_percentage = std::stoi(get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE"));
}
+
+ this->skip_writes = ("TRUE" == get_env("TYPESENSE_SKIP_WRITES"));
}
void load_config_file(cmdline::parser & options) {
@@ -392,7 +406,7 @@ public:
}
config_file_validity = 1;
-
+
if(reader.Exists("server", "data-dir")) {
this->data_dir = reader.Get("server", "data-dir", "");
}
@@ -513,6 +527,11 @@ public:
if(reader.Exists("server", "disk-used-max-percentage")) {
this->disk_used_max_percentage = (int) reader.GetInteger("server", "disk-used-max-percentage", 100);
}
+
+ if(reader.Exists("server", "skip-writes")) {
+ auto skip_writes_str = reader.Get("server", "skip-writes", "false");
+ this->skip_writes = (skip_writes_str == "true");
+ }
}
void load_config_cmd_args(cmdline::parser & options) {
@@ -633,6 +652,10 @@ public:
if(options.exist("disk-used-max-percentage")) {
this->disk_used_max_percentage = options.get<int>("disk-used-max-percentage");
}
+
+ if(options.exist("skip-writes")) {
+ this->skip_writes = options.get<bool>("skip-writes");
+ }
}
void set_cors_domains(std::string& cors_domains_value) {
@@ -659,4 +682,6 @@ public:
return Option<bool>(true);
}
+
+ Option<bool> update_config(const nlohmann::json& req_json);
};
diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cpp
index 99e27a71..248c31b5 100644
--- a/src/batched_indexer.cpp
+++ b/src/batched_indexer.cpp
@@ -2,9 +2,11 @@
#include "core_api.h"
#include "thread_local_vars.h"
-BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, const size_t num_threads):
+BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, const size_t num_threads,
+ const std::atomic<bool>& skip_writes):
server(server), store(store), meta_store(meta_store), num_threads(num_threads),
- last_gc_run(std::chrono::high_resolution_clock::now()), quit(false) {
+ last_gc_run(std::chrono::high_resolution_clock::now()), quit(false),
+ skip_writes(skip_writes) {
queues.resize(num_threads);
qmutuxes = new await_t[num_threads];
}
@@ -194,6 +196,13 @@ void BatchedIndexer::run() {
//LOG(INFO) << "index req " << req_id << ", chunk index: " << orig_req_res.next_chunk_index;
if(route_found) {
+ if(skip_writes && found_rpath->handler != post_config) {
+ orig_res->set(422, "Skipping write.");
+ async_req_res_t* async_req_res = new async_req_res_t(orig_req, orig_res, true);
+ server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, async_req_res);
+ break;
+ }
+
async_res = found_rpath->async_res;
try {
found_rpath->handler(orig_req, orig_res);
diff --git a/src/config.cpp b/src/config.cpp
new file mode 100644
index 00000000..de599c71
--- /dev/null
+++ b/src/config.cpp
@@ -0,0 +1,56 @@
+#include "config.h"
+#include "option.h"
+#include "json.hpp"
+
+Option<bool> Config::update_config(const nlohmann::json& req_json) {
+ bool found_config = false;
+
+ if(req_json.count("log-slow-requests-time-ms") != 0) {
+ if(!req_json["log-slow-requests-time-ms"].is_number_integer()) {
+ return Option<bool>(400, "Configuration `log-slow-requests-time-ms` must be an integer.");
+ }
+
+ set_log_slow_requests_time_ms(req_json["log-slow-requests-time-ms"].get<int>());
+ found_config = true;
+ }
+
+ if(req_json.count("healthy-read-lag") != 0) {
+ if(!req_json["healthy-read-lag"].is_number_integer()) {
+ return Option<bool>(400, "Configuration `healthy-read-lag` must be a positive integer.");
+ }
+
+ int read_lag = req_json["healthy-read-lag"].get<int>();
+ if(read_lag <= 0) {
+ return Option<bool>(400, "Configuration `healthy-read-lag` must be a positive integer.");
+ }
+
+ set_healthy_read_lag(read_lag);
+ found_config = true;
+ }
+
+ if(req_json.count("healthy-write-lag") != 0) {
+ if(!req_json["healthy-write-lag"].is_number_integer()) {
+ return Option<bool>(400, "Configuration `healthy-write-lag` must be an integer.");
+ }
+
+ size_t write_lag = req_json["healthy-write-lag"].get<int>();
+ if(write_lag <= 0) {
+ return Option<bool>(400, "Configuration `healthy-write-lag` must be a positive integer.");
+ }
+
+ set_healthy_write_lag(write_lag);
+ found_config = true;
+ }
+
+ if(req_json.count("skip-writes") != 0) {
+ if(!req_json["skip-writes"].is_boolean()) {
+ return Option<bool>(400, ("Configuration `skip-writes` must be a boolean."));
+ }
+
+ bool skip_writes = req_json["skip-writes"].get<bool>();
+ set_skip_writes(skip_writes);
+ found_config = true;
+ }
+
+ return Option<bool>(true);
+}
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 3b19c152..4baf0edd 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1449,52 +1449,10 @@ bool post_config(const std::shared_ptr<http_req>& req, const std::shared_ptr<htt
return false;
}
- bool found_config = false;
+ auto config_update_op = Config::get_instance().update_config(req_json);
- if(req_json.count("log-slow-requests-time-ms") != 0) {
- if(!req_json["log-slow-requests-time-ms"].is_number_integer()) {
- res->set_400("Configuration `log-slow-requests-time-ms` must be an integer.");
- return false;
- }
-
- Config::get_instance().set_log_slow_requests_time_ms(req_json["log-slow-requests-time-ms"].get<int>());
- found_config = true;
- }
-
- if(req_json.count("healthy-read-lag") != 0) {
- if(!req_json["healthy-read-lag"].is_number_integer()) {
- res->set_400("Configuration `healthy-read-lag` must be a positive integer.");
- return false;
- }
-
- size_t read_lag = req_json["healthy-read-lag"].get<int>();
- if(read_lag <= 0) {
- res->set_400("Configuration `healthy-read-lag` must be a positive integer.");
- return false;
- }
-
- Config::get_instance().set_healthy_read_lag(read_lag);
- found_config = true;
- }
-
- if(req_json.count("healthy-write-lag") != 0) {
- if(!req_json["healthy-write-lag"].is_number_integer()) {
- res->set_400("Configuration `healthy-write-lag` must be an integer.");
- return false;
- }
-
- size_t write_lag = req_json["healthy-write-lag"].get<int>();
- if(write_lag <= 0) {
- res->set_400("Configuration `healthy-write-lag` must be a positive integer.");
- return false;
- }
-
- Config::get_instance().set_healthy_write_lag(req_json["healthy-write-lag"].get<int>());
- found_config = true;
- }
-
- if(!found_config) {
- res->set_400("Invalid configuration.");
+ if(!config_update_op.ok()) {
+ res->set(config_update_op.code(), config_update_op.error());
} else {
nlohmann::json response;
response["success"] = true;
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 3df93670..a4e5398e 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -192,6 +192,12 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const std
return message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);
}
+ if(config->get_skip_writes() && request->path_without_query != "/config") {
+ response->set_422("Skipping writes.");
+ auto req_res = new async_req_res_t(request, response, true);
+ return message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);
+ }
+
std::shared_lock lock(node_mutex);
if(!node) {
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 9e56a8f3..3e9e51ad 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -101,6 +101,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<bool>("enable-access-logging", '\0', "Enable access logging.", false, false);
options.add<int>("disk-used-max-percentage", '\0', "Reject writes when used disk space exceeds this percentage. Default: 100 (never reject).", false, 100);
+ options.add<bool>("skip-writes", '\0', "Skip all writes except config changes. Default: false.", false, false);
// DEPRECATED
options.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");
@@ -435,7 +436,8 @@ int run_server(const Config & config, const std::string & version, void (*master
bool ssl_enabled = (!config.get_ssl_cert().empty() && !config.get_ssl_cert_key().empty());
- BatchedIndexer* batch_indexer = new BatchedIndexer(server, &store, &meta_store, num_threads);
+ BatchedIndexer* batch_indexer = new BatchedIndexer(server, &store, &meta_store, num_threads,
+ config.get_skip_writes());
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),