Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add guard for protecting writes from low disk space.

Kishore Nallan73cc256e2f6b
diff --git a/include/config.h b/include/config.h
index 5be9288f..cbd510a4 100644
--- a/include/config.h
+++ b/include/config.h
@@ -52,6 +52,8 @@ private:
bool enable_access_logging;
+ int disk_used_max_percentage;
+
protected:
Config() {
@@ -69,6 +71,7 @@ protected:
this->thread_pool_size = 0; // will be set dynamically if not overridden
this->ssl_refresh_interval_seconds = 8 * 60 * 60;
this->enable_access_logging = false;
+ this->disk_used_max_percentage = 100;
}
Config(Config const&) {
@@ -246,6 +249,10 @@ public:
return this->enable_access_logging;
}
+ int get_disk_used_max_percentage() const {
+ return this->disk_used_max_percentage;
+ }
+
std::string get_access_log_path() const {
if(this->log_dir.empty()) {
return "";
@@ -350,6 +357,10 @@ public:
}
this->enable_access_logging = ("TRUE" == get_env("TYPESENSE_ENABLE_ACCESS_LOGGING"));
+
+ if(!get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE").empty()) {
+ this->disk_used_max_percentage = std::stoi(get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE"));
+ }
}
void load_config_file(cmdline::parser & options) {
@@ -481,9 +492,13 @@ public:
}
if(reader.Exists("server", "enable-access-logging")) {
- auto enable_access_logging_str = reader.Get("server", "enable-cors", "false");
+ auto enable_access_logging_str = reader.Get("server", "enable-access-logging", "false");
this->enable_access_logging = (enable_access_logging_str == "true");
}
+
+ if(reader.Exists("server", "disk-used-max-percentage")) {
+ this->disk_used_max_percentage = (int) reader.GetInteger("server", "disk-used-max-percentage", 100);
+ }
}
void load_config_cmd_args(cmdline::parser & options) {
@@ -596,6 +611,10 @@ public:
if(options.exist("enable-access-logging")) {
this->enable_access_logging = options.get<bool>("enable-access-logging");
}
+
+ if(options.exist("disk-used-max-percentage")) {
+ this->disk_used_max_percentage = options.get<int>("disk-used-max-percentage");
+ }
}
void set_cors_domains(std::string& cors_domains_value) {
diff --git a/include/raft_server.h b/include/raft_server.h
index c69344a3..6e3ecc41 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -8,6 +8,7 @@
#include <braft/protobuf_file.h> // braft::ProtoBufFile
#include <rocksdb/db.h>
#include <future>
+#include <sys/statvfs.h>
#include "http_data.h"
#include "threadpool.h"
@@ -81,6 +82,29 @@ public:
void Run();
};
+struct cached_disk_stat_t {
+ const static size_t REFRESH_INTERVAL_SECS = 30;
+ uint64_t disk_total_bytes = 0;
+ uint64_t disk_used_bytes = 0;
+ uint64_t last_checked_ts = 0;
+
+ bool has_enough_space(const std::string& data_dir_path, const int disk_used_max_percentage) {
+ uint64_t now = std::chrono::duration_cast<std::chrono::seconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ if((now - last_checked_ts) > REFRESH_INTERVAL_SECS) {
+ struct statvfs st{};
+ statvfs(data_dir_path.c_str(), &st);
+ disk_total_bytes = st.f_blocks * st.f_frsize;
+ disk_used_bytes = (st.f_blocks - st.f_bavail) * st.f_frsize;
+ last_checked_ts = now;
+ }
+
+ int disk_used_percentage = (double(disk_used_bytes)/double(disk_total_bytes)) * 100;
+ return disk_used_percentage <= disk_used_max_percentage;
+ }
+};
+
// Implements braft::StateMachine.
class ReplicationState : public braft::StateMachine {
@@ -123,6 +147,8 @@ private:
std::atomic<bool> shutting_down;
std::atomic<size_t> pending_writes;
+ cached_disk_stat_t cached_disk_stat;
+
public:
static constexpr const char* log_dir_name = "log";
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index dc5622f6..63906142 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -182,6 +182,13 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const std
return ;
}
+ // reject write if disk space is running out
+ if(!cached_disk_stat.has_enough_space(raft_dir_path, config->get_disk_used_max_percentage())) {
+ response->set_500("Rejecting write: running out of disk space!");
+ auto req_res = new async_req_res_t(request, response, true);
+ return message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);
+ }
+
std::shared_lock lock(node_mutex);
if(!node) {
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index b214354b..3cde441d 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -98,6 +98,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("config", '\0', "Path to the configuration file.", false, "");
options.add<bool>("enable-access-logging", '\0', "Enable access logging.", false, false);
+ options.add<int>("disk-used-max-percentage", '\0', "Reject writes when used disk space exceeds this percentage. Default: 100 (never reject).", false, 100);
// DEPRECATED
options.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");