Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Parameterize db compaction interval.

Kishore Nalland87a48363377
diff --git a/include/store.h b/include/store.h
index ae50d1ab..9cf7a1d3 100644
--- a/include/store.h
+++ b/include/store.h
@@ -84,7 +84,8 @@ public:
Store(const std::string & state_dir_path,
const size_t wal_ttl_secs = 24*60*60,
- const size_t wal_size_mb = 1024, bool disable_wal = true): state_dir_path(state_dir_path) {
+ const size_t wal_size_mb = 1024, bool disable_wal = true,
+ const size_t db_compaction_interval = 604800): state_dir_path(state_dir_path) {
// Optimize RocksDB
options.IncreaseParallelism();
options.OptimizeLevelStyleCompaction();
@@ -94,7 +95,7 @@ public:
options.max_write_buffer_number = 2;
options.merge_operator.reset(new UInt64AddOperator);
options.compression = rocksdb::CompressionType::kSnappyCompression;
- options.periodic_compaction_seconds = 604800;
+ options.periodic_compaction_seconds = db_compaction_interval;
options.max_log_file_size = 4*1048576;
options.keep_log_file_num = 5;
diff --git a/include/tsconfig.h b/include/tsconfig.h
index 67a924e8..da12ffe0 100644
--- a/include/tsconfig.h
+++ b/include/tsconfig.h
@@ -72,6 +72,8 @@ private:
uint32_t housekeeping_interval;
+ uint32_t db_compaction_interval;
+
protected:
Config() {
@@ -100,6 +102,7 @@ protected:
this->enable_search_analytics = false;
this->analytics_flush_interval = 3600; // in seconds
this->housekeeping_interval = 1800; // in seconds
+ this->db_compaction_interval = 604800; // in seconds
}
Config(Config const&) {
@@ -309,6 +312,10 @@ public:
return this->housekeeping_interval;
}
+ size_t get_db_compaction_interval() const {
+ return this->db_compaction_interval;
+ }
+
size_t get_thread_pool_size() const {
return this->thread_pool_size;
}
@@ -449,6 +456,10 @@ public:
this->housekeeping_interval = std::stoi(get_env("TYPESENSE_HOUSEKEEPING_INTERVAL"));
}
+ if(!get_env("TYPESENSE_DB_COMPACTION_INTERVAL").empty()) {
+ this->db_compaction_interval = std::stoi(get_env("TYPESENSE_DB_COMPACTION_INTERVAL"));
+ }
+
if(!get_env("TYPESENSE_THREAD_POOL_SIZE").empty()) {
this->thread_pool_size = std::stoi(get_env("TYPESENSE_THREAD_POOL_SIZE"));
}
@@ -620,6 +631,10 @@ public:
this->housekeeping_interval = (int) reader.GetInteger("server", "housekeeping-interval", 1800);
}
+ if(reader.Exists("server", "db-compaction-interval")) {
+ this->db_compaction_interval = (int) reader.GetInteger("server", "db-compaction-interval", 1800);
+ }
+
if(reader.Exists("server", "thread-pool-size")) {
this->thread_pool_size = (int) reader.GetInteger("server", "thread-pool-size", 0);
}
@@ -782,6 +797,10 @@ public:
this->housekeeping_interval = options.get<uint32_t>("housekeeping-interval");
}
+ if(options.exist("db-compaction-interval")) {
+ this->db_compaction_interval = options.get<uint32_t>("db-compaction-interval");
+ }
+
if(options.exist("thread-pool-size")) {
this->thread_pool_size = options.get<uint32_t>("thread-pool-size");
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index db927485..dcc8f136 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -109,6 +109,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<int>("cache-num-entries", '\0', "Number of entries to cache.", false, 1000);
options.add<uint32_t>("analytics-flush-interval", '\0', "Frequency of persisting analytics data to disk (in seconds).", false, 3600);
options.add<uint32_t>("housekeeping-interval", '\0', "Frequency of housekeeping background job (in seconds).", false, 1800);
+ options.add<uint32_t>("db-compaction-interval", '\0', "Frequency of RocksDB compaction (in seconds).", false, 604800);
// DEPRECATED
options.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");
@@ -394,7 +395,7 @@ int run_server(const Config & config, const std::string & version, void (*master
ThreadPool replication_thread_pool(num_threads);
// primary DB used for storing the documents: we will not use WAL since Raft provides that
- Store store(db_dir);
+ Store store(db_dir, 24*60*60, 1024, true, config.get_db_compaction_interval());
// meta DB for storing house keeping things
Store meta_store(meta_dir, 24*60*60, 1024, false);
@@ -402,7 +403,7 @@ int run_server(const Config & config, const std::string & version, void (*master
//analytics DB for storing query click events
std::unique_ptr<Store> analytics_store = nullptr;
if(!analytics_dir.empty()) {
- analytics_store.reset(new Store(analytics_dir, 24 * 60 * 60, 1024, false));
+ analytics_store.reset(new Store(analytics_dir, 24 * 60 * 60, 1024, true, config.get_db_compaction_interval()));
}
curl_global_init(CURL_GLOBAL_SSL);