Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Parameterize db compaction interval.
Kishore Nalland87a48363377
diff --git a/include/store.h b/include/store.hindex ae50d1ab..9cf7a1d3 100644--- a/include/store.h+++ b/include/store.h@@ -84,7 +84,8 @@ public:Store(const std::string & state_dir_path,const size_t wal_ttl_secs = 24*60*60,- const size_t wal_size_mb = 1024, bool disable_wal = true): state_dir_path(state_dir_path) {+ const size_t wal_size_mb = 1024, bool disable_wal = true,+ const size_t db_compaction_interval = 604800): state_dir_path(state_dir_path) {// Optimize RocksDBoptions.IncreaseParallelism();options.OptimizeLevelStyleCompaction();@@ -94,7 +95,7 @@ public:options.max_write_buffer_number = 2;options.merge_operator.reset(new UInt64AddOperator);options.compression = rocksdb::CompressionType::kSnappyCompression;- options.periodic_compaction_seconds = 604800;+ options.periodic_compaction_seconds = db_compaction_interval;options.max_log_file_size = 4*1048576;options.keep_log_file_num = 5;diff --git a/include/tsconfig.h b/include/tsconfig.hindex 67a924e8..da12ffe0 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -72,6 +72,8 @@ private:uint32_t housekeeping_interval;+ uint32_t db_compaction_interval;+protected:Config() {@@ -100,6 +102,7 @@ protected:this->enable_search_analytics = false;this->analytics_flush_interval = 3600; // in secondsthis->housekeeping_interval = 1800; // in seconds+ this->db_compaction_interval = 604800; // in seconds}Config(Config const&) {@@ -309,6 +312,10 @@ public:return this->housekeeping_interval;}+ size_t get_db_compaction_interval() const {+ return this->db_compaction_interval;+ }+size_t get_thread_pool_size() const {return this->thread_pool_size;}@@ -449,6 +456,10 @@ public:this->housekeeping_interval = std::stoi(get_env("TYPESENSE_HOUSEKEEPING_INTERVAL"));}+ if(!get_env("TYPESENSE_DB_COMPACTION_INTERVAL").empty()) {+ this->db_compaction_interval = std::stoi(get_env("TYPESENSE_DB_COMPACTION_INTERVAL"));+ }+if(!get_env("TYPESENSE_THREAD_POOL_SIZE").empty()) {this->thread_pool_size = std::stoi(get_env("TYPESENSE_THREAD_POOL_SIZE"));}@@ -620,6 +631,10 @@ public:this->housekeeping_interval = (int) reader.GetInteger("server", "housekeeping-interval", 1800);}+ if(reader.Exists("server", "db-compaction-interval")) {+ this->db_compaction_interval = (int) reader.GetInteger("server", "db-compaction-interval", 1800);+ }+if(reader.Exists("server", "thread-pool-size")) {this->thread_pool_size = (int) reader.GetInteger("server", "thread-pool-size", 0);}@@ -782,6 +797,10 @@ public:this->housekeeping_interval = options.get<uint32_t>("housekeeping-interval");}+ if(options.exist("db-compaction-interval")) {+ this->db_compaction_interval = options.get<uint32_t>("db-compaction-interval");+ }+if(options.exist("thread-pool-size")) {this->thread_pool_size = options.get<uint32_t>("thread-pool-size");}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex db927485..dcc8f136 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -109,6 +109,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<int>("cache-num-entries", '\0', "Number of entries to cache.", false, 1000);options.add<uint32_t>("analytics-flush-interval", '\0', "Frequency of persisting analytics data to disk (in seconds).", false, 3600);options.add<uint32_t>("housekeeping-interval", '\0', "Frequency of housekeeping background job (in seconds).", false, 1800);+ options.add<uint32_t>("db-compaction-interval", '\0', "Frequency of RocksDB compaction (in seconds).", false, 604800);// DEPRECATEDoptions.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");@@ -394,7 +395,7 @@ int run_server(const Config & config, const std::string & version, void (*masterThreadPool replication_thread_pool(num_threads);// primary DB used for storing the documents: we will not use WAL since Raft provides that- Store store(db_dir);+ Store store(db_dir, 24*60*60, 1024, true, config.get_db_compaction_interval());// meta DB for storing house keeping thingsStore meta_store(meta_dir, 24*60*60, 1024, false);@@ -402,7 +403,7 @@ int run_server(const Config & config, const std::string & version, void (*master//analytics DB for storing query click eventsstd::unique_ptr<Store> analytics_store = nullptr;if(!analytics_dir.empty()) {- analytics_store.reset(new Store(analytics_dir, 24 * 60 * 60, 1024, false));+ analytics_store.reset(new Store(analytics_dir, 24 * 60 * 60, 1024, true, config.get_db_compaction_interval()));}curl_global_init(CURL_GLOBAL_SSL);