Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Parameterize db compaction interval.
Kishore Nallan741705b367e0
diff --git a/include/store.h b/include/store.hindex ae50d1ab..9cf7a1d3 100644--- a/include/store.h+++ b/include/store.h@@ -84,7 +84,8 @@ public:Store(const std::string & state_dir_path,const size_t wal_ttl_secs = 24*60*60,- const size_t wal_size_mb = 1024, bool disable_wal = true): state_dir_path(state_dir_path) {+ const size_t wal_size_mb = 1024, bool disable_wal = true,+ const size_t db_compaction_interval = 604800): state_dir_path(state_dir_path) {// Optimize RocksDBoptions.IncreaseParallelism();options.OptimizeLevelStyleCompaction();@@ -94,7 +95,7 @@ public:options.max_write_buffer_number = 2;options.merge_operator.reset(new UInt64AddOperator);options.compression = rocksdb::CompressionType::kSnappyCompression;- options.periodic_compaction_seconds = 604800;+ options.periodic_compaction_seconds = db_compaction_interval;options.max_log_file_size = 4*1048576;options.keep_log_file_num = 5;diff --git a/include/tsconfig.h b/include/tsconfig.hindex 5e754d84..f5816965 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -71,6 +71,8 @@ private:uint32_t housekeeping_interval;+ uint32_t db_compaction_interval;+protected:Config() {@@ -99,6 +101,7 @@ protected:this->enable_search_analytics = false;this->analytics_flush_interval = 3600; // in secondsthis->housekeeping_interval = 1800; // in seconds+ this->db_compaction_interval = 604800; // in seconds}Config(Config const&) {@@ -301,6 +304,10 @@ public:return this->housekeeping_interval;}+ size_t get_db_compaction_interval() const {+ return this->db_compaction_interval;+ }+size_t get_thread_pool_size() const {return this->thread_pool_size;}@@ -440,6 +447,10 @@ public:this->housekeeping_interval = std::stoi(get_env("TYPESENSE_HOUSEKEEPING_INTERVAL"));}+ if(!get_env("TYPESENSE_DB_COMPACTION_INTERVAL").empty()) {+ this->db_compaction_interval = std::stoi(get_env("TYPESENSE_DB_COMPACTION_INTERVAL"));+ }+if(!get_env("TYPESENSE_THREAD_POOL_SIZE").empty()) {this->thread_pool_size = std::stoi(get_env("TYPESENSE_THREAD_POOL_SIZE"));}@@ -607,6 +618,10 @@ public:this->housekeeping_interval = (int) reader.GetInteger("server", "housekeeping-interval", 1800);}+ if(reader.Exists("server", "db-compaction-interval")) {+ this->db_compaction_interval = (int) reader.GetInteger("server", "db-compaction-interval", 1800);+ }+if(reader.Exists("server", "thread-pool-size")) {this->thread_pool_size = (int) reader.GetInteger("server", "thread-pool-size", 0);}@@ -765,6 +780,10 @@ public:this->housekeeping_interval = options.get<uint32_t>("housekeeping-interval");}+ if(options.exist("db-compaction-interval")) {+ this->db_compaction_interval = options.get<uint32_t>("db-compaction-interval");+ }+if(options.exist("thread-pool-size")) {this->thread_pool_size = options.get<uint32_t>("thread-pool-size");}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 195db3f2..5fd6f547 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -105,6 +105,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<int>("cache-num-entries", '\0', "Number of entries to cache.", false, 1000);options.add<uint32_t>("analytics-flush-interval", '\0', "Frequency of persisting analytics data to disk (in seconds).", false, 3600);options.add<uint32_t>("housekeeping-interval", '\0', "Frequency of housekeeping background job (in seconds).", false, 1800);+ options.add<uint32_t>("db-compaction-interval", '\0', "Frequency of RocksDB compaction (in seconds).", false, 604800);// DEPRECATEDoptions.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");@@ -389,11 +390,17 @@ int run_server(const Config & config, const std::string & version, void (*masterThreadPool replication_thread_pool(num_threads);// primary DB used for storing the documents: we will not use WAL since Raft provides that- Store store(db_dir);+ Store store(db_dir, 24*60*60, 1024, true, config.get_db_compaction_interval());// meta DB for storing house keeping thingsStore meta_store(meta_dir, 24*60*60, 1024, false);+ //analytics DB for storing query click events+ std::unique_ptr<Store> analytics_store = nullptr;+ if(!analytics_dir.empty()) {+ analytics_store.reset(new Store(analytics_dir, 24 * 60 * 60, 1024, true, config.get_db_compaction_interval()));+ }+curl_global_init(CURL_GLOBAL_SSL);HttpClient & httpClient = HttpClient::get_instance();httpClient.init(config.get_api_key());