Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Do explicit compaction instead of automatic periodic compaction.
Kishore Nallanb1805978b6d2
diff --git a/include/housekeeper.h b/include/housekeeper.hindex 41243cbd..89c70717 100644--- a/include/housekeeper.h+++ b/include/housekeeper.h@@ -9,7 +9,7 @@ private:std::condition_variable cv;std::atomic<bool> quit = false;- std::atomic<uint32_t> interval_seconds = 1800;+ std::atomic<uint32_t> hnsw_repair_interval_s = 1800;HouseKeeper() {}diff --git a/include/store.h b/include/store.hindex 9cf7a1d3..68b7f635 100644--- a/include/store.h+++ b/include/store.h@@ -84,8 +84,7 @@ public:Store(const std::string & state_dir_path,const size_t wal_ttl_secs = 24*60*60,- const size_t wal_size_mb = 1024, bool disable_wal = true,- const size_t db_compaction_interval = 604800): state_dir_path(state_dir_path) {+ const size_t wal_size_mb = 1024, bool disable_wal = true): state_dir_path(state_dir_path) {// Optimize RocksDBoptions.IncreaseParallelism();options.OptimizeLevelStyleCompaction();@@ -95,7 +94,6 @@ public:options.max_write_buffer_number = 2;options.merge_operator.reset(new UInt64AddOperator);options.compression = rocksdb::CompressionType::kSnappyCompression;- options.periodic_compaction_seconds = db_compaction_interval;options.max_log_file_size = 4*1048576;options.keep_log_file_num = 5;diff --git a/include/tsconfig.h b/include/tsconfig.hindex da12ffe0..3c48dea1 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -102,7 +102,7 @@ protected:this->enable_search_analytics = false;this->analytics_flush_interval = 3600; // in secondsthis->housekeeping_interval = 1800; // in seconds- this->db_compaction_interval = 604800; // in seconds+ this->db_compaction_interval = 0; // in seconds, disabled}Config(Config const&) {@@ -632,7 +632,7 @@ public:}if(reader.Exists("server", "db-compaction-interval")) {- this->db_compaction_interval = (int) reader.GetInteger("server", "db-compaction-interval", 1800);+ this->db_compaction_interval = (int) reader.GetInteger("server", "db-compaction-interval", 0);}if(reader.Exists("server", "thread-pool-size")) {diff --git a/src/housekeeper.cpp b/src/housekeeper.cppindex 370a3074..1dccfee3 100644--- a/src/housekeeper.cpp+++ b/src/housekeeper.cpp@@ -2,7 +2,10 @@#include "housekeeper.h"void HouseKeeper::run() {- uint64_t prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(+ uint64_t prev_hnsw_repair_s = std::chrono::duration_cast<std::chrono::seconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ uint64_t prev_db_compaction_s = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();while(!quit) {@@ -17,29 +20,38 @@ void HouseKeeper::run() {auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();- if(now_ts_seconds - prev_persistence_s < interval_seconds) {- continue;+ // perform compaction on underlying store if enabled+ if(Config::get_instance().get_db_compaction_interval() > 0) {+ if(now_ts_seconds - prev_db_compaction_s >= Config::get_instance().get_db_compaction_interval()) {+ LOG(INFO) << "Starting DB compaction.";+ CollectionManager::get_instance().get_store()->compact_all();+ LOG(INFO) << "Finished DB compaction.";+ prev_db_compaction_s = std::chrono::duration_cast<std::chrono::seconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();+ }}- // iterate through all collections and repair all hnsw graphs- auto coll_names = CollectionManager::get_instance().get_collection_names();+ if(now_ts_seconds - prev_hnsw_repair_s >= hnsw_repair_interval_s) {+ // iterate through all collections and repair all hnsw graphs (if any)+ auto coll_names = CollectionManager::get_instance().get_collection_names();++ for(auto& coll_name: coll_names) {+ auto coll = CollectionManager::get_instance().get_collection(coll_name);+ if(coll == nullptr) {+ continue;+ }- for(auto& coll_name: coll_names) {- auto coll = CollectionManager::get_instance().get_collection(coll_name);- if(coll == nullptr) {- continue;+ coll->do_housekeeping();}- coll->do_housekeeping();- }+ if(!coll_names.empty()) {+ LOG(INFO) << "Ran housekeeping for " << coll_names.size() << " collections.";+ }- if(!coll_names.empty()) {- LOG(INFO) << "Ran housekeeping for " << coll_names.size() << " collections.";+ prev_hnsw_repair_s = std::chrono::duration_cast<std::chrono::seconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();}- prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(- std::chrono::system_clock::now().time_since_epoch()).count();-lk.unlock();}}@@ -50,5 +62,5 @@ void HouseKeeper::stop() {}void HouseKeeper::init(uint32_t interval_seconds) {- this->interval_seconds = interval_seconds;+ this->hnsw_repair_interval_s = interval_seconds;}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex dcc8f136..d51f0df8 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -395,7 +395,7 @@ int run_server(const Config & config, const std::string & version, void (*masterThreadPool replication_thread_pool(num_threads);// primary DB used for storing the documents: we will not use WAL since Raft provides that- Store store(db_dir, 24*60*60, 1024, true, config.get_db_compaction_interval());+ Store store(db_dir, 24*60*60, 1024, true);// meta DB for storing house keeping thingsStore meta_store(meta_dir, 24*60*60, 1024, false);@@ -403,7 +403,7 @@ int run_server(const Config & config, const std::string & version, void (*master//analytics DB for storing query click eventsstd::unique_ptr<Store> analytics_store = nullptr;if(!analytics_dir.empty()) {- analytics_store.reset(new Store(analytics_dir, 24 * 60 * 60, 1024, true, config.get_db_compaction_interval()));+ analytics_store.reset(new Store(analytics_dir, 24 * 60 * 60, 1024, true));}curl_global_init(CURL_GLOBAL_SSL);