Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Gracefully quit during loading on signal.
Kishore Nallanebbab76a43f7
diff --git a/include/batched_indexer.h b/include/batched_indexer.hindex 32b90196..f1d41da8 100644--- a/include/batched_indexer.h+++ b/include/batched_indexer.h@@ -42,7 +42,7 @@ private:std::chrono::high_resolution_clock::time_point last_gc_run;- std::atomic<bool> exit;+ std::atomic<bool> quit;static const size_t GC_INTERVAL_SECONDS = 60;static const size_t GC_PRUNE_MAX_SECONDS = 3600;diff --git a/include/collection_manager.h b/include/collection_manager.hindex 8027755c..0d5b31a8 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -71,6 +71,8 @@ private:std::atomic<float> max_memory_ratio;+ std::atomic<bool>* quit;+CollectionManager();~CollectionManager() = default;@@ -105,7 +107,9 @@ public:float max_memory_ratio);static Option<bool> load_collection(const nlohmann::json& collection_meta,- const size_t init_batch_size, const StoreStatus& next_coll_id_status);+ const size_t init_batch_size,+ const StoreStatus& next_coll_id_status,+ const std::atomic<bool>& quit);void add_to_collections(Collection* collection);@@ -115,10 +119,11 @@ public:// PUBLICLY EXPOSED API- void init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio, const std::string & auth_key);+ void init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio,+ const std::string & auth_key, std::atomic<bool>& quit);// only for tests!- void init(Store *store, const float max_memory_ratio, const std::string & auth_key);+ void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit);Option<bool> load(const size_t collection_batch_size, const size_t document_batch_size);diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cppindex fe04b19d..17c30014 100644--- a/src/batched_indexer.cpp+++ b/src/batched_indexer.cpp@@ -4,7 +4,7 @@BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, const size_t num_threads):server(server), store(store), num_threads(num_threads),- last_gc_run(std::chrono::high_resolution_clock::now()), exit(false) {+ last_gc_run(std::chrono::high_resolution_clock::now()), quit(false) {thread_pool = new ThreadPool(num_threads);queues.resize(num_threads);qmutuxes = new std::mutex[num_threads];@@ -112,7 +112,7 @@ void BatchedIndexer::run() {std::mutex& queue_mutex = qmutuxes[i];thread_pool->enqueue([&queue, &queue_mutex, this, i]() {- while(!exit) {+ while(!quit) {std::unique_lock<std::mutex> qlk(queue_mutex);if(queue.empty()) {@@ -169,6 +169,10 @@ void BatchedIndexer::run() {queued_writes--;iter->Next();++ if(quit) {+ break;+ }}delete iter;@@ -187,7 +191,7 @@ void BatchedIndexer::run() {});}- while(!exit) {+ while(!quit) {std::this_thread::sleep_for(std::chrono::milliseconds (1000));//LOG(INFO) << "Batch indexer main thread";@@ -240,7 +244,7 @@ BatchedIndexer::~BatchedIndexer() {}void BatchedIndexer::stop() {- exit = true;+ quit = true;}int64_t BatchedIndexer::get_queued_writes() {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex a4b039b6..115b550d 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -91,19 +91,22 @@ void CollectionManager::add_to_collections(Collection* collection) {void CollectionManager::init(Store *store, ThreadPool* thread_pool,const float max_memory_ratio,- const std::string & auth_key) {+ const std::string & auth_key,+ std::atomic<bool>& quit) {std::unique_lock lock(mutex);this->store = store;this->thread_pool = thread_pool;this->bootstrap_auth_key = auth_key;this->max_memory_ratio = max_memory_ratio;+ this->quit = &quit;}// used only in tests!-void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key) {+void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key,+ std::atomic<bool>& quit) {ThreadPool* thread_pool = new ThreadPool(8);- init(store, thread_pool, max_memory_ratio, auth_key);+ init(store, thread_pool, max_memory_ratio, auth_key, quit);}Option<bool> CollectionManager::load(const size_t collection_batch_size, const size_t document_batch_size) {@@ -154,8 +157,8 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sauto captured_store = store;loading_pool.enqueue([captured_store, num_collections, collection_meta, document_batch_size,- &m_process, &cv_process, &num_processed, &next_coll_id_status]() {- Option<bool> res = load_collection(collection_meta, document_batch_size, next_coll_id_status);+ &m_process, &cv_process, &num_processed, &next_coll_id_status, quit = quit]() {+ Option<bool> res = load_collection(collection_meta, document_batch_size, next_coll_id_status, *quit);if(!res.ok()) {LOG(ERROR) << "Error while loading collection. " << res.error();LOG(ERROR) << "Typesense is quitting.";@@ -977,7 +980,8 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsoOption<bool> CollectionManager::load_collection(const nlohmann::json &collection_meta,const size_t init_batch_size,- const StoreStatus& next_coll_id_status) {+ const StoreStatus& next_coll_id_status,+ const std::atomic<bool>& quit) {auto& cm = CollectionManager::get_instance();@@ -1065,6 +1069,8 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionsize_t num_valid_docs = 0;size_t num_indexed_docs = 0;+ auto begin = std::chrono::high_resolution_clock::now();+while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {num_found_docs++;const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());@@ -1106,6 +1112,21 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionindex_records.clear();num_indexed_docs += num_indexed;}++ if(num_found_docs % 20*1000 == 0) {+ // having a cheaper higher layer check to prevent checking clock too often+ auto time_elapsed = std::chrono::duration_cast<std::chrono::seconds>(+ std::chrono::high_resolution_clock::now() - begin).count();++ if(time_elapsed > 30) {+ begin = std::chrono::high_resolution_clock::now();+ LOG(INFO) << "Loaded " << num_found_docs << " documents from " << collection->get_name() << " so far.";+ }+ }++ if(quit) {+ break;+ }}cm.add_to_collections(collection);diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex e7ab2586..70bd05fb 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -366,7 +366,8 @@ int run_server(const Config & config, const std::string & version, void (*masterStore meta_store(meta_dir, 24*60*60, 1024, false);CollectionManager & collectionManager = CollectionManager::get_instance();- collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(), config.get_api_key());+ collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),+ config.get_api_key(), quit_raft_service);curl_global_init(CURL_GLOBAL_SSL);HttpClient & httpClient = HttpClient::get_instance();diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex a4ebdc96..d356037f 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -10,6 +10,7 @@ class CollectionAllFieldsTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;std::vector<std::string> query_fields;std::vector<sort_by> sort_fields;@@ -20,7 +21,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);}diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex 2e95dade..7982d041 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -10,6 +10,7 @@ class CollectionFacetingTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;std::vector<std::string> query_fields;std::vector<sort_by> sort_fields;@@ -20,7 +21,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);}diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cppindex c12abe8a..9e4471ad 100644--- a/test/collection_filtering_test.cpp+++ b/test/collection_filtering_test.cpp@@ -10,6 +10,7 @@ class CollectionFilteringTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;std::vector<std::string> query_fields;std::vector<sort_by> sort_fields;@@ -20,7 +21,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);}diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cppindex c8e2a8b2..32e0de7a 100644--- a/test/collection_grouping_test.cpp+++ b/test/collection_grouping_test.cpp@@ -10,6 +10,7 @@ class CollectionGroupingTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;Collection *coll_group;void setupCollection() {@@ -18,7 +19,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);std::vector<field> fields = {diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex cf1d8a33..f54759bc 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -6,6 +6,7 @@ class CollectionLocaleTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;std::vector<std::string> query_fields;std::vector<sort_by> sort_fields;@@ -16,7 +17,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);}diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 7361f9c2..505bf364 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -10,6 +10,7 @@ class CollectionManagerTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;Collection *collection1;std::vector<field> search_fields;std::vector<sort_by> sort_fields;@@ -20,7 +21,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);search_fields = {@@ -250,7 +251,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {// create a new collection manager to ensure that it restores the records from the disk backed storeCollectionManager & collectionManager2 = CollectionManager::get_instance();- collectionManager2.init(store, 1.0, "auth_key");+ collectionManager2.init(store, 1.0, "auth_key", quit);auto load_op = collectionManager2.load(8, 1000);if(!load_op.ok()) {@@ -345,7 +346,7 @@ TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {// create a new collection manager to ensure that it restores the records from the disk backed storeCollectionManager & collectionManager2 = CollectionManager::get_instance();- collectionManager2.init(store, 1.0, "auth_key");+ collectionManager2.init(store, 1.0, "auth_key", quit);auto load_op = collectionManager2.load(8, 1000);if(!load_op.ok()) {@@ -454,7 +455,7 @@ TEST_F(CollectionManagerTest, Symlinking) {std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db";system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());Store *new_store = new Store(state_dir_path);- cmanager.init(new_store, 1.0, "auth_key");+ cmanager.init(new_store, 1.0, "auth_key", quit);cmanager.load(8, 1000);// try resolving on a blank slate@@ -529,7 +530,7 @@ TEST_F(CollectionManagerTest, Symlinking) {// should be able to restore state on initCollectionManager & cmanager2 = CollectionManager::get_instance();- cmanager2.init(store, 1.0, "auth_key");+ cmanager2.init(store, 1.0, "auth_key", quit);cmanager2.load(8, 1000);collection_option = cmanager2.resolve_symlink("company");@@ -557,7 +558,7 @@ TEST_F(CollectionManagerTest, LoadMultipleCollections) {std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db";system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());Store *new_store = new Store(state_dir_path);- cmanager.init(new_store, 1.0, "auth_key");+ cmanager.init(new_store, 1.0, "auth_key", quit);cmanager.load(8, 1000);for(size_t i = 0; i < 100; i++) {@@ -579,7 +580,7 @@ TEST_F(CollectionManagerTest, LoadMultipleCollections) {delete new_store;new_store = new Store(state_dir_path);- cmanager.init(new_store, 1.0, "auth_key");+ cmanager.init(new_store, 1.0, "auth_key", quit);cmanager.load(8, 1000);ASSERT_EQ(100, cmanager.get_collections().size());diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cppindex 4d65bf16..31042182 100644--- a/test/collection_override_test.cpp+++ b/test/collection_override_test.cpp@@ -10,6 +10,7 @@ class CollectionOverrideTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;Collection *coll_mul_fields;void setupCollection() {@@ -18,7 +19,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 8d3c0d1b..c69905b9 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -10,6 +10,7 @@ class CollectionSortingTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;std::vector<std::string> query_fields;std::vector<sort_by> sort_fields;@@ -20,7 +21,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);}diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 001fdf41..5c44b666 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -10,6 +10,7 @@ class CollectionSpecificTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;std::vector<std::string> query_fields;std::vector<sort_by> sort_fields;@@ -20,7 +21,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);}diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex b8cb8ace..5f9807b3 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -10,6 +10,7 @@ class CollectionSynonymsTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;Collection *coll_mul_fields;void setupCollection() {@@ -18,7 +19,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 515cc729..fc4a0cd7 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -12,6 +12,7 @@ protected:std::vector<std::string> query_fields;Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;std::vector<sort_by> sort_fields;// used for generating random text@@ -23,7 +24,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex bb0cce12..3dcadc68 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -9,6 +9,7 @@ class CoreAPIUtilsTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;std::vector<std::string> query_fields;std::vector<sort_by> sort_fields;@@ -19,7 +20,7 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);- collectionManager.init(store, 1.0, "auth_key");+ collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);}