Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Gracefully quit during loading on signal.

Kishore Nallanebbab76a43f7
diff --git a/include/batched_indexer.h b/include/batched_indexer.h
index 32b90196..f1d41da8 100644
--- a/include/batched_indexer.h
+++ b/include/batched_indexer.h
@@ -42,7 +42,7 @@ private:
std::chrono::high_resolution_clock::time_point last_gc_run;
- std::atomic<bool> exit;
+ std::atomic<bool> quit;
static const size_t GC_INTERVAL_SECONDS = 60;
static const size_t GC_PRUNE_MAX_SECONDS = 3600;
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 8027755c..0d5b31a8 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -71,6 +71,8 @@ private:
std::atomic<float> max_memory_ratio;
+ std::atomic<bool>* quit;
+
CollectionManager();
~CollectionManager() = default;
@@ -105,7 +107,9 @@ public:
float max_memory_ratio);
static Option<bool> load_collection(const nlohmann::json& collection_meta,
- const size_t init_batch_size, const StoreStatus& next_coll_id_status);
+ const size_t init_batch_size,
+ const StoreStatus& next_coll_id_status,
+ const std::atomic<bool>& quit);
void add_to_collections(Collection* collection);
@@ -115,10 +119,11 @@ public:
// PUBLICLY EXPOSED API
- void init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio, const std::string & auth_key);
+ void init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio,
+ const std::string & auth_key, std::atomic<bool>& quit);
// only for tests!
- void init(Store *store, const float max_memory_ratio, const std::string & auth_key);
+ void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit);
Option<bool> load(const size_t collection_batch_size, const size_t document_batch_size);
diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cpp
index fe04b19d..17c30014 100644
--- a/src/batched_indexer.cpp
+++ b/src/batched_indexer.cpp
@@ -4,7 +4,7 @@
BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, const size_t num_threads):
server(server), store(store), num_threads(num_threads),
- last_gc_run(std::chrono::high_resolution_clock::now()), exit(false) {
+ last_gc_run(std::chrono::high_resolution_clock::now()), quit(false) {
thread_pool = new ThreadPool(num_threads);
queues.resize(num_threads);
qmutuxes = new std::mutex[num_threads];
@@ -112,7 +112,7 @@ void BatchedIndexer::run() {
std::mutex& queue_mutex = qmutuxes[i];
thread_pool->enqueue([&queue, &queue_mutex, this, i]() {
- while(!exit) {
+ while(!quit) {
std::unique_lock<std::mutex> qlk(queue_mutex);
if(queue.empty()) {
@@ -169,6 +169,10 @@ void BatchedIndexer::run() {
queued_writes--;
iter->Next();
+
+ if(quit) {
+ break;
+ }
}
delete iter;
@@ -187,7 +191,7 @@ void BatchedIndexer::run() {
});
}
- while(!exit) {
+ while(!quit) {
std::this_thread::sleep_for(std::chrono::milliseconds (1000));
//LOG(INFO) << "Batch indexer main thread";
@@ -240,7 +244,7 @@ BatchedIndexer::~BatchedIndexer() {
}
void BatchedIndexer::stop() {
- exit = true;
+ quit = true;
}
int64_t BatchedIndexer::get_queued_writes() {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index a4b039b6..115b550d 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -91,19 +91,22 @@ void CollectionManager::add_to_collections(Collection* collection) {
void CollectionManager::init(Store *store, ThreadPool* thread_pool,
const float max_memory_ratio,
- const std::string & auth_key) {
+ const std::string & auth_key,
+ std::atomic<bool>& quit) {
std::unique_lock lock(mutex);
this->store = store;
this->thread_pool = thread_pool;
this->bootstrap_auth_key = auth_key;
this->max_memory_ratio = max_memory_ratio;
+ this->quit = &quit;
}
// used only in tests!
-void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key) {
+void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key,
+ std::atomic<bool>& quit) {
ThreadPool* thread_pool = new ThreadPool(8);
- init(store, thread_pool, max_memory_ratio, auth_key);
+ init(store, thread_pool, max_memory_ratio, auth_key, quit);
}
Option<bool> CollectionManager::load(const size_t collection_batch_size, const size_t document_batch_size) {
@@ -154,8 +157,8 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
auto captured_store = store;
loading_pool.enqueue([captured_store, num_collections, collection_meta, document_batch_size,
- &m_process, &cv_process, &num_processed, &next_coll_id_status]() {
- Option<bool> res = load_collection(collection_meta, document_batch_size, next_coll_id_status);
+ &m_process, &cv_process, &num_processed, &next_coll_id_status, quit = quit]() {
+ Option<bool> res = load_collection(collection_meta, document_batch_size, next_coll_id_status, *quit);
if(!res.ok()) {
LOG(ERROR) << "Error while loading collection. " << res.error();
LOG(ERROR) << "Typesense is quitting.";
@@ -977,7 +980,8 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
Option<bool> CollectionManager::load_collection(const nlohmann::json &collection_meta,
const size_t init_batch_size,
- const StoreStatus& next_coll_id_status) {
+ const StoreStatus& next_coll_id_status,
+ const std::atomic<bool>& quit) {
auto& cm = CollectionManager::get_instance();
@@ -1065,6 +1069,8 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection
size_t num_valid_docs = 0;
size_t num_indexed_docs = 0;
+ auto begin = std::chrono::high_resolution_clock::now();
+
while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {
num_found_docs++;
const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());
@@ -1106,6 +1112,21 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection
index_records.clear();
num_indexed_docs += num_indexed;
}
+
+ if(num_found_docs % 20*1000 == 0) {
+ // having a cheaper higher layer check to prevent checking clock too often
+ auto time_elapsed = std::chrono::duration_cast<std::chrono::seconds>(
+ std::chrono::high_resolution_clock::now() - begin).count();
+
+ if(time_elapsed > 30) {
+ begin = std::chrono::high_resolution_clock::now();
+ LOG(INFO) << "Loaded " << num_found_docs << " documents from " << collection->get_name() << " so far.";
+ }
+ }
+
+ if(quit) {
+ break;
+ }
}
cm.add_to_collections(collection);
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index e7ab2586..70bd05fb 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -366,7 +366,8 @@ int run_server(const Config & config, const std::string & version, void (*master
Store meta_store(meta_dir, 24*60*60, 1024, false);
CollectionManager & collectionManager = CollectionManager::get_instance();
- collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(), config.get_api_key());
+ collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),
+ config.get_api_key(), quit_raft_service);
curl_global_init(CURL_GLOBAL_SSL);
HttpClient & httpClient = HttpClient::get_instance();
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index a4ebdc96..d356037f 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -10,6 +10,7 @@ class CollectionAllFieldsTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
std::vector<std::string> query_fields;
std::vector<sort_by> sort_fields;
@@ -20,7 +21,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
}
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index 2e95dade..7982d041 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -10,6 +10,7 @@ class CollectionFacetingTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
std::vector<std::string> query_fields;
std::vector<sort_by> sort_fields;
@@ -20,7 +21,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
}
diff --git a/test/collection_filtering_test.cpp b/test/collection_filtering_test.cpp
index c12abe8a..9e4471ad 100644
--- a/test/collection_filtering_test.cpp
+++ b/test/collection_filtering_test.cpp
@@ -10,6 +10,7 @@ class CollectionFilteringTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
std::vector<std::string> query_fields;
std::vector<sort_by> sort_fields;
@@ -20,7 +21,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
}
diff --git a/test/collection_grouping_test.cpp b/test/collection_grouping_test.cpp
index c8e2a8b2..32e0de7a 100644
--- a/test/collection_grouping_test.cpp
+++ b/test/collection_grouping_test.cpp
@@ -10,6 +10,7 @@ class CollectionGroupingTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
Collection *coll_group;
void setupCollection() {
@@ -18,7 +19,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
std::vector<field> fields = {
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index cf1d8a33..f54759bc 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -6,6 +6,7 @@ class CollectionLocaleTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
std::vector<std::string> query_fields;
std::vector<sort_by> sort_fields;
@@ -16,7 +17,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
}
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 7361f9c2..505bf364 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -10,6 +10,7 @@ class CollectionManagerTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
Collection *collection1;
std::vector<field> search_fields;
std::vector<sort_by> sort_fields;
@@ -20,7 +21,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
search_fields = {
@@ -250,7 +251,7 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
// create a new collection manager to ensure that it restores the records from the disk backed store
CollectionManager & collectionManager2 = CollectionManager::get_instance();
- collectionManager2.init(store, 1.0, "auth_key");
+ collectionManager2.init(store, 1.0, "auth_key", quit);
auto load_op = collectionManager2.load(8, 1000);
if(!load_op.ok()) {
@@ -345,7 +346,7 @@ TEST_F(CollectionManagerTest, RestoreAutoSchemaDocsOnRestart) {
// create a new collection manager to ensure that it restores the records from the disk backed store
CollectionManager & collectionManager2 = CollectionManager::get_instance();
- collectionManager2.init(store, 1.0, "auth_key");
+ collectionManager2.init(store, 1.0, "auth_key", quit);
auto load_op = collectionManager2.load(8, 1000);
if(!load_op.ok()) {
@@ -454,7 +455,7 @@ TEST_F(CollectionManagerTest, Symlinking) {
std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db";
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
Store *new_store = new Store(state_dir_path);
- cmanager.init(new_store, 1.0, "auth_key");
+ cmanager.init(new_store, 1.0, "auth_key", quit);
cmanager.load(8, 1000);
// try resolving on a blank slate
@@ -529,7 +530,7 @@ TEST_F(CollectionManagerTest, Symlinking) {
// should be able to restore state on init
CollectionManager & cmanager2 = CollectionManager::get_instance();
- cmanager2.init(store, 1.0, "auth_key");
+ cmanager2.init(store, 1.0, "auth_key", quit);
cmanager2.load(8, 1000);
collection_option = cmanager2.resolve_symlink("company");
@@ -557,7 +558,7 @@ TEST_F(CollectionManagerTest, LoadMultipleCollections) {
std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db";
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
Store *new_store = new Store(state_dir_path);
- cmanager.init(new_store, 1.0, "auth_key");
+ cmanager.init(new_store, 1.0, "auth_key", quit);
cmanager.load(8, 1000);
for(size_t i = 0; i < 100; i++) {
@@ -579,7 +580,7 @@ TEST_F(CollectionManagerTest, LoadMultipleCollections) {
delete new_store;
new_store = new Store(state_dir_path);
- cmanager.init(new_store, 1.0, "auth_key");
+ cmanager.init(new_store, 1.0, "auth_key", quit);
cmanager.load(8, 1000);
ASSERT_EQ(100, cmanager.get_collections().size());
diff --git a/test/collection_override_test.cpp b/test/collection_override_test.cpp
index 4d65bf16..31042182 100644
--- a/test/collection_override_test.cpp
+++ b/test/collection_override_test.cpp
@@ -10,6 +10,7 @@ class CollectionOverrideTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
Collection *coll_mul_fields;
void setupCollection() {
@@ -18,7 +19,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index 8d3c0d1b..c69905b9 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -10,6 +10,7 @@ class CollectionSortingTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
std::vector<std::string> query_fields;
std::vector<sort_by> sort_fields;
@@ -20,7 +21,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
}
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 001fdf41..5c44b666 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -10,6 +10,7 @@ class CollectionSpecificTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
std::vector<std::string> query_fields;
std::vector<sort_by> sort_fields;
@@ -20,7 +21,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
}
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
index b8cb8ace..5f9807b3 100644
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@ -10,6 +10,7 @@ class CollectionSynonymsTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
Collection *coll_mul_fields;
void setupCollection() {
@@ -18,7 +19,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
std::ifstream infile(std::string(ROOT_DIR)+"test/multi_field_documents.jsonl");
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 515cc729..fc4a0cd7 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -12,6 +12,7 @@ protected:
std::vector<std::string> query_fields;
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
std::vector<sort_by> sort_fields;
// used for generating random text
@@ -23,7 +24,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
std::ifstream infile(std::string(ROOT_DIR)+"test/documents.jsonl");
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index bb0cce12..3dcadc68 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -9,6 +9,7 @@ class CoreAPIUtilsTest : public ::testing::Test {
protected:
Store *store;
CollectionManager & collectionManager = CollectionManager::get_instance();
+ std::atomic<bool> quit = false;
std::vector<std::string> query_fields;
std::vector<sort_by> sort_fields;
@@ -19,7 +20,7 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
- collectionManager.init(store, 1.0, "auth_key");
+ collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
}