Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Load collections parallelly during start up.
kishorencaf6d2e94e528
diff --git a/CMakeLists.txt b/CMakeLists.txtindex 1d585a1a..aaef720d 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -3,8 +3,8 @@ project(typesense)cmake_policy(SET CMP0074 NEW)-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Wno-unused-parameter -O2 -g -DNDEBUG")-set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-parameter -std=c++17 -O0 -g")+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -Wall -Wextra -Wno-unused-parameter -Werror=return-type -O2 -g -DNDEBUG")+set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -Wall -Wextra -Wno-unused-parameter -Werror=return-type -std=c++17 -O0 -g")set(DEP_ROOT_DIR ${CMAKE_SOURCE_DIR}/external-${CMAKE_SYSTEM_NAME})set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/")diff --git a/include/collection_manager.h b/include/collection_manager.hindex ff253a9a..99c4db4f 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -69,7 +69,7 @@ private:std::string bootstrap_auth_key;- float max_memory_ratio;+ std::atomic<float> max_memory_ratio;/// During load, sleep `LOAD_THROTTLE_PERCENT`% of time taken for indexing to prevent CPU saturationfloat LOAD_THROTTLE_PERCENT = 10;@@ -102,13 +102,19 @@ public:CollectionManager(CollectionManager const&) = delete;void operator=(CollectionManager const&) = delete;- Collection* init_collection(const nlohmann::json & collection_meta, const uint32_t collection_next_seq_id);+ static Collection* init_collection(const nlohmann::json & collection_meta,+ const uint32_t collection_next_seq_id,+ Store* store,+ float max_memory_ratio);++ static Option<bool> load_collection(const nlohmann::json& collection_meta,+ const size_t init_batch_size, const StoreStatus& next_coll_id_status);void add_to_collections(Collection* collection);std::vector<Collection*> get_collections() const;- Collection* get_collection_unafe(const std::string & collection_name) const;+ Collection* get_collection_unsafe(const std::string & collection_name) const;// PUBLICLY EXPOSED APIdiff --git a/include/option.h b/include/option.hindex e837920b..1889a09b 100644--- a/include/option.h+++ b/include/option.h@@ -13,6 +13,8 @@ private:public:+ explicit Option() = delete;+explicit Option(const T & value): value(value), is_ok(true) {}diff --git a/src/collection.cpp b/src/collection.cppindex f2003e44..c5be2187 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -857,8 +857,6 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::std::unique_lock<std::mutex> lock_process(m_process);cv_process.wait(lock_process, [&](){ return num_processed == num_indices; });- Option<nlohmann::json> index_search_op({}); // stores the last error across all index threads-// for grouping we have re-aggregateconst size_t topster_size = std::max((size_t)1, max_hits);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex dd40aed1..ac3c2282 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -11,7 +11,9 @@ CollectionManager::CollectionManager() {}Collection* CollectionManager::init_collection(const nlohmann::json & collection_meta,- const uint32_t collection_next_seq_id) {+ const uint32_t collection_next_seq_id,+ Store* store,+ float max_memory_ratio) {std::string this_collection_name = collection_meta[Collection::COLLECTION_NAME_KEY].get<std::string>();std::vector<field> fields;@@ -68,8 +70,11 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection}void CollectionManager::add_to_collections(Collection* collection) {- collections.emplace(collection->get_name(), collection);- collection_id_names.emplace(collection->get_collection_id(), collection->get_name());+ const std::string& collection_name = collection->get_name();+ const uint32_t collection_id = collection->get_collection_id();+ std::unique_lock lock(mutex);+ collections.emplace(collection_name, collection);+ collection_id_names.emplace(collection_id, collection_name);}void CollectionManager::init(Store *store, ThreadPool* thread_pool,@@ -116,151 +121,53 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {LOG(INFO) << "Found " << collection_meta_jsons.size() << " collection(s) on disk.";- for(auto & collection_meta_json: collection_meta_jsons) {- nlohmann::json collection_meta;+ const size_t PROC_COUNT = std::max<size_t>(1, std::thread::hardware_concurrency());+ std::vector<nlohmann::json> coll_meta_buffer;- try {- collection_meta = nlohmann::json::parse(collection_meta_json);- } catch(...) {+ for(size_t coll_index = 0; coll_index < collection_meta_jsons.size(); coll_index++) {+ const auto& collection_meta_json = collection_meta_jsons[coll_index];+ nlohmann::json collection_meta = nlohmann::json::parse(collection_meta_json, nullptr, false);+ if(collection_meta == nlohmann::json::value_t::discarded) {+ LOG(ERROR) << "Error while parsing collection meta.";return Option<bool>(500, "Error while parsing collection meta.");}- const std::string & this_collection_name = collection_meta[Collection::COLLECTION_NAME_KEY].get<std::string>();- std::string collection_next_seq_id_str;- StoreStatus next_seq_id_status = store->get(Collection::get_next_seq_id_key(this_collection_name),- collection_next_seq_id_str);-- if(next_seq_id_status == StoreStatus::ERROR) {- return Option<bool>(500, "Error while fetching collection's next sequence ID from the disk for collection "- "`" + this_collection_name + "`");- }-- if(next_seq_id_status == StoreStatus::NOT_FOUND && next_coll_id_status == StoreStatus::FOUND) {- return Option<bool>(500, "Next collection id was found, but collection's next sequence ID is missing for "- "`" + this_collection_name + "`");- }-- uint32_t collection_next_seq_id = next_seq_id_status == StoreStatus::NOT_FOUND ? 0 :- StringUtils::deserialize_uint32_t(collection_next_seq_id_str);-- if(get_collection(this_collection_name) != nullptr) {- // To maintain idempotency, if the collection already exists in-memory, drop it from memory- LOG(WARNING) << "Dropping duplicate collection " << this_collection_name << " before loading it again.";- drop_collection(this_collection_name, false);- }-- std::unique_lock lock(mutex);- Collection* collection = init_collection(collection_meta, collection_next_seq_id);-- LOG(INFO) << "Loading collection " << collection->get_name();-- // initialize overrides- std::vector<std::string> collection_override_jsons;- store->scan_fill(Collection::get_override_key(this_collection_name, ""), collection_override_jsons);-- for(const auto & collection_override_json: collection_override_jsons) {- nlohmann::json collection_override = nlohmann::json::parse(collection_override_json);- override_t override;- auto parse_op = override_t::parse(collection_override, "", override);- if(parse_op.ok()) {- collection->add_override(override);- } else {- LOG(ERROR) << "Skipping loading of override: " << parse_op.error();- }- }-- // initialize synonyms- std::vector<std::string> collection_synonym_jsons;- store->scan_fill(Collection::get_synonym_key(this_collection_name, ""), collection_synonym_jsons);-- for(const auto & collection_synonym_json: collection_synonym_jsons) {- nlohmann::json collection_synonym = nlohmann::json::parse(collection_synonym_json);- synonym_t synonym(collection_synonym);- collection->add_synonym(synonym);- }-- // Fetch records from the store and re-create memory index- std::vector<std::string> documents;- const std::string seq_id_prefix = collection->get_seq_id_collection_prefix();-- rocksdb::Iterator* iter = store->scan(seq_id_prefix);- std::unique_ptr<rocksdb::Iterator> iter_guard(iter);-- std::vector<std::vector<index_record>> iter_batch;-- for(size_t i = 0; i < collection->get_num_memory_shards(); i++) {- iter_batch.push_back(std::vector<index_record>());- }-- size_t num_found_docs = 0;- size_t num_valid_docs = 0;- size_t num_indexed_docs = 0;-- auto begin = std::chrono::high_resolution_clock::now();-- while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {- num_found_docs++;- const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());-- nlohmann::json document;-- try {- document = nlohmann::json::parse(iter->value().ToString());- } catch(const std::exception& e) {- LOG(ERROR) << "JSON error: " << e.what();- return Option<bool>(false, "Bad JSON.");+ coll_meta_buffer.push_back(collection_meta);++ if(coll_meta_buffer.size() % PROC_COUNT == 0 || coll_index == collection_meta_jsons.size() - 1) {+ size_t num_processed = 0;+ std::mutex m_process;+ std::condition_variable cv_process;+ std::vector<Option<bool>> results;++ LOG(INFO) << "coll_meta_buffer.size(): " << coll_meta_buffer.size();++ for(const auto& buff_coll_meta: coll_meta_buffer) {+ thread_pool->enqueue([&buff_coll_meta, init_batch_size, &next_coll_id_status, &m_process, &cv_process,+ &num_processed, &results]() {+ Option<bool> res = load_collection(buff_coll_meta, init_batch_size, next_coll_id_status);+ std::unique_lock<std::mutex> lock(m_process);+ results.push_back(res);+ num_processed++;+ cv_process.notify_one();+ });}- auto dirty_values = DIRTY_VALUES::DROP;-- num_valid_docs++;-- iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document, CREATE, dirty_values));-- // Peek and check for last record right here so that we handle batched indexing correctly- // Without doing this, the "last batch" would have to be indexed outside the loop.- iter->Next();- bool last_record = !(iter->Valid() && iter->key().starts_with(seq_id_prefix));-- // batch must match atleast the number of shards- const size_t batch_size = std::max(init_batch_size, collection->get_num_memory_shards());-- if((num_valid_docs % batch_size == 0) || last_record) {- std::vector<size_t> indexed_counts;- indexed_counts.reserve(iter_batch.size());+ const size_t num_load_collections = coll_meta_buffer.size();- collection->par_index_in_memory(iter_batch, indexed_counts);+ std::unique_lock<std::mutex> lock_process(m_process);+ cv_process.wait(lock_process, [&](){+ return num_processed == num_load_collections;+ });- for(size_t i = 0; i < collection->get_num_memory_shards(); i++) {- size_t num_records = iter_batch[i].size();- size_t num_indexed = indexed_counts[i];-- if(num_indexed != num_records) {- const Option<std::string> & index_error_op = get_first_index_error(iter_batch[i]);- if(!index_error_op.ok()) {- return Option<bool>(false, index_error_op.get());- }- }- iter_batch[i].clear();- num_indexed_docs += num_indexed;+ for(size_t i = 0; i < num_load_collections; i++) {+ if(!results[i].ok()) {+ return results[i];}}- auto time_millis = std::chrono::duration_cast<std::chrono::milliseconds>(- std::chrono::high_resolution_clock::now() - begin).count();-- auto throttle_time_millis = uint64_t((LOAD_THROTTLE_PERCENT/100) * time_millis);-- if(throttle_time_millis != 0) {- // we throttle only when we have accumulated atleast 1 ms worth of throttling time- begin = std::chrono::high_resolution_clock::now();- std::this_thread::sleep_for(std::chrono::milliseconds(throttle_time_millis));- }+ coll_meta_buffer.clear();}-- add_to_collections(collection);- LOG(INFO) << "Indexed " << num_indexed_docs << "/" << num_found_docs- << " documents into collection " << collection->get_name();}std::string symlink_prefix_key = std::string(SYMLINK_PREFIX) + "_";@@ -274,6 +181,8 @@ Option<bool> CollectionManager::load(const size_t init_batch_size) {delete iter;+ LOG(INFO) << "Done loading all collections.";+return Option<bool>(true);}@@ -315,7 +224,6 @@ Option<Collection*> CollectionManager::create_collection(const std::string& nameconst std::string& default_sorting_field,const uint64_t created_at,const std::string& fallback_field_type) {- std::unique_lock lock(mutex);if(store->contains(Collection::get_meta_key(name))) {return Option<Collection*>(409, std::string("A collection with name `") + name + "` already exists.");@@ -366,7 +274,7 @@ Option<Collection*> CollectionManager::create_collection(const std::string& namereturn Option<Collection*>(new_collection);}-Collection* CollectionManager::get_collection_unafe(const std::string & collection_name) const {+Collection* CollectionManager::get_collection_unsafe(const std::string & collection_name) const {if(collections.count(collection_name) != 0) {return collections.at(collection_name);}@@ -384,7 +292,7 @@ Collection* CollectionManager::get_collection_unafe(const std::string & collectilocked_resource_view_t<Collection> CollectionManager::get_collection(const std::string & collection_name) const {std::shared_lock lock(mutex);- Collection* coll = get_collection_unafe(collection_name);+ Collection* coll = get_collection_unsafe(collection_name);return locked_resource_view_t<Collection>(mutex, coll);}@@ -415,9 +323,8 @@ std::vector<Collection*> CollectionManager::get_collections() const {}Option<nlohmann::json> CollectionManager::drop_collection(const std::string& collection_name, const bool remove_from_store) {- std::unique_lock lock(mutex);+ auto collection = get_collection(collection_name); // locked_resource_view_t- Collection* collection = get_collection_unafe(collection_name);if(collection == nullptr) {return Option<nlohmann::json>(404, "No collection with name `" + collection_name + "` found.");}@@ -445,7 +352,7 @@ Option<nlohmann::json> CollectionManager::drop_collection(const std::string& colcollections.erase(actual_coll_name);collection_id_names.erase(collection->get_collection_id());- delete collection;+ delete collection.get();return Option<nlohmann::json>(collection_json);}@@ -860,3 +767,158 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsofields, default_sorting_field, created_at,fallback_field_type);}++Option<bool> CollectionManager::load_collection(const nlohmann::json &collection_meta,+ const size_t init_batch_size,+ const StoreStatus& next_coll_id_status) {++ auto& cm = CollectionManager::get_instance();++ const std::string & this_collection_name = collection_meta[Collection::COLLECTION_NAME_KEY].get<std::string>();++ std::string collection_next_seq_id_str;+ StoreStatus next_seq_id_status = cm.store->get(Collection::get_next_seq_id_key(this_collection_name),+ collection_next_seq_id_str);++ if(next_seq_id_status == StoreStatus::ERROR) {+ LOG(ERROR) << "Error while fetching collection's next sequence ID";+ return Option<bool>(500, "Error while fetching collection's next sequence ID from the disk for collection "+ "`" + this_collection_name + "`");+ }++ if(next_seq_id_status == StoreStatus::NOT_FOUND && next_coll_id_status == StoreStatus::FOUND) {+ LOG(ERROR) << "collection's next sequence ID is missing";+ return Option<bool>(500, "Next collection id was found, but collection's next sequence ID is missing for "+ "`" + this_collection_name + "`");+ }++ uint32_t collection_next_seq_id = next_seq_id_status == StoreStatus::NOT_FOUND ? 0 :+ StringUtils::deserialize_uint32_t(collection_next_seq_id_str);++ {+ std::shared_lock lock(cm.mutex);+ Collection *existing_collection = cm.get_collection_unsafe(this_collection_name);++ if(existing_collection != nullptr) {+ // To maintain idempotency, if the collection already exists in-memory, drop it from memory+ LOG(WARNING) << "Dropping duplicate collection " << this_collection_name << " before loading it again.";+ lock.unlock();+ cm.drop_collection(this_collection_name, false);+ }+ }++ Collection* collection = init_collection(collection_meta, collection_next_seq_id, cm.store, 1.0f);++ LOG(INFO) << "Loading collection " << collection->get_name();++ // initialize overrides+ std::vector<std::string> collection_override_jsons;+ cm.store->scan_fill(Collection::get_override_key(this_collection_name, ""), collection_override_jsons);++ for(const auto & collection_override_json: collection_override_jsons) {+ nlohmann::json collection_override = nlohmann::json::parse(collection_override_json);+ override_t override;+ auto parse_op = override_t::parse(collection_override, "", override);+ if(parse_op.ok()) {+ collection->add_override(override);+ } else {+ LOG(ERROR) << "Skipping loading of override: " << parse_op.error();+ }+ }++ // initialize synonyms+ std::vector<std::string> collection_synonym_jsons;+ cm.store->scan_fill(Collection::get_synonym_key(this_collection_name, ""), collection_synonym_jsons);++ for(const auto & collection_synonym_json: collection_synonym_jsons) {+ nlohmann::json collection_synonym = nlohmann::json::parse(collection_synonym_json);+ synonym_t synonym(collection_synonym);+ collection->add_synonym(synonym);+ }++ // Fetch records from the store and re-create memory index+ std::vector<std::string> documents;+ const std::string seq_id_prefix = collection->get_seq_id_collection_prefix();++ rocksdb::Iterator* iter = cm.store->scan(seq_id_prefix);+ std::unique_ptr<rocksdb::Iterator> iter_guard(iter);++ std::vector<std::vector<index_record>> iter_batch;++ for(size_t i = 0; i < collection->get_num_memory_shards(); i++) {+ iter_batch.emplace_back(std::vector<index_record>());+ }++ size_t num_found_docs = 0;+ size_t num_valid_docs = 0;+ size_t num_indexed_docs = 0;++ auto begin = std::chrono::high_resolution_clock::now();++ while(iter->Valid() && iter->key().starts_with(seq_id_prefix)) {+ num_found_docs++;+ const uint32_t seq_id = Collection::get_seq_id_from_key(iter->key().ToString());++ nlohmann::json document;++ try {+ document = nlohmann::json::parse(iter->value().ToString());+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ return Option<bool>(false, "Bad JSON.");+ }++ auto dirty_values = DIRTY_VALUES::DROP;++ num_valid_docs++;++ iter_batch[seq_id % collection->get_num_memory_shards()].emplace_back(index_record(0, seq_id, document, CREATE, dirty_values));++ // Peek and check for last record right here so that we handle batched indexing correctly+ // Without doing this, the "last batch" would have to be indexed outside the loop.+ iter->Next();+ bool last_record = !(iter->Valid() && iter->key().starts_with(seq_id_prefix));++ // batch must match atleast the number of shards+ const size_t batch_size = std::max(init_batch_size, collection->get_num_memory_shards());++ if((num_valid_docs % batch_size == 0) || last_record) {+ std::vector<size_t> indexed_counts;+ indexed_counts.reserve(iter_batch.size());++ collection->par_index_in_memory(iter_batch, indexed_counts);++ for(size_t i = 0; i < collection->get_num_memory_shards(); i++) {+ size_t num_records = iter_batch[i].size();+ size_t num_indexed = indexed_counts[i];++ if(num_indexed != num_records) {+ const Option<std::string> & index_error_op = get_first_index_error(iter_batch[i]);+ if(!index_error_op.ok()) {+ return Option<bool>(false, index_error_op.get());+ }+ }+ iter_batch[i].clear();+ num_indexed_docs += num_indexed;+ }+ }++ auto time_millis = std::chrono::duration_cast<std::chrono::milliseconds>(+ std::chrono::high_resolution_clock::now() - begin).count();++ auto throttle_time_millis = uint64_t((cm.LOAD_THROTTLE_PERCENT/100) * time_millis);++ if(throttle_time_millis != 0) {+ // we throttle only when we have accumulated atleast 1 ms worth of throttling time+ begin = std::chrono::high_resolution_clock::now();+ std::this_thread::sleep_for(std::chrono::milliseconds(throttle_time_millis));+ }+ }++ cm.add_to_collections(collection);++ LOG(INFO) << "Indexed " << num_indexed_docs << "/" << num_found_docs+ << " documents into collection " << collection->get_name();++ return Option<bool>(true);+}diff --git a/src/cvt.cpp b/src/cvt.cppindex df377cae..ca8da972 100644--- a/src/cvt.cpp+++ b/src/cvt.cpp@@ -39,6 +39,8 @@ bool CVTrie::add(const char *key, const uint8_t length, void *value) {}}*/++ return true;}void *CVTrie::find(const char *key, const uint8_t length) {diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 72fd5d5b..bec4bbcb 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -41,9 +41,11 @@ protected:}virtual void TearDown() {- collectionManager.drop_collection("collection1");- collectionManager.dispose();- delete store;+ if(store != nullptr) {+ collectionManager.drop_collection("collection1");+ collectionManager.dispose();+ delete store;+ }}};@@ -102,7 +104,7 @@ TEST_F(CollectionManagerTest, ShouldInitCollection) {nlohmann::json::parse("{\"name\": \"foobar\", \"id\": 100, \"fields\": [{\"name\": \"org\", \"type\": ""\"string\", \"facet\": false}], \"default_sorting_field\": \"foo\"}");- Collection *collection = collectionManager.init_collection(collection_meta1, 100);+ Collection *collection = collectionManager.init_collection(collection_meta1, 100, store, 1.0f);ASSERT_EQ("foobar", collection->get_name());ASSERT_EQ(100, collection->get_collection_id());ASSERT_EQ(1, collection->get_fields().size());@@ -118,7 +120,7 @@ TEST_F(CollectionManagerTest, ShouldInitCollection) {"\"string\", \"facet\": false}], \"created_at\": 12345,""\"default_sorting_field\": \"foo\"}");- collection = collectionManager.init_collection(collection_meta2, 100);+ collection = collectionManager.init_collection(collection_meta2, 100, store, 1.0f);ASSERT_EQ(12345, collection->get_created_at());delete collection;@@ -524,6 +526,50 @@ TEST_F(CollectionManagerTest, Symlinking) {delete new_store;}+TEST_F(CollectionManagerTest, LoadMultipleCollections) {+ // to prevent fixture tear down from running as we are fudging with CollectionManager singleton+ delete store;+ store = nullptr;++ CollectionManager & cmanager = CollectionManager::get_instance();+ std::string state_dir_path = "/tmp/typesense_test/cmanager_test_db";+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());+ Store *new_store = new Store(state_dir_path);+ cmanager.init(new_store, 1.0, "auth_key");+ cmanager.load();++ for(size_t i = 0; i < 100; i++) {+ auto schema = {+ field("title", field_types::STRING, false),+ field("starring", field_types::STRING, false),+ field("cast", field_types::STRING_ARRAY, true, true),+ field(".*_year", field_types::INT32, true, true),+ field("location", field_types::GEOPOINT, false, true, true, 14),+ field("points", field_types::INT32, false)+ };++ cmanager.create_collection("collection" + std::to_string(i), 4, schema, "points").get();+ }++ ASSERT_EQ(100, cmanager.get_collections().size());++ cmanager.dispose();+ delete new_store;++ new_store = new Store(state_dir_path);+ cmanager.init(new_store, 1.0, "auth_key");+ cmanager.load();++ ASSERT_EQ(100, cmanager.get_collections().size());++ for(size_t i = 0; i < 100; i++) {+ collectionManager.drop_collection("collection" + std::to_string(i));+ }++ collectionManager.dispose();+ delete new_store;+}+TEST_F(CollectionManagerTest, ParseSortByClause) {std::vector<sort_by> sort_fields;bool sort_by_parsed = CollectionManager::parse_sort_by_str("points:desc,loc(24.56,10.45):ASC", sort_fields);