Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor a couple of methods in collection manager.
Kishore Nallanb2f0ca495df9
diff --git a/TODO.md b/TODO.mdindex 6fcb2190..e23bbf8f 100644--- a/TODO.md+++ b/TODO.md@@ -58,20 +58,23 @@- ~~Prefix-search strings should not be null terminated~~- ~~sort results by float field~~- ~~json::parse must be wrapped in try catch~~-- Collection Manager collections map should store plain collection name-- init_collection of Collection manager should probably take seq_id as param+- ~~Collection Manager collections map should store plain collection name~~+- ~~init_collection of Collection manager should probably take seq_id as param~~+- node score should be int32, no longer uint16 like in document struct+- Proper logging- https support- Validate before string to int conversion in the http api layer- When field of "id" but not string, what happens?- Typo in prefix search-- node score should be int32, no longer uint16 like in document struct- test for num_documents- test for string filter comparison: title < "foo"- test for token ranking on float field- test for float int field deletion during doc deletion+- Test for sorted_array::indexOf when length is 0+- Test for snippets+- Test for pagination- > INT32_MAX validation for float field- art bool support-- Proper logging- Add docs/explanation around ranking calc- Use rocksdb batch put for atomic insertion- Query token ids should match query token ordering@@ -79,9 +82,6 @@- Group results by field- Handle store-get() not finding a key- Delete using range: https://github.com/facebook/rocksdb/wiki/Delete-A-Range-Of-Keys-- Test for sorted_array::indexOf when length is 0-- Test for snippets-- Test for pagination- Test for string utils- Prevent string copy during indexing- clean special chars before indexingdiff --git a/include/collection_manager.h b/include/collection_manager.hindex ea1a6b8e..479c1a0f 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -47,7 +47,7 @@ public:// frees in-memory data structures when server is shutdown - helps us run a memory leak detecter properlyvoid dispose();- Option<Collection*> init_collection(const std::string & collection_meta_json);+ Collection* init_collection(const nlohmann::json & collection_meta, const uint32_t collection_next_seq_id);void add_to_collections(Collection* collection);diff --git a/include/replicator.h b/include/replicator.hindex 514368fb..a44c4c1b 100644--- a/include/replicator.h+++ b/include/replicator.h@@ -130,17 +130,18 @@ public:}if(replication_event->type == "ADD_COLLECTION_META") {- CollectionManager & collection_manager = CollectionManager::get_instance();- Option<Collection*> collection_op = collection_manager.init_collection(replication_event->value);-- if(!collection_op.ok()) {- std::cerr << "Failed to initialize collection. Error: " << collection_op.error() << std::endl;+ nlohmann::json collection_meta;+ try {+ collection_meta = nlohmann::json::parse(replication_event->value);+ } catch(...) {+ std::cerr << "Failed to parse collection meta JSON." << std::endl;std::cerr << "Replication event value: " << replication_event->value << std::endl;delete replication_event;exit(1);}- Collection* collection = collection_op.get();+ CollectionManager & collection_manager = CollectionManager::get_instance();+ Collection* collection = collection_manager.init_collection(replication_event->value, 0);collection_manager.add_to_collections(collection);}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex ddfa19a0..a1655486 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -8,15 +8,8 @@ CollectionManager::CollectionManager() {}-Option<Collection*> CollectionManager::init_collection(const std::string & collection_meta_json) {- nlohmann::json collection_meta;-- try {- collection_meta = nlohmann::json::parse(collection_meta_json);- } catch(...) {- return Option<Collection*>(500, "Error while parsing collection meta.");- }-+Collection* CollectionManager::init_collection(const nlohmann::json & collection_meta,+ const uint32_t collection_next_seq_id) {std::string this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();std::vector<field> search_fields;@@ -33,11 +26,6 @@ Option<Collection*> CollectionManager::init_collection(const std::string & collefacet_fields.push_back({it.value()[fields::name], it.value()[fields::type]});}- std::string collection_next_seq_id_str;- store->get(Collection::get_next_seq_id_key(this_collection_name), collection_next_seq_id_str);- uint32_t collection_next_seq_id = collection_next_seq_id_str.size() == 0 ? 0 :- (const uint32_t) std::stoi(collection_next_seq_id_str);-std::vector<field> collection_sort_fields;nlohmann::json sort_fields_map = collection_meta[COLLECTION_SORT_FIELDS_KEY];@@ -56,11 +44,11 @@ Option<Collection*> CollectionManager::init_collection(const std::string & collecollection_sort_fields,token_ranking_field);- return Option<Collection*>(collection);+ return collection;}void CollectionManager::add_to_collections(Collection* collection) {- collections.emplace(Collection::get_meta_key(collection->get_name()), collection);+ collections.emplace(collection->get_name(), collection);collection_id_names.emplace(collection->get_collection_id(), collection->get_name());}@@ -80,13 +68,21 @@ Option<bool> CollectionManager::init(Store *store, const std::string & auth_key)store->scan_fill(Collection::COLLECTION_META_PREFIX, collection_meta_jsons);for(auto & collection_meta_json: collection_meta_jsons) {- Option<Collection*> collection_op = init_collection(collection_meta_json);+ nlohmann::json collection_meta;- if(!collection_op.ok()) {- return Option<bool>(collection_op.code(), collection_op.error());+ try {+ collection_meta = nlohmann::json::parse(collection_meta_json);+ } catch(...) {+ return Option<bool>(500, "Error while parsing collection meta.");}- Collection* collection = collection_op.get();+ const std::string & this_collection_name = collection_meta[COLLECTION_NAME_KEY].get<std::string>();+ std::string collection_next_seq_id_str;+ store->get(Collection::get_next_seq_id_key(this_collection_name), collection_next_seq_id_str);+ uint32_t collection_next_seq_id = collection_next_seq_id_str.size() == 0 ? 0 :+ (const uint32_t) std::stoi(collection_next_seq_id_str);++ Collection* collection = init_collection(collection_meta, collection_next_seq_id);// Fetch records from the store and re-create memory indexstd::vector<std::string> documents;@@ -187,8 +183,8 @@ Option<Collection*> CollectionManager::create_collection(std::string name, const}Collection* CollectionManager::get_collection(const std::string & collection_name) {- if(collections.count(Collection::get_meta_key(collection_name)) != 0) {- return collections.at(Collection::get_meta_key(collection_name));+ if(collections.count(collection_name) != 0) {+ return collections.at(collection_name);}return nullptr;@@ -238,7 +234,7 @@ Option<bool> CollectionManager::drop_collection(std::string collection_name, constore->remove(Collection::get_meta_key(collection_name));}- collections.erase(Collection::get_meta_key(collection_name));+ collections.erase(collection_name);collection_id_names.erase(collection->get_collection_id());delete collection;