Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix bug in restoring a collection having reference.
Harpreet Sangarc30df4a69a34
diff --git a/include/collection.h b/include/collection.hindex 40b6c1df..f694a0fb 100644--- a/include/collection.h+++ b/include/collection.h@@ -590,6 +590,10 @@ public:bool is_referenced_in(const std::string& collection_name) const;+ void add_referenced_in(const reference_pair& pair);++ void add_referenced_in(const std::string& collection_name, const std::string& field_name);+Option<std::string> get_reference_field(const std::string& collection_name) const;Option<uint32_t> get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const;diff --git a/src/collection.cpp b/src/collection.cppindex e67a3f83..1c0ccb40 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -4033,6 +4033,7 @@ SynonymIndex* Collection::get_synonym_index() {}spp::sparse_hash_map<std::string, reference_pair> Collection::get_reference_fields() {+ std::shared_lock lock(mutex);return reference_fields;}@@ -5072,7 +5073,7 @@ Index* Collection::init_index() {auto ref_coll = collectionManager.get_collection(ref_coll_name);if (ref_coll != nullptr) {// Passing reference helper field helps perform operation on doc_id instead of field value.- ref_coll->referenced_in.emplace(name, field.name + REFERENCE_HELPER_FIELD_SUFFIX);+ ref_coll->add_referenced_in(name, field.name + REFERENCE_HELPER_FIELD_SUFFIX);}}}@@ -5448,6 +5449,15 @@ bool Collection::is_referenced_in(const std::string& collection_name) const {return referenced_in.count(collection_name) > 0;}+void Collection::add_referenced_in(const reference_pair& pair) {+ add_referenced_in(pair.collection, pair.field);+}++void Collection::add_referenced_in(const std::string& collection_name, const std::string& field_name) {+ std::shared_lock lock(mutex);+ referenced_in.emplace(collection_name, field_name);+}+Option<std::string> Collection::get_reference_field(const std::string& collection_name) const {std::shared_lock lock(mutex);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 82c26a81..328520ca 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -219,6 +219,8 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sThreadPool loading_pool(collection_batch_size);size_t num_processed = 0;+ // Collection name -> Referenced in+ std::map<std::string, std::vector<reference_pair>> referenced_ins = {};std::mutex m_process;std::condition_variable cv_process;@@ -232,7 +234,8 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sauto captured_store = store;loading_pool.enqueue([captured_store, num_collections, collection_meta, document_batch_size,- &m_process, &cv_process, &num_processed, &next_coll_id_status, quit = quit]() {+ &m_process, &cv_process, &num_processed, &next_coll_id_status, quit = quit,+ &referenced_ins]() {//auto begin = std::chrono::high_resolution_clock::now();Option<bool> res = load_collection(collection_meta, document_batch_size, next_coll_id_status, *quit);@@ -249,6 +252,20 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sstd::unique_lock<std::mutex> lock(m_process);num_processed++;++ auto& cm = CollectionManager::get_instance();+ auto const& collection_name = collection_meta.at("name");+ auto collection = cm.get_collection(collection_name);+ if (collection != nullptr) {+ for (const auto &item: collection->get_reference_fields()) {+ auto const& ref_coll_name = item.second.collection;+ if (referenced_ins.count(ref_coll_name) == 0) {+ referenced_ins[ref_coll_name] = {};+ }+ auto const field_name = item.first + Collection::REFERENCE_HELPER_FIELD_SUFFIX;+ referenced_ins.at(ref_coll_name).emplace_back(reference_pair{collection_name, field_name});+ }+ }cv_process.notify_one();size_t progress_modulo = std::max<size_t>(1, (num_collections / 10)); // every 10%@@ -264,6 +281,17 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sreturn num_processed == num_collections;});+ // Initialize references+ for (const auto &item: referenced_ins) {+ auto& cm = CollectionManager::get_instance();+ auto collection = cm.get_collection(item.first);+ if (collection != nullptr) {+ for (const auto &reference_pair: item.second) {+ collection->add_referenced_in(reference_pair);+ }+ }+ }+// load aliasesstd::string symlink_prefix_key = std::string(SYMLINK_PREFIX) + "_";diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex d54b8bc4..af3263fa 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -86,6 +86,10 @@ TEST_F(CollectionManagerTest, CollectionCreation) {ASSERT_EQ("points", collection1->get_default_sorting_field());ASSERT_EQ(false, schema.at("not_stored").index);+ ASSERT_EQ(1, collection1->get_reference_fields().size());+ ASSERT_EQ("Products", collection1->get_reference_fields().at("product_id").collection);+ ASSERT_EQ("product_id", collection1->get_reference_fields().at("product_id").field);+// check storage as wellrocksdb::Iterator* it = store->get_iterator();size_t num_keys = 0;@@ -486,6 +490,10 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {ASSERT_EQ(schema.size(), collection1->get_schema().size());ASSERT_EQ("points", collection1->get_default_sorting_field());+ ASSERT_EQ(1, collection1->get_reference_fields().size());+ ASSERT_EQ("Products", collection1->get_reference_fields().at("product_id").collection);+ ASSERT_EQ("product_id", collection1->get_reference_fields().at("product_id").field);+auto restored_schema = collection1->get_schema();ASSERT_EQ(true, restored_schema.at("cast").optional);ASSERT_EQ(true, restored_schema.at("cast").facet);