Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix bug in restoring a collection having reference.

Harpreet Sangarc30df4a69a34
diff --git a/include/collection.h b/include/collection.h
index 40b6c1df..f694a0fb 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -590,6 +590,10 @@ public:
bool is_referenced_in(const std::string& collection_name) const;
+ void add_referenced_in(const reference_pair& pair);
+
+ void add_referenced_in(const std::string& collection_name, const std::string& field_name);
+
Option<std::string> get_reference_field(const std::string& collection_name) const;
Option<uint32_t> get_sort_indexed_field_value(const std::string& field_name, const uint32_t& seq_id) const;
diff --git a/src/collection.cpp b/src/collection.cpp
index e67a3f83..1c0ccb40 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -4033,6 +4033,7 @@ SynonymIndex* Collection::get_synonym_index() {
}
spp::sparse_hash_map<std::string, reference_pair> Collection::get_reference_fields() {
+ std::shared_lock lock(mutex);
return reference_fields;
}
@@ -5072,7 +5073,7 @@ Index* Collection::init_index() {
auto ref_coll = collectionManager.get_collection(ref_coll_name);
if (ref_coll != nullptr) {
// Passing reference helper field helps perform operation on doc_id instead of field value.
- ref_coll->referenced_in.emplace(name, field.name + REFERENCE_HELPER_FIELD_SUFFIX);
+ ref_coll->add_referenced_in(name, field.name + REFERENCE_HELPER_FIELD_SUFFIX);
}
}
}
@@ -5448,6 +5449,15 @@ bool Collection::is_referenced_in(const std::string& collection_name) const {
return referenced_in.count(collection_name) > 0;
}
+void Collection::add_referenced_in(const reference_pair& pair) {
+ add_referenced_in(pair.collection, pair.field);
+}
+
+void Collection::add_referenced_in(const std::string& collection_name, const std::string& field_name) {
+ std::shared_lock lock(mutex);
+ referenced_in.emplace(collection_name, field_name);
+}
+
Option<std::string> Collection::get_reference_field(const std::string& collection_name) const {
std::shared_lock lock(mutex);
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 82c26a81..328520ca 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -219,6 +219,8 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
ThreadPool loading_pool(collection_batch_size);
size_t num_processed = 0;
+ // Collection name -> Referenced in
+ std::map<std::string, std::vector<reference_pair>> referenced_ins = {};
std::mutex m_process;
std::condition_variable cv_process;
@@ -232,7 +234,8 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
auto captured_store = store;
loading_pool.enqueue([captured_store, num_collections, collection_meta, document_batch_size,
- &m_process, &cv_process, &num_processed, &next_coll_id_status, quit = quit]() {
+ &m_process, &cv_process, &num_processed, &next_coll_id_status, quit = quit,
+ &referenced_ins]() {
//auto begin = std::chrono::high_resolution_clock::now();
Option<bool> res = load_collection(collection_meta, document_batch_size, next_coll_id_status, *quit);
@@ -249,6 +252,20 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
std::unique_lock<std::mutex> lock(m_process);
num_processed++;
+
+ auto& cm = CollectionManager::get_instance();
+ auto const& collection_name = collection_meta.at("name");
+ auto collection = cm.get_collection(collection_name);
+ if (collection != nullptr) {
+ for (const auto &item: collection->get_reference_fields()) {
+ auto const& ref_coll_name = item.second.collection;
+ if (referenced_ins.count(ref_coll_name) == 0) {
+ referenced_ins[ref_coll_name] = {};
+ }
+ auto const field_name = item.first + Collection::REFERENCE_HELPER_FIELD_SUFFIX;
+ referenced_ins.at(ref_coll_name).emplace_back(reference_pair{collection_name, field_name});
+ }
+ }
cv_process.notify_one();
size_t progress_modulo = std::max<size_t>(1, (num_collections / 10)); // every 10%
@@ -264,6 +281,17 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
return num_processed == num_collections;
});
+ // Initialize references
+ for (const auto &item: referenced_ins) {
+ auto& cm = CollectionManager::get_instance();
+ auto collection = cm.get_collection(item.first);
+ if (collection != nullptr) {
+ for (const auto &reference_pair: item.second) {
+ collection->add_referenced_in(reference_pair);
+ }
+ }
+ }
+
// load aliases
std::string symlink_prefix_key = std::string(SYMLINK_PREFIX) + "_";
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index d54b8bc4..af3263fa 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -86,6 +86,10 @@ TEST_F(CollectionManagerTest, CollectionCreation) {
ASSERT_EQ("points", collection1->get_default_sorting_field());
ASSERT_EQ(false, schema.at("not_stored").index);
+ ASSERT_EQ(1, collection1->get_reference_fields().size());
+ ASSERT_EQ("Products", collection1->get_reference_fields().at("product_id").collection);
+ ASSERT_EQ("product_id", collection1->get_reference_fields().at("product_id").field);
+
// check storage as well
rocksdb::Iterator* it = store->get_iterator();
size_t num_keys = 0;
@@ -486,6 +490,10 @@ TEST_F(CollectionManagerTest, RestoreRecordsOnRestart) {
ASSERT_EQ(schema.size(), collection1->get_schema().size());
ASSERT_EQ("points", collection1->get_default_sorting_field());
+ ASSERT_EQ(1, collection1->get_reference_fields().size());
+ ASSERT_EQ("Products", collection1->get_reference_fields().at("product_id").collection);
+ ASSERT_EQ("product_id", collection1->get_reference_fields().at("product_id").field);
+
auto restored_schema = collection1->get_schema();
ASSERT_EQ(true, restored_schema.at("cast").optional);
ASSERT_EQ(true, restored_schema.at("cast").facet);