Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Allow cloning of collection & assets from a reference collection.
Kishore Nallan4c6eac9840bb
diff --git a/include/collection.h b/include/collection.hindex 30eed5f6..c6ab24af 100644--- a/include/collection.h+++ b/include/collection.h@@ -388,6 +388,8 @@ public:void synonym_reduction(const std::vector<std::string>& tokens,std::vector<std::vector<std::string>>& results) const;+ SynonymIndex* get_synonym_index();+// highlight opsstatic void highlight_text(const string& highlight_start_tag, const string& highlight_end_tag,diff --git a/include/collection_manager.h b/include/collection_manager.hindex c0e1dbd0..78feeaa2 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -118,6 +118,8 @@ public:const StoreStatus& next_coll_id_status,const std::atomic<bool>& quit);+ Option<Collection*> clone_collection(const std::string& existing_name, const nlohmann::json& req_json);+void add_to_collections(Collection* collection);std::vector<Collection*> get_collections() const;diff --git a/src/collection.cpp b/src/collection.cppindex 4cc94ae7..03c585cc 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2637,19 +2637,19 @@ Option<bool> Collection::parse_pinned_hits(const std::string& pinned_hits_str,}if(index == 0) {- return Option<bool>(false, "Pinned hits are not in expected format.");+ return Option<bool>(400, "Pinned hits are not in expected format.");}std::string pinned_id = pinned_hits_part.substr(0, index);std::string pinned_pos = pinned_hits_part.substr(index+1);if(!StringUtils::is_positive_integer(pinned_pos)) {- return Option<bool>(false, "Pinned hits are not in expected format.");+ return Option<bool>(400, "Pinned hits are not in expected format.");}int position = std::stoi(pinned_pos);if(position == 0) {- return Option<bool>(false, "Pinned hits must start from position 1.");+ return Option<bool>(400, "Pinned hits must start from position 1.");}pinned_hits[position].emplace_back(pinned_id);@@ -2692,6 +2692,10 @@ spp::sparse_hash_map<std::string, synonym_t> Collection::get_synonyms() {return synonym_index->get_synonyms();}+SynonymIndex* Collection::get_synonym_index() {+ return synonym_index;+}+Option<bool> Collection::persist_collection_meta() {std::string coll_meta_json;StoreStatus status = store->get(Collection::get_meta_key(name), coll_meta_json);@@ -2777,7 +2781,7 @@ Option<bool> Collection::batch_alter_data(const std::unordered_map<std::string,try {document = nlohmann::json::parse(iter->value().ToString());} catch(const std::exception& e) {- return Option<bool>(false, "Bad JSON in document: " + document.dump(-1, ' ', false,+ return Option<bool>(400, "Bad JSON in document: " + document.dump(-1, ' ', false,nlohmann::detail::error_handler_t::ignore));}@@ -3070,7 +3074,7 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,try {document = nlohmann::json::parse(iter->value().ToString());} catch(const std::exception& e) {- return Option<bool>(false, "Bad JSON in document: " + document.dump(-1, ' ', false,+ return Option<bool>(400, "Bad JSON in document: " + document.dump(-1, ' ', false,nlohmann::detail::error_handler_t::ignore));}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 8b293309..bb3b7801 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1163,7 +1163,7 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectiondocument = nlohmann::json::parse(iter->value().ToString());} catch(const std::exception& e) {LOG(ERROR) << "JSON error: " << e.what();- return Option<bool>(false, "Bad JSON.");+ return Option<bool>(400, "Bad JSON.");}auto dirty_values = DIRTY_VALUES::DROP;@@ -1185,7 +1185,7 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectionif(num_indexed != num_records) {const Option<std::string> & index_error_op = get_first_index_error(index_records);if(!index_error_op.ok()) {- return Option<bool>(false, index_error_op.get());+ return Option<bool>(400, index_error_op.get());}}@@ -1261,3 +1261,62 @@ Option<bool> CollectionManager::delete_preset(const string& preset_name) {preset_configs.erase(preset_name);return Option<bool>(true);}++Option<Collection*> CollectionManager::clone_collection(const string& existing_name, const nlohmann::json& req_json) {+ std::shared_lock lock(mutex);++ if(collections.count(existing_name) == 0) {+ return Option<Collection*>(400, "Collection with name `" + existing_name + "` not found.");+ }++ if(req_json.count("name") == 0 || !req_json["name"].is_string()) {+ return Option<Collection*>(400, "Collection name must be provided.");+ }++ const std::string& new_name = req_json["name"].get<std::string>();++ if(collections.count(new_name) != 0) {+ return Option<Collection*>(400, "Collection with name `" + new_name + "` already exists.");+ }++ Collection* existing_coll = collections[existing_name];++ std::vector<std::string> symbols_to_index;+ std::vector<std::string> token_separators;++ for(auto c: existing_coll->get_symbols_to_index()) {+ symbols_to_index.emplace_back(1, c);+ }++ for(auto c: existing_coll->get_token_separators()) {+ token_separators.emplace_back(1, c);+ }++ lock.unlock();++ auto coll_create_op = create_collection(new_name, DEFAULT_NUM_MEMORY_SHARDS, existing_coll->get_fields(),+ existing_coll->get_default_sorting_field(), static_cast<uint64_t>(std::time(nullptr)),+ existing_coll->get_fallback_field_type(), symbols_to_index, token_separators);++ lock.lock();++ if(!coll_create_op.ok()) {+ return Option<Collection*>(coll_create_op.code(), coll_create_op.error());+ }++ Collection* new_coll = coll_create_op.get();++ // copy synonyms+ auto synonyms = existing_coll->get_synonyms();+ for(const auto& synonym: synonyms) {+ new_coll->get_synonym_index()->add_synonym(new_name, synonym.second);+ }++ // copy overrides+ auto overrides = existing_coll->get_overrides();+ for(const auto& override: overrides) {+ new_coll->add_override(override.second);+ }++ return Option<Collection*>(new_coll);+}diff --git a/src/core_api.cpp b/src/core_api.cppindex 60f25f75..c13f9ed7 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -165,8 +165,12 @@ bool post_create_collection(const std::shared_ptr<http_req>& req, const std::shareturn false;}- CollectionManager & collectionManager = CollectionManager::get_instance();- const Option<Collection*> & collection_op = collectionManager.create_collection(req_json);+ const std::string SRC_COLL_NAME = "src_name";++ CollectionManager& collectionManager = CollectionManager::get_instance();+ const Option<Collection*> &collection_op = req->params.count(SRC_COLL_NAME) != 0 ?+ collectionManager.clone_collection(req->params[SRC_COLL_NAME], req_json) :+ CollectionManager::create_collection(req_json);if(collection_op.ok()) {nlohmann::json json_response = collection_op.get()->get_summary_json();diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex b8faad9e..2b40068f 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -826,3 +826,60 @@ TEST_F(CollectionManagerTest, Presets) {preset_op = collectionManager.get_preset("preset1", preset);ASSERT_TRUE(preset_op.ok());}++TEST_F(CollectionManagerTest, CloneCollection) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string"}+ ],+ "symbols_to_index":["+"],+ "token_separators":["-", "?"]+ })"_json;++ auto create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(create_op.ok());+ auto coll1 = create_op.get();++ nlohmann::json synonym1 = R"({+ "id": "ipod-synonyms",+ "synonyms": ["ipod", "i pod", "pod"]+ })"_json;++ ASSERT_TRUE(coll1->add_synonym(synonym1).ok());++ nlohmann::json override_json = {+ {"id", "dynamic-cat-filter"},+ {+ "rule", {+ {"query", "{categories}"},+ {"match", override_t::MATCH_EXACT}+ }+ },+ {"remove_matched_tokens", true},+ {"filter_by", "category: {categories}"}+ };++ override_t override;+ auto op = override_t::parse(override_json, "dynamic-cat-filter", override);+ ASSERT_TRUE(op.ok());+ coll1->add_override(override);++ nlohmann::json req = R"({"name": "coll2"})"_json;+ collectionManager.clone_collection("coll1", req);++ auto coll2 = collectionManager.get_collection_unsafe("coll2");+ ASSERT_FALSE(coll2 == nullptr);+ ASSERT_EQ("coll2", coll2->get_name());+ ASSERT_EQ(1, coll2->get_fields().size());+ ASSERT_EQ(1, coll2->get_synonyms().size());+ ASSERT_EQ(1, coll2->get_overrides().size());+ ASSERT_EQ("", coll2->get_fallback_field_type());++ ASSERT_EQ(1, coll2->get_symbols_to_index().size());+ ASSERT_EQ(2, coll2->get_token_separators().size());++ ASSERT_EQ('+', coll2->get_symbols_to_index().at(0));+ ASSERT_EQ('-', coll2->get_token_separators().at(0));+ ASSERT_EQ('?', coll2->get_token_separators().at(1));+}