Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
facilitate to add metadata while creating collection (#1491)
Krunal Gandhie51055712c5e
diff --git a/include/collection.h b/include/collection.hindex 4f313729..30b78a00 100644--- a/include/collection.h+++ b/include/collection.h@@ -372,6 +372,8 @@ public:static constexpr const char* COLLECTION_SEPARATORS = "token_separators";static constexpr const char* COLLECTION_VOICE_QUERY_MODEL = "voice_query_model";+ static constexpr const char* COLLECTION_METADATA = "metadata";+// methodsCollection() = delete;diff --git a/include/collection_manager.h b/include/collection_manager.hindex 11df6703..a88e3f7b 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -169,7 +169,8 @@ public:const std::string& fallback_field_type = "",const std::vector<std::string>& symbols_to_index = {},const std::vector<std::string>& token_separators = {},- const bool enable_nested_fields = false, std::shared_ptr<VQModel> model = nullptr);+ const bool enable_nested_fields = false, std::shared_ptr<VQModel> model = nullptr,+ const nlohmann::json& metadata = {});locked_resource_view_t<Collection> get_collection(const std::string & collection_name) const;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex dbd2b97c..7acc90bb 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -442,7 +442,8 @@ Option<Collection*> CollectionManager::create_collection(const std::string& nameconst std::string& fallback_field_type,const std::vector<std::string>& symbols_to_index,const std::vector<std::string>& token_separators,- const bool enable_nested_fields, std::shared_ptr<VQModel> model) {+ const bool enable_nested_fields, std::shared_ptr<VQModel> model,+ const nlohmann::json& metadata) {std::unique_lock lock(mutex);if(store->contains(Collection::get_meta_key(name))) {@@ -485,6 +486,10 @@ Option<Collection*> CollectionManager::create_collection(const std::string& namecollection_meta[Collection::COLLECTION_VOICE_QUERY_MODEL]["model_name"] = model->get_model_name();}+ if(!metadata.empty()) {+ collection_meta[Collection::COLLECTION_METADATA] = metadata;+ }+rocksdb::WriteBatch batch;batch.Put(Collection::get_next_seq_id_key(name), StringUtils::serialize_uint32_t(0));batch.Put(Collection::get_meta_key(name), collection_meta.dump());@@ -1869,6 +1874,7 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsoconst char* TOKEN_SEPARATORS = "token_separators";const char* ENABLE_NESTED_FIELDS = "enable_nested_fields";const char* DEFAULT_SORTING_FIELD = "default_sorting_field";+ const char* METADATA = "metadata";// validate presence of mandatory fields@@ -1949,6 +1955,14 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso"`name`, `type` and optionally, `facet` properties.");}+ if(req_json.count(METADATA) != 0) {+ if(!req_json[METADATA].is_object()) {+ return Option<Collection *>(400, "The `metadata` value should be an object.");+ }+ } else {+ req_json[METADATA] = {};+ }+const std::string& default_sorting_field = req_json[DEFAULT_SORTING_FIELD].get<std::string>();if(default_sorting_field == "id") {@@ -1998,7 +2012,8 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jsofallback_field_type,req_json[SYMBOLS_TO_INDEX],req_json[TOKEN_SEPARATORS],- req_json[ENABLE_NESTED_FIELDS], model);+ req_json[ENABLE_NESTED_FIELDS],+ model, req_json[METADATA]);}Option<bool> CollectionManager::load_collection(const nlohmann::json &collection_meta,@@ -2316,4 +2331,4 @@ void CollectionManager::process_embedding_field_delete(const std::string& model_EmbedderManager::get_instance().delete_text_embedder(model_name);EmbedderManager::get_instance().delete_image_embedder(model_name);}-}+}\ No newline at end of filediff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 0a4ef186..08f8400e 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -1728,4 +1728,130 @@ TEST_F(CollectionManagerTest, ReferencedInBacklog) {get_reference_field_op = create_op.get()->get_referenced_in_field_with_lock("foo");ASSERT_FALSE(get_reference_field_op.ok());ASSERT_EQ("Could not find any field in `Products` referencing the collection `foo`.", get_reference_field_op.error());+}++TEST_F(CollectionManagerTest, CollectionCreationWithMetadata) {+ CollectionManager & collectionManager3 = CollectionManager::get_instance();++ nlohmann::json schema1 = R"({+ "name": "collection_meta",+ "enable_nested_fields": true,+ "fields": [+ {"name": "value.color", "type": "string", "optional": false, "facet": true },+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }+ ],+ "metadata": "abc"+ })"_json;++ auto op = collectionManager.create_collection(schema1);+ ASSERT_FALSE(op.ok());+ ASSERT_EQ("The `metadata` value should be an object.", op.error());++ nlohmann::json schema2 = R"({+ "name": "collection_meta",+ "enable_nested_fields": true,+ "fields": [+ {"name": "value.color", "type": "string", "optional": false, "facet": true },+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }+ ],+ "metadata": {+ "batch_job":"",+ "indexed_from":"2023-04-20T00:00:00.000Z",+ "total_docs": 0+ }+ })"_json;++ op = collectionManager.create_collection(schema2);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();+++ std::string collection_meta_json;+ nlohmann::json collection_meta;+ std::string next_seq_id;+ std::string next_collection_id;++ store->get(Collection::get_meta_key("collection_meta"), collection_meta_json);+ store->get(Collection::get_next_seq_id_key("collection_meta"), next_seq_id);++ //LOG(INFO) << collection_meta_json;++ nlohmann::json expected_meta_json = R"(+ {+ "created_at":1705482381,+ "default_sorting_field":"",+ "enable_nested_fields":true,+ "fallback_field_type":"",+ "fields":[+ {+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.color",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":false,+ "store":true,+ "type":"string"+ },+ {+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.r",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":true,+ "store":true,+ "type":"int32"+ },{+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.g",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":true,+ "store":true,+ "type":"int32"+ },{+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.b",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":true,+ "store":true,+ "type":"int32"+ }+ ],+ "id":1,+ "metadata":{+ "batch_job":"",+ "indexed_from":"2023-04-20T00:00:00.000Z",+ "total_docs":0+ },+ "name":"collection_meta",+ "num_memory_shards":4,+ "symbols_to_index":[],+ "token_separators":[]+ })"_json;++ auto actual_json = nlohmann::json::parse(collection_meta_json);+ expected_meta_json["created_at"] = actual_json["created_at"];++ ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());}\ No newline at end of file