Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
support updation of collection meta (#1779)
Krunal Gandhi629def6f0f04
diff --git a/include/collection.h b/include/collection.hindex 834e7b3c..ca14a4d1 100644--- a/include/collection.h+++ b/include/collection.h@@ -432,6 +432,8 @@ public:std::string get_default_sorting_field();+ void update_metadata(const nlohmann::json& meta);+static Option<bool> add_reference_helper_fields(nlohmann::json& document, const tsl::htrie_map<char, field>& schema,const spp::sparse_hash_map<std::string, reference_pair>& reference_fields,tsl::htrie_set<char>& object_reference_helper_fields,diff --git a/include/collection_manager.h b/include/collection_manager.hindex 106b3caf..4a2b5c77 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -245,4 +245,6 @@ public:std::unordered_set<std::string> get_collection_references(const std::string& coll_name);bool is_valid_api_key_collection(const std::vector<std::string>& api_key_collections, Collection* coll) const;++ bool update_collection_metadata(const std::string& collection, const nlohmann::json& metadata);};diff --git a/src/collection.cpp b/src/collection.cppindex 81e6f4f9..3c282e67 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -4748,6 +4748,11 @@ std::string Collection::get_default_sorting_field() {return default_sorting_field;}+void Collection::update_metadata(const nlohmann::json& meta) {+ std::shared_lock lock(mutex);+ metadata = meta;+}+Option<bool> Collection::get_document_from_store(const uint32_t& seq_id,nlohmann::json& document, bool raw_doc) const {return get_document_from_store(get_seq_id_key(seq_id), document, raw_doc);@@ -5645,13 +5650,14 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,}if(schema_changes.size() != 1) {- return Option<bool>(400, "Only `fields` can be updated at the moment.");+ return Option<bool>(400, "Only `fields` and `metadata` can be updated at the moment.");}const std::string err_msg = "The `fields` value should be an array of objects containing ""the field `name` and other properties.";- if(!schema_changes.contains("fields") || !schema_changes["fields"].is_array() || schema_changes["fields"].empty()) {+ if((!schema_changes.contains("fields") || !schema_changes["fields"].is_array()+ || schema_changes["fields"].empty())) {return Option<bool>(400, err_msg);}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex cab255f3..6d3fb453 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -2513,4 +2513,16 @@ bool CollectionManager::is_valid_api_key_collection(const std::vector<std::strin}}return api_collections.size() > 0 ? false : true;+}++bool CollectionManager::update_collection_metadata(const std::string& collection, const nlohmann::json& metadata) {+ std::string collection_meta_str;+ auto collection_metakey = Collection::get_meta_key(collection);+ store->get(collection_metakey, collection_meta_str);++ auto collection_meta_json = nlohmann::json::parse(collection_meta_str);++ collection_meta_json[Collection::COLLECTION_METADATA] = metadata;++ return store->insert(collection_metakey, collection_meta_json.dump());}\ No newline at end of filediff --git a/src/core_api.cpp b/src/core_api.cppindex e6396503..8634aa21 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -305,6 +305,7 @@ bool post_create_collection(const std::shared_ptr<http_req>& req, const std::shabool patch_update_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {nlohmann::json req_json;+ std::set<std::string> allowed_keys = {"metadata", "fields"};try {req_json = nlohmann::json::parse(req->body);@@ -315,6 +316,14 @@ bool patch_update_collection(const std::shared_ptr<http_req>& req, const std::shreturn false;}+ for(auto it : req_json.items()) {+ if(allowed_keys.count(it.key()) == 0) {+ res->set_400("Only `fields` and `metadata` can be updated at the moment.");+ alter_in_progress = false;+ return false;+ }+ }+CollectionManager & collectionManager = CollectionManager::get_instance();auto collection = collectionManager.get_collection(req->params["collection"]);@@ -324,11 +333,28 @@ bool patch_update_collection(const std::shared_ptr<http_req>& req, const std::shreturn false;}- auto alter_op = collection->alter(req_json);- if(!alter_op.ok()) {- res->set(alter_op.code(), alter_op.error());- alter_in_progress = false;- return false;+ if(req_json.contains("metadata")) {+ if(!req_json["metadata"].is_object()) {+ res->set_400("The `metadata` value should be an object.");+ alter_in_progress = false;+ return false;+ }++ collection->update_metadata(req_json["metadata"]);++ //update in db+ collectionManager.update_collection_metadata(req->params["collection"], req_json["metadata"]);+ }++ if(req_json.contains("fields")) {+ nlohmann::json alter_payload;+ alter_payload["fields"] = req_json["fields"];+ auto alter_op = collection->alter(alter_payload);+ if(!alter_op.ok()) {+ res->set(alter_op.code(), alter_op.error());+ alter_in_progress = false;+ return false;+ }}alter_in_progress = false;diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex fa820752..5ee1b034 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -485,7 +485,7 @@ TEST_F(CollectionSchemaChangeTest, AlterValidations) {alter_op = coll1->alter(schema_changes);ASSERT_FALSE(alter_op.ok());- ASSERT_EQ("Only `fields` can be updated at the moment.",alter_op.error());+ ASSERT_EQ("Only `fields` and `metadata` can be updated at the moment.",alter_op.error());// 9. bad datatype in alterschema_changes = R"({diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex f0ae9e3e..f1424077 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -1804,4 +1804,264 @@ TEST_F(CoreAPIUtilsTest, SynonymsPagination) {get_collections(req, resp);ASSERT_EQ(400, resp->status_code);ASSERT_EQ("{\"message\": \"Limit param should be unsigned integer.\"}", resp->body);+}+++TEST_F(CoreAPIUtilsTest, CollectionMetadataUpdate) {+ CollectionManager & collectionManager3 = CollectionManager::get_instance();++ nlohmann::json schema = R"({+ "name": "collection_meta",+ "enable_nested_fields": true,+ "fields": [+ {"name": "value.color", "type": "string", "optional": false, "facet": true },+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }+ ],+ "metadata": {+ "batch_job":"",+ "indexed_from":"2023-04-20T00:00:00.000Z",+ "total_docs": 0+ }+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ std::string collection_meta_json;+ nlohmann::json collection_meta;+ std::string next_seq_id;+ std::string next_collection_id;++ store->get(Collection::get_meta_key("collection_meta"), collection_meta_json);++ nlohmann::json expected_meta_json = R"(+ {+ "created_at":1705482381,+ "default_sorting_field":"",+ "enable_nested_fields":true,+ "fallback_field_type":"",+ "fields":[+ {+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.color",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":false,+ "store":true,+ "type":"string"+ },+ {+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.r",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":true,+ "store":true,+ "type":"int32"+ },{+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.g",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":true,+ "store":true,+ "type":"int32"+ },{+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.b",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":true,+ "store":true,+ "type":"int32"+ }+ ],+ "id":1,+ "metadata":{+ "batch_job":"",+ "indexed_from":"2023-04-20T00:00:00.000Z",+ "total_docs":0+ },+ "name":"collection_meta",+ "num_memory_shards":4,+ "symbols_to_index":[],+ "token_separators":[]+ })"_json;++ auto actual_json = nlohmann::json::parse(collection_meta_json);+ expected_meta_json["created_at"] = actual_json["created_at"];++ ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());++ //try setting empty metadata+ auto metadata = R"({+ "metadata": {}+ })"_json;++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ req->params["collection"] = "collection_meta";+ req->body = metadata.dump();+ patch_update_collection(req, res);++ expected_meta_json = R"(+ {+ "created_at":1705482381,+ "default_sorting_field":"",+ "enable_nested_fields":true,+ "fallback_field_type":"",+ "fields":[+ {+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.color",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":false,+ "store":true,+ "type":"string"+ },+ {+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.r",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":true,+ "store":true,+ "type":"int32"+ },{+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.g",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":true,+ "store":true,+ "type":"int32"+ },{+ "facet":true,+ "index":true,+ "infix":false,+ "locale":"",+ "name":"value.b",+ "nested":true,+ "nested_array":2,+ "optional":false,+ "sort":true,+ "store":true,+ "type":"int32"+ }+ ],+ "id":1,+ "metadata":{+ },+ "name":"collection_meta",+ "num_memory_shards":4,+ "symbols_to_index":[],+ "token_separators":[]+ })"_json;++ store->get(Collection::get_meta_key("collection_meta"), collection_meta_json);+ actual_json = nlohmann::json::parse(collection_meta_json);+ expected_meta_json["created_at"] = actual_json["created_at"];+ ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());+}++TEST_F(CoreAPIUtilsTest, CollectionUpdateValidation) {+ CollectionManager & collectionManager3 = CollectionManager::get_instance();++ nlohmann::json schema = R"({+ "name": "collection_meta",+ "enable_nested_fields": true,+ "fields": [+ {"name": "value.color", "type": "string", "optional": false, "facet": true },+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }+ ],+ "metadata": {+ "batch_job":"",+ "indexed_from":"2023-04-20T00:00:00.000Z",+ "total_docs": 0+ }+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll1 = op.get();++ auto alter_schema = R"({+ "metadata": {},+ "fields":[+ {"name": "value.color", "drop": true },+ {"name": "value.color", "type": "string", "facet": true }+ ]+ })"_json;++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ req->params["collection"] = "collection_meta";+ req->body = alter_schema.dump();++ ASSERT_TRUE(patch_update_collection(req, res));++ alter_schema = R"({+ "metadata": {},+ "symbols_to_index":[]+ })"_json;++ req->body = alter_schema.dump();+ ASSERT_FALSE(patch_update_collection(req, res));+ ASSERT_EQ("{\"message\": \"Only `fields` and `metadata` can be updated at the moment.\"}", res->body);++ alter_schema = R"({+ "symbols_to_index":[]+ })"_json;++ req->body = alter_schema.dump();+ ASSERT_FALSE(patch_update_collection(req, res));+ ASSERT_EQ("{\"message\": \"Only `fields` and `metadata` can be updated at the moment.\"}", res->body);++ alter_schema = R"({+ "name": "collection_meta2",+ "metadata": {},+ "fields":[+ {"name": "value.hue", "type": "int32", "optional": false, "facet": true }+ ]+ })"_json;++ req->body = alter_schema.dump();+ ASSERT_FALSE(patch_update_collection(req, res));+ ASSERT_EQ("{\"message\": \"Only `fields` and `metadata` can be updated at the moment.\"}", res->body);}\ No newline at end of file