Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

support updation of collection meta (#1779)

Krunal Gandhi629def6f0f04
diff --git a/include/collection.h b/include/collection.h
index 834e7b3c..ca14a4d1 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -432,6 +432,8 @@ public:
std::string get_default_sorting_field();
+ void update_metadata(const nlohmann::json& meta);
+
static Option<bool> add_reference_helper_fields(nlohmann::json& document, const tsl::htrie_map<char, field>& schema,
const spp::sparse_hash_map<std::string, reference_pair>& reference_fields,
tsl::htrie_set<char>& object_reference_helper_fields,
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 106b3caf..4a2b5c77 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -245,4 +245,6 @@ public:
std::unordered_set<std::string> get_collection_references(const std::string& coll_name);
bool is_valid_api_key_collection(const std::vector<std::string>& api_key_collections, Collection* coll) const;
+
+ bool update_collection_metadata(const std::string& collection, const nlohmann::json& metadata);
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 81e6f4f9..3c282e67 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -4748,6 +4748,11 @@ std::string Collection::get_default_sorting_field() {
return default_sorting_field;
}
+void Collection::update_metadata(const nlohmann::json& meta) {
+ std::shared_lock lock(mutex);
+ metadata = meta;
+}
+
Option<bool> Collection::get_document_from_store(const uint32_t& seq_id,
nlohmann::json& document, bool raw_doc) const {
return get_document_from_store(get_seq_id_key(seq_id), document, raw_doc);
@@ -5645,13 +5650,14 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
}
if(schema_changes.size() != 1) {
- return Option<bool>(400, "Only `fields` can be updated at the moment.");
+ return Option<bool>(400, "Only `fields` and `metadata` can be updated at the moment.");
}
const std::string err_msg = "The `fields` value should be an array of objects containing "
"the field `name` and other properties.";
- if(!schema_changes.contains("fields") || !schema_changes["fields"].is_array() || schema_changes["fields"].empty()) {
+ if((!schema_changes.contains("fields") || !schema_changes["fields"].is_array()
+ || schema_changes["fields"].empty())) {
return Option<bool>(400, err_msg);
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index cab255f3..6d3fb453 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -2513,4 +2513,16 @@ bool CollectionManager::is_valid_api_key_collection(const std::vector<std::strin
}
}
return api_collections.size() > 0 ? false : true;
+}
+
+bool CollectionManager::update_collection_metadata(const std::string& collection, const nlohmann::json& metadata) {
+ std::string collection_meta_str;
+ auto collection_metakey = Collection::get_meta_key(collection);
+ store->get(collection_metakey, collection_meta_str);
+
+ auto collection_meta_json = nlohmann::json::parse(collection_meta_str);
+
+ collection_meta_json[Collection::COLLECTION_METADATA] = metadata;
+
+ return store->insert(collection_metakey, collection_meta_json.dump());
}
\ No newline at end of file
diff --git a/src/core_api.cpp b/src/core_api.cpp
index e6396503..8634aa21 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -305,6 +305,7 @@ bool post_create_collection(const std::shared_ptr<http_req>& req, const std::sha
bool patch_update_collection(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
nlohmann::json req_json;
+ std::set<std::string> allowed_keys = {"metadata", "fields"};
try {
req_json = nlohmann::json::parse(req->body);
@@ -315,6 +316,14 @@ bool patch_update_collection(const std::shared_ptr<http_req>& req, const std::sh
return false;
}
+ for(auto it : req_json.items()) {
+ if(allowed_keys.count(it.key()) == 0) {
+ res->set_400("Only `fields` and `metadata` can be updated at the moment.");
+ alter_in_progress = false;
+ return false;
+ }
+ }
+
CollectionManager & collectionManager = CollectionManager::get_instance();
auto collection = collectionManager.get_collection(req->params["collection"]);
@@ -324,11 +333,28 @@ bool patch_update_collection(const std::shared_ptr<http_req>& req, const std::sh
return false;
}
- auto alter_op = collection->alter(req_json);
- if(!alter_op.ok()) {
- res->set(alter_op.code(), alter_op.error());
- alter_in_progress = false;
- return false;
+ if(req_json.contains("metadata")) {
+ if(!req_json["metadata"].is_object()) {
+ res->set_400("The `metadata` value should be an object.");
+ alter_in_progress = false;
+ return false;
+ }
+
+ collection->update_metadata(req_json["metadata"]);
+
+ //update in db
+ collectionManager.update_collection_metadata(req->params["collection"], req_json["metadata"]);
+ }
+
+ if(req_json.contains("fields")) {
+ nlohmann::json alter_payload;
+ alter_payload["fields"] = req_json["fields"];
+ auto alter_op = collection->alter(alter_payload);
+ if(!alter_op.ok()) {
+ res->set(alter_op.code(), alter_op.error());
+ alter_in_progress = false;
+ return false;
+ }
}
alter_in_progress = false;
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index fa820752..5ee1b034 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -485,7 +485,7 @@ TEST_F(CollectionSchemaChangeTest, AlterValidations) {
alter_op = coll1->alter(schema_changes);
ASSERT_FALSE(alter_op.ok());
- ASSERT_EQ("Only `fields` can be updated at the moment.",alter_op.error());
+ ASSERT_EQ("Only `fields` and `metadata` can be updated at the moment.",alter_op.error());
// 9. bad datatype in alter
schema_changes = R"({
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index f0ae9e3e..f1424077 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -1804,4 +1804,264 @@ TEST_F(CoreAPIUtilsTest, SynonymsPagination) {
get_collections(req, resp);
ASSERT_EQ(400, resp->status_code);
ASSERT_EQ("{\"message\": \"Limit param should be unsigned integer.\"}", resp->body);
+}
+
+
+TEST_F(CoreAPIUtilsTest, CollectionMetadataUpdate) {
+ CollectionManager & collectionManager3 = CollectionManager::get_instance();
+
+ nlohmann::json schema = R"({
+ "name": "collection_meta",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "value.color", "type": "string", "optional": false, "facet": true },
+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }
+ ],
+ "metadata": {
+ "batch_job":"",
+ "indexed_from":"2023-04-20T00:00:00.000Z",
+ "total_docs": 0
+ }
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ std::string collection_meta_json;
+ nlohmann::json collection_meta;
+ std::string next_seq_id;
+ std::string next_collection_id;
+
+ store->get(Collection::get_meta_key("collection_meta"), collection_meta_json);
+
+ nlohmann::json expected_meta_json = R"(
+ {
+ "created_at":1705482381,
+ "default_sorting_field":"",
+ "enable_nested_fields":true,
+ "fallback_field_type":"",
+ "fields":[
+ {
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.color",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":false,
+ "store":true,
+ "type":"string"
+ },
+ {
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.r",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":true,
+ "store":true,
+ "type":"int32"
+ },{
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.g",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":true,
+ "store":true,
+ "type":"int32"
+ },{
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.b",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":true,
+ "store":true,
+ "type":"int32"
+ }
+ ],
+ "id":1,
+ "metadata":{
+ "batch_job":"",
+ "indexed_from":"2023-04-20T00:00:00.000Z",
+ "total_docs":0
+ },
+ "name":"collection_meta",
+ "num_memory_shards":4,
+ "symbols_to_index":[],
+ "token_separators":[]
+ })"_json;
+
+ auto actual_json = nlohmann::json::parse(collection_meta_json);
+ expected_meta_json["created_at"] = actual_json["created_at"];
+
+ ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());
+
+ //try setting empty metadata
+ auto metadata = R"({
+ "metadata": {}
+ })"_json;
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ req->params["collection"] = "collection_meta";
+ req->body = metadata.dump();
+ patch_update_collection(req, res);
+
+ expected_meta_json = R"(
+ {
+ "created_at":1705482381,
+ "default_sorting_field":"",
+ "enable_nested_fields":true,
+ "fallback_field_type":"",
+ "fields":[
+ {
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.color",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":false,
+ "store":true,
+ "type":"string"
+ },
+ {
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.r",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":true,
+ "store":true,
+ "type":"int32"
+ },{
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.g",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":true,
+ "store":true,
+ "type":"int32"
+ },{
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.b",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":true,
+ "store":true,
+ "type":"int32"
+ }
+ ],
+ "id":1,
+ "metadata":{
+ },
+ "name":"collection_meta",
+ "num_memory_shards":4,
+ "symbols_to_index":[],
+ "token_separators":[]
+ })"_json;
+
+ store->get(Collection::get_meta_key("collection_meta"), collection_meta_json);
+ actual_json = nlohmann::json::parse(collection_meta_json);
+ expected_meta_json["created_at"] = actual_json["created_at"];
+ ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());
+}
+
+TEST_F(CoreAPIUtilsTest, CollectionUpdateValidation) {
+ CollectionManager & collectionManager3 = CollectionManager::get_instance();
+
+ nlohmann::json schema = R"({
+ "name": "collection_meta",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "value.color", "type": "string", "optional": false, "facet": true },
+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }
+ ],
+ "metadata": {
+ "batch_job":"",
+ "indexed_from":"2023-04-20T00:00:00.000Z",
+ "total_docs": 0
+ }
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+ auto alter_schema = R"({
+ "metadata": {},
+ "fields":[
+ {"name": "value.color", "drop": true },
+ {"name": "value.color", "type": "string", "facet": true }
+ ]
+ })"_json;
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ req->params["collection"] = "collection_meta";
+ req->body = alter_schema.dump();
+
+ ASSERT_TRUE(patch_update_collection(req, res));
+
+ alter_schema = R"({
+ "metadata": {},
+ "symbols_to_index":[]
+ })"_json;
+
+ req->body = alter_schema.dump();
+ ASSERT_FALSE(patch_update_collection(req, res));
+ ASSERT_EQ("{\"message\": \"Only `fields` and `metadata` can be updated at the moment.\"}", res->body);
+
+ alter_schema = R"({
+ "symbols_to_index":[]
+ })"_json;
+
+ req->body = alter_schema.dump();
+ ASSERT_FALSE(patch_update_collection(req, res));
+ ASSERT_EQ("{\"message\": \"Only `fields` and `metadata` can be updated at the moment.\"}", res->body);
+
+ alter_schema = R"({
+ "name": "collection_meta2",
+ "metadata": {},
+ "fields":[
+ {"name": "value.hue", "type": "int32", "optional": false, "facet": true }
+ ]
+ })"_json;
+
+ req->body = alter_schema.dump();
+ ASSERT_FALSE(patch_update_collection(req, res));
+ ASSERT_EQ("{\"message\": \"Only `fields` and `metadata` can be updated at the moment.\"}", res->body);
}
\ No newline at end of file