Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

facilitate to add metadata while creating collection (#1491)

Krunal Gandhie51055712c5e
diff --git a/include/collection.h b/include/collection.h
index 4f313729..30b78a00 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -372,6 +372,8 @@ public:
static constexpr const char* COLLECTION_SEPARATORS = "token_separators";
static constexpr const char* COLLECTION_VOICE_QUERY_MODEL = "voice_query_model";
+ static constexpr const char* COLLECTION_METADATA = "metadata";
+
// methods
Collection() = delete;
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 11df6703..a88e3f7b 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -169,7 +169,8 @@ public:
const std::string& fallback_field_type = "",
const std::vector<std::string>& symbols_to_index = {},
const std::vector<std::string>& token_separators = {},
- const bool enable_nested_fields = false, std::shared_ptr<VQModel> model = nullptr);
+ const bool enable_nested_fields = false, std::shared_ptr<VQModel> model = nullptr,
+ const nlohmann::json& metadata = {});
locked_resource_view_t<Collection> get_collection(const std::string & collection_name) const;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index dbd2b97c..7acc90bb 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -442,7 +442,8 @@ Option<Collection*> CollectionManager::create_collection(const std::string& name
const std::string& fallback_field_type,
const std::vector<std::string>& symbols_to_index,
const std::vector<std::string>& token_separators,
- const bool enable_nested_fields, std::shared_ptr<VQModel> model) {
+ const bool enable_nested_fields, std::shared_ptr<VQModel> model,
+ const nlohmann::json& metadata) {
std::unique_lock lock(mutex);
if(store->contains(Collection::get_meta_key(name))) {
@@ -485,6 +486,10 @@ Option<Collection*> CollectionManager::create_collection(const std::string& name
collection_meta[Collection::COLLECTION_VOICE_QUERY_MODEL]["model_name"] = model->get_model_name();
}
+ if(!metadata.empty()) {
+ collection_meta[Collection::COLLECTION_METADATA] = metadata;
+ }
+
rocksdb::WriteBatch batch;
batch.Put(Collection::get_next_seq_id_key(name), StringUtils::serialize_uint32_t(0));
batch.Put(Collection::get_meta_key(name), collection_meta.dump());
@@ -1869,6 +1874,7 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
const char* TOKEN_SEPARATORS = "token_separators";
const char* ENABLE_NESTED_FIELDS = "enable_nested_fields";
const char* DEFAULT_SORTING_FIELD = "default_sorting_field";
+ const char* METADATA = "metadata";
// validate presence of mandatory fields
@@ -1949,6 +1955,14 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
"`name`, `type` and optionally, `facet` properties.");
}
+ if(req_json.count(METADATA) != 0) {
+ if(!req_json[METADATA].is_object()) {
+ return Option<Collection *>(400, "The `metadata` value should be an object.");
+ }
+ } else {
+ req_json[METADATA] = {};
+ }
+
const std::string& default_sorting_field = req_json[DEFAULT_SORTING_FIELD].get<std::string>();
if(default_sorting_field == "id") {
@@ -1998,7 +2012,8 @@ Option<Collection*> CollectionManager::create_collection(nlohmann::json& req_jso
fallback_field_type,
req_json[SYMBOLS_TO_INDEX],
req_json[TOKEN_SEPARATORS],
- req_json[ENABLE_NESTED_FIELDS], model);
+ req_json[ENABLE_NESTED_FIELDS],
+ model, req_json[METADATA]);
}
Option<bool> CollectionManager::load_collection(const nlohmann::json &collection_meta,
@@ -2316,4 +2331,4 @@ void CollectionManager::process_embedding_field_delete(const std::string& model_
EmbedderManager::get_instance().delete_text_embedder(model_name);
EmbedderManager::get_instance().delete_image_embedder(model_name);
}
-}
+}
\ No newline at end of file
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 0a4ef186..08f8400e 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -1728,4 +1728,130 @@ TEST_F(CollectionManagerTest, ReferencedInBacklog) {
get_reference_field_op = create_op.get()->get_referenced_in_field_with_lock("foo");
ASSERT_FALSE(get_reference_field_op.ok());
ASSERT_EQ("Could not find any field in `Products` referencing the collection `foo`.", get_reference_field_op.error());
+}
+
+TEST_F(CollectionManagerTest, CollectionCreationWithMetadata) {
+ CollectionManager & collectionManager3 = CollectionManager::get_instance();
+
+ nlohmann::json schema1 = R"({
+ "name": "collection_meta",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "value.color", "type": "string", "optional": false, "facet": true },
+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }
+ ],
+ "metadata": "abc"
+ })"_json;
+
+ auto op = collectionManager.create_collection(schema1);
+ ASSERT_FALSE(op.ok());
+ ASSERT_EQ("The `metadata` value should be an object.", op.error());
+
+ nlohmann::json schema2 = R"({
+ "name": "collection_meta",
+ "enable_nested_fields": true,
+ "fields": [
+ {"name": "value.color", "type": "string", "optional": false, "facet": true },
+ {"name": "value.r", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.g", "type": "int32", "optional": false, "facet": true },
+ {"name": "value.b", "type": "int32", "optional": false, "facet": true }
+ ],
+ "metadata": {
+ "batch_job":"",
+ "indexed_from":"2023-04-20T00:00:00.000Z",
+ "total_docs": 0
+ }
+ })"_json;
+
+ op = collectionManager.create_collection(schema2);
+ ASSERT_TRUE(op.ok());
+ Collection* coll1 = op.get();
+
+
+ std::string collection_meta_json;
+ nlohmann::json collection_meta;
+ std::string next_seq_id;
+ std::string next_collection_id;
+
+ store->get(Collection::get_meta_key("collection_meta"), collection_meta_json);
+ store->get(Collection::get_next_seq_id_key("collection_meta"), next_seq_id);
+
+ //LOG(INFO) << collection_meta_json;
+
+ nlohmann::json expected_meta_json = R"(
+ {
+ "created_at":1705482381,
+ "default_sorting_field":"",
+ "enable_nested_fields":true,
+ "fallback_field_type":"",
+ "fields":[
+ {
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.color",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":false,
+ "store":true,
+ "type":"string"
+ },
+ {
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.r",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":true,
+ "store":true,
+ "type":"int32"
+ },{
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.g",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":true,
+ "store":true,
+ "type":"int32"
+ },{
+ "facet":true,
+ "index":true,
+ "infix":false,
+ "locale":"",
+ "name":"value.b",
+ "nested":true,
+ "nested_array":2,
+ "optional":false,
+ "sort":true,
+ "store":true,
+ "type":"int32"
+ }
+ ],
+ "id":1,
+ "metadata":{
+ "batch_job":"",
+ "indexed_from":"2023-04-20T00:00:00.000Z",
+ "total_docs":0
+ },
+ "name":"collection_meta",
+ "num_memory_shards":4,
+ "symbols_to_index":[],
+ "token_separators":[]
+ })"_json;
+
+ auto actual_json = nlohmann::json::parse(collection_meta_json);
+ expected_meta_json["created_at"] = actual_json["created_at"];
+
+ ASSERT_EQ(expected_meta_json.dump(), actual_json.dump());
}
\ No newline at end of file