Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Reduce duplication in embed validation.
Kishore Nallan31b744a3ca20
diff --git a/include/field.h b/include/field.hindex 470789f1..6b4ee937 100644--- a/include/field.h+++ b/include/field.h@@ -423,6 +423,11 @@ struct field {std::string& fallback_field_type,std::vector<field>& the_fields);+ static Option<bool> validate_and_init_embed_fields(const std::vector<std::pair<size_t, size_t>>& embed_json_field_indices,+ const tsl::htrie_map<char, field>& search_schema,+ nlohmann::json& fields_json,+ std::vector<field>& fields_vec);+static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,const field& the_field, const std::string& flat_name,const std::unordered_map<std::string, field>& dyn_fields,diff --git a/src/collection.cpp b/src/collection.cppindex 4713b127..f5fe1831 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -4175,8 +4175,11 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,}std::unordered_map<std::string, field> new_dynamic_fields;+ std::vector<std::pair<size_t, size_t>> embed_json_field_indices;+ int json_array_index = -1;for(const auto& kv: schema_changes["fields"].items()) {+ json_array_index++;const std::string& field_name = kv.value()["name"].get<std::string>();const auto& field_it = search_schema.find(field_name);auto found_field = (field_it != search_schema.end());@@ -4302,6 +4305,10 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,}}+ if(!f.embed.empty() && !diff_fields.empty()) {+ embed_json_field_indices.emplace_back(json_array_index, diff_fields.size()-1);+ }+} else {// partial update is not supported for nowreturn Option<bool>(400, "Field `" + field_name + "` is already part of the schema: To "@@ -4310,63 +4317,10 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,}}- for(const auto& kv: schema_changes["fields"].items()) {- // validate embedding fields externally- auto& field_json = kv.value();- if(field_json.count(fields::embed) != 0 && !field_json[fields::embed].empty()) {- if(!field_json[fields::embed].is_object()) {- return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");- }-- if(field_json[fields::embed].count(fields::from) == 0) {- return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");- }-- if(!field_json[fields::embed][fields::from].is_array()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");- }-- if(field_json[fields::embed][fields::from].empty()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");- }-- for(auto& embed_from_field : field_json[fields::embed][fields::from]) {- if(!embed_from_field.is_string()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");- }- }-- if(field_json[fields::type] != field_types::FLOAT_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` is only allowed on a float array field.");- }-- for(auto& embed_from_field : field_json[fields::embed][fields::from]) {- bool flag = false;- for(const auto& field : search_schema) {- if(field.name == embed_from_field) {- if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");- }- flag = true;- break;- }- }- if(!flag) {- for(const auto& other_kv: schema_changes["fields"].items()) {- if(other_kv.value()["name"] == embed_from_field) {- if(other_kv.value()[fields::type] != field_types::STRING && other_kv.value()[fields::type] != field_types::STRING_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");- }- flag = true;- break;- }- }- }- if(!flag) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");- }- }- }+ auto validation_op = field::validate_and_init_embed_fields(embed_json_field_indices, search_schema,+ schema_changes["fields"], diff_fields);+ if(!validation_op.ok()) {+ return validation_op;}if(num_auto_detect_fields > 1) {diff --git a/src/field.cpp b/src/field.cppindex 5386378b..25da190f 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -673,6 +673,62 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}}+ if(field_json.count(fields::embed) != 0) {+ if(!field_json[fields::embed].is_object()) {+ return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");+ }++ auto& embed_json = field_json[fields::embed];++ if(field_json[fields::embed].count(fields::from) == 0) {+ return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");+ }++ if(!field_json[fields::embed][fields::from].is_array()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");+ }++ if(field_json[fields::embed][fields::from].empty()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");+ }++ if(embed_json.count(fields::model_config) == 0) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "` not found.");+ }++ auto& model_config = embed_json[fields::model_config];++ if(model_config.count(fields::model_name) == 0) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "`not found");+ }++ if(!model_config[fields::model_name].is_string()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` must be a string.");+ }++ if(model_config[fields::model_name].get<std::string>().empty()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` cannot be empty.");+ }++ if(model_config.count(fields::indexing_prefix) != 0) {+ if(!model_config[fields::indexing_prefix].is_string()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::indexing_prefix + "` must be a string.");+ }+ }++ if(model_config.count(fields::query_prefix) != 0) {+ if(!model_config[fields::query_prefix].is_string()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::query_prefix + "` must be a string.");+ }+ }++ for(auto& embed_from_field : field_json[fields::embed][fields::from]) {+ if(!embed_from_field.is_string()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");+ }+ }+ }+auto DEFAULT_VEC_DIST_METRIC = magic_enum::enum_name(vector_distance_type_t::cosine);if(field_json.count(fields::num_dim) == 0) {@@ -709,7 +765,6 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}}-if(field_json.count(fields::optional) == 0) {// dynamic type fields are always optionalbool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);@@ -999,114 +1054,77 @@ void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {}Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::json &fields_json, string &fallback_field_type,- std::vector<field> &the_fields) {+ std::vector<field>& the_fields) {size_t num_auto_detect_fields = 0;+ std::vector<std::pair<size_t, size_t>> embed_json_field_indices;- for(nlohmann::json & field_json: fields_json) {- if(field_json.count(fields::embed) != 0) {- if(!field_json[fields::embed].is_object()) {- return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");- }-- auto& embed_json = field_json[fields::embed];-- if(field_json[fields::embed].count(fields::from) == 0) {- return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");- }-- if(!field_json[fields::embed][fields::from].is_array()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");- }-- if(field_json[fields::embed][fields::from].empty()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");- }-- for(auto& embed_from_field : field_json[fields::embed][fields::from]) {- if(!embed_from_field.is_string()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");- }-- bool flag = false;- for(const auto& field : fields_json) {- if(field[fields::name] == embed_from_field) {- if(field[fields::type] != field_types::STRING && field[fields::type] != field_types::STRING_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");- }- flag = true;- break;- }- }- if(!flag) {- for(const auto& field : the_fields) {- if(field.name == embed_from_field) {- if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");- }- flag = true;- break;- }- }- }-- if(!flag) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");- }- }-- if(field_json[fields::type] != field_types::FLOAT_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` is only allowed on a float array field.");- }-- if(embed_json.count(fields::model_config) == 0) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "` not found.");- }+ for(size_t i = 0; i < fields_json.size(); i++) {+ nlohmann::json& field_json = fields_json[i];+ auto op = json_field_to_field(enable_nested_fields,+ field_json, the_fields, fallback_field_type, num_auto_detect_fields);+ if(!op.ok()) {+ return op;+ }- auto& model_config = embed_json[fields::model_config];+ if(!the_fields.empty() && !the_fields.back().embed.empty()) {+ embed_json_field_indices.emplace_back(i, i);+ }+ }- if(model_config.count(fields::model_name) == 0) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "`not found");- }+ const tsl::htrie_map<char, field> dummy_search_schema;+ auto validation_op = field::validate_and_init_embed_fields(embed_json_field_indices, dummy_search_schema,+ fields_json, the_fields);+ if(!validation_op.ok()) {+ return validation_op;+ }- if(!model_config[fields::model_name].is_string()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` must be a string.");- }- if(model_config[fields::model_name].get<std::string>().empty()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` cannot be empty.");- }+ if(num_auto_detect_fields > 1) {+ return Option<bool>(400,"There can be only one field named `.*`.");+ }- if(model_config.count(fields::indexing_prefix) != 0) {- if(!model_config[fields::indexing_prefix].is_string()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::indexing_prefix + "` must be a string.");- }- }+ return Option<bool>(true);+}- if(model_config.count(fields::query_prefix) != 0) {- if(!model_config[fields::query_prefix].is_string()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::query_prefix + "` must be a string.");+Option<bool> field::validate_and_init_embed_fields(const std::vector<std::pair<size_t, size_t>>& embed_json_field_indices,+ const tsl::htrie_map<char, field>& search_schema,+ nlohmann::json& fields_json,+ std::vector<field>& fields_vec) {++ for(const auto& json_field_index: embed_json_field_indices) {+ auto& field_json = fields_json[json_field_index.first];+ const std::string err_msg = "Property `" + fields::embed + "." + fields::from ++ "` can only refer to string or string array fields.";++ LOG(INFO) << "field_json: " << field_json;++ for(auto& field_name : field_json[fields::embed][fields::from].get<std::vector<std::string>>()) {+ auto embed_field = std::find_if(fields_json.begin(), fields_json.end(), [&field_name](const nlohmann::json& x) {+ return x["name"].get<std::string>() == field_name;+ });++ if(embed_field == fields_json.end()) {+ const auto& embed_field2 = search_schema.find(field_name);+ if (embed_field2 == search_schema.end()) {+ return Option<bool>(400, err_msg);+ } else if (embed_field2->type != field_types::STRING && embed_field2->type != field_types::STRING_ARRAY) {+ return Option<bool>(400, err_msg);}+ } else if((*embed_field)[fields::type] != field_types::STRING &&+ (*embed_field)[fields::type] != field_types::STRING_ARRAY) {+ return Option<bool>(400, err_msg);}-- size_t num_dim = 0;- auto res = TextEmbedderManager::validate_and_init_model(model_config, num_dim);- if(!res.ok()) {- return Option<bool>(res.code(), res.error());- }-- LOG(INFO) << "Model init done.";- embed_json[fields::num_dim] = num_dim;- field_json[fields::num_dim] = num_dim;}- auto op = json_field_to_field(enable_nested_fields,- field_json, the_fields, fallback_field_type, num_auto_detect_fields);- if(!op.ok()) {- return op;+ const auto& model_config = field_json[fields::embed][fields::model_config];+ size_t num_dim = 0;+ auto res = TextEmbedderManager::validate_and_init_model(model_config, num_dim);+ if(!res.ok()) {+ return Option<bool>(res.code(), res.error());}- }- if(num_auto_detect_fields > 1) {- return Option<bool>(400,"There can be only one field named `.*`.");+ LOG(INFO) << "Model init done.";+ field_json[fields::num_dim] = num_dim;+ fields_vec[json_field_index.second].num_dim = num_dim;}return Option<bool>(true);diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 15e193a3..e76bb45a 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1688,14 +1688,17 @@ TEST_F(CollectionAllFieldsTest, WrongDataTypeForEmbedFrom) {field_json["embed"]["model_config"] = nlohmann::json::object();field_json["embed"]["model_config"]["model_name"] = "ts/e5-small";- std::vector<field> fields;- std::string fallback_field_type;- auto arr = nlohmann::json::array();- arr.push_back(field_json);- field_json["name"] = "age";- field_json["type"] = "int32";- arr.push_back(field_json);- auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);- ASSERT_FALSE(field_op.ok());- ASSERT_EQ("Property `embed.from` can only refer to string or string array fields.", field_op.error());-}\ No newline at end of file+ nlohmann::json schema = R"({+ "name": "obj_coll",+ "fields": [+ {"name": "age", "type": "int32"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["age"],+ "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ auto obj_coll_op = collectionManager.create_collection(schema);++ ASSERT_FALSE(obj_coll_op.ok());+ ASSERT_EQ("Property `embed.from` can only refer to string or string array fields.", obj_coll_op.error());+}diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex f081a3f1..61f53e7a 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -1633,42 +1633,102 @@ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {TEST_F(CollectionSchemaChangeTest, DropAndReindexEmbeddingField) {nlohmann::json schema = R"({- "name": "objects",- "fields": [- {"name": "name", "type": "string"},- {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}- ]- })"_json;+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- auto op = collectionManager.create_collection(schema);-- ASSERT_TRUE(op.ok());+ auto create_op = collectionManager.create_collection(schema);+ ASSERT_TRUE(create_op.ok());+ auto coll = create_op.get();// drop the embedding field and reindex- nlohmann::json schema_without_embedding = R"({- "fields": [- {"name": "embedding", "drop": true},- {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}- ]- })"_json;+ nlohmann::json alter_schema = R"({+ "fields": [+ {"name": "embedding", "drop": true},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;- auto update_op = op.get()->alter(schema_without_embedding);-+ auto update_op = coll->alter(alter_schema);ASSERT_TRUE(update_op.ok());- auto embedding_fields_map = op.get()->get_embedding_fields();+ auto embedding_fields_map = coll->get_embedding_fields();ASSERT_EQ(1, embedding_fields_map.size());// try adding a documentnlohmann::json doc;doc["name"] = "hello";- auto add_op = op.get()->add(doc.dump());-+ auto add_op = coll->add(doc.dump());ASSERT_TRUE(add_op.ok());auto added_doc = add_op.get();+ ASSERT_EQ(384, added_doc["embedding"].get<std::vector<float>>().size());++ // alter with bad schema+ alter_schema = R"({+ "fields": [+ {"name": "embedding", "drop": true},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["namez"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ update_op = coll->alter(alter_schema);+ ASSERT_FALSE(update_op.ok());+ ASSERT_EQ("Property `embed.from` can only refer to string or string array fields.", update_op.error());+ // alter with bad model name+ alter_schema = R"({+ "fields": [+ {"name": "embedding", "drop": true},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/x5-small"}}}+ ]+ })"_json;++ update_op = coll->alter(alter_schema);+ ASSERT_FALSE(update_op.ok());+ ASSERT_EQ("Model not found", update_op.error());++ // should still be able to add doc after aborted alter+ add_op = coll->add(doc.dump());+ ASSERT_TRUE(add_op.ok());+ added_doc = add_op.get();ASSERT_EQ(384, added_doc["embedding"].get<std::vector<float>>().size());-}\ No newline at end of file+}++TEST_F(CollectionSchemaChangeTest, EmbeddingFieldAlterDropTest) {+ nlohmann::json schema = R"({+ "name": "objects",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection* coll = op.get();++ auto& vec_index = coll->_get_index()->_get_vector_index();+ ASSERT_EQ(1, vec_index.size());+ ASSERT_EQ(1, vec_index.count("embedding"));+++ nlohmann::json schema_change = R"({+ "fields": [+ {"name": "embedding", "drop": true}+ ]+ })"_json;++ auto schema_change_op = coll->alter(schema_change);++ ASSERT_TRUE(schema_change_op.ok());+ ASSERT_EQ(0, vec_index.size());+ ASSERT_EQ(0, vec_index.count("embedding"));+}diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 66cd9223..2337d967 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -921,36 +921,3 @@ TEST_F(CollectionVectorTest, DistanceThresholdTest) {ASSERT_FLOAT_EQ(0.8, results_op.get()["hits"][0]["document"]["vec"].get<std::vector<float>>()[2]);}--TEST_F(CollectionVectorTest, EmbeddingFieldAlterDropTest) {- nlohmann::json schema = R"({- "name": "objects",- "fields": [- {"name": "name", "type": "string"},- {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}- ]- })"_json;-- TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");-- auto op = collectionManager.create_collection(schema);- ASSERT_TRUE(op.ok());- Collection* coll = op.get();-- auto& vec_index = coll->_get_index()->_get_vector_index();- ASSERT_EQ(1, vec_index.size());- ASSERT_EQ(1, vec_index.count("embedding"));--- nlohmann::json schema_change = R"({- "fields": [- {"name": "embedding", "drop": true}- ]- })"_json;-- auto schema_change_op = coll->alter(schema_change);-- ASSERT_TRUE(schema_change_op.ok());- ASSERT_EQ(0, vec_index.size());- ASSERT_EQ(0, vec_index.count("embedding"));-}\ No newline at end of file