Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Reduce duplication in embed validation.

Kishore Nallan31b744a3ca20
diff --git a/include/field.h b/include/field.h
index 470789f1..6b4ee937 100644
--- a/include/field.h
+++ b/include/field.h
@@ -423,6 +423,11 @@ struct field {
std::string& fallback_field_type,
std::vector<field>& the_fields);
+ static Option<bool> validate_and_init_embed_fields(const std::vector<std::pair<size_t, size_t>>& embed_json_field_indices,
+ const tsl::htrie_map<char, field>& search_schema,
+ nlohmann::json& fields_json,
+ std::vector<field>& fields_vec);
+
static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,
const field& the_field, const std::string& flat_name,
const std::unordered_map<std::string, field>& dyn_fields,
diff --git a/src/collection.cpp b/src/collection.cpp
index 4713b127..f5fe1831 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -4175,8 +4175,11 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
}
std::unordered_map<std::string, field> new_dynamic_fields;
+ std::vector<std::pair<size_t, size_t>> embed_json_field_indices;
+ int json_array_index = -1;
for(const auto& kv: schema_changes["fields"].items()) {
+ json_array_index++;
const std::string& field_name = kv.value()["name"].get<std::string>();
const auto& field_it = search_schema.find(field_name);
auto found_field = (field_it != search_schema.end());
@@ -4302,6 +4305,10 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
}
}
+ if(!f.embed.empty() && !diff_fields.empty()) {
+ embed_json_field_indices.emplace_back(json_array_index, diff_fields.size()-1);
+ }
+
} else {
// partial update is not supported for now
return Option<bool>(400, "Field `" + field_name + "` is already part of the schema: To "
@@ -4310,63 +4317,10 @@ Option<bool> Collection::validate_alter_payload(nlohmann::json& schema_changes,
}
}
- for(const auto& kv: schema_changes["fields"].items()) {
- // validate embedding fields externally
- auto& field_json = kv.value();
- if(field_json.count(fields::embed) != 0 && !field_json[fields::embed].empty()) {
- if(!field_json[fields::embed].is_object()) {
- return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");
- }
-
- if(field_json[fields::embed].count(fields::from) == 0) {
- return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");
- }
-
- if(!field_json[fields::embed][fields::from].is_array()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");
- }
-
- if(field_json[fields::embed][fields::from].empty()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");
- }
-
- for(auto& embed_from_field : field_json[fields::embed][fields::from]) {
- if(!embed_from_field.is_string()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");
- }
- }
-
- if(field_json[fields::type] != field_types::FLOAT_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` is only allowed on a float array field.");
- }
-
- for(auto& embed_from_field : field_json[fields::embed][fields::from]) {
- bool flag = false;
- for(const auto& field : search_schema) {
- if(field.name == embed_from_field) {
- if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
- }
- flag = true;
- break;
- }
- }
- if(!flag) {
- for(const auto& other_kv: schema_changes["fields"].items()) {
- if(other_kv.value()["name"] == embed_from_field) {
- if(other_kv.value()[fields::type] != field_types::STRING && other_kv.value()[fields::type] != field_types::STRING_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
- }
- flag = true;
- break;
- }
- }
- }
- if(!flag) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
- }
- }
- }
+ auto validation_op = field::validate_and_init_embed_fields(embed_json_field_indices, search_schema,
+ schema_changes["fields"], diff_fields);
+ if(!validation_op.ok()) {
+ return validation_op;
}
if(num_auto_detect_fields > 1) {
diff --git a/src/field.cpp b/src/field.cpp
index 5386378b..25da190f 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -673,6 +673,62 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
}
}
+ if(field_json.count(fields::embed) != 0) {
+ if(!field_json[fields::embed].is_object()) {
+ return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");
+ }
+
+ auto& embed_json = field_json[fields::embed];
+
+ if(field_json[fields::embed].count(fields::from) == 0) {
+ return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");
+ }
+
+ if(!field_json[fields::embed][fields::from].is_array()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");
+ }
+
+ if(field_json[fields::embed][fields::from].empty()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");
+ }
+
+ if(embed_json.count(fields::model_config) == 0) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "` not found.");
+ }
+
+ auto& model_config = embed_json[fields::model_config];
+
+ if(model_config.count(fields::model_name) == 0) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "`not found");
+ }
+
+ if(!model_config[fields::model_name].is_string()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` must be a string.");
+ }
+
+ if(model_config[fields::model_name].get<std::string>().empty()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` cannot be empty.");
+ }
+
+ if(model_config.count(fields::indexing_prefix) != 0) {
+ if(!model_config[fields::indexing_prefix].is_string()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::indexing_prefix + "` must be a string.");
+ }
+ }
+
+ if(model_config.count(fields::query_prefix) != 0) {
+ if(!model_config[fields::query_prefix].is_string()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::query_prefix + "` must be a string.");
+ }
+ }
+
+ for(auto& embed_from_field : field_json[fields::embed][fields::from]) {
+ if(!embed_from_field.is_string()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");
+ }
+ }
+ }
+
auto DEFAULT_VEC_DIST_METRIC = magic_enum::enum_name(vector_distance_type_t::cosine);
if(field_json.count(fields::num_dim) == 0) {
@@ -709,7 +765,6 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
}
}
-
if(field_json.count(fields::optional) == 0) {
// dynamic type fields are always optional
bool is_dynamic = field::is_dynamic(field_json[fields::name], field_json[fields::type]);
@@ -999,114 +1054,77 @@ void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {
}
Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::json &fields_json, string &fallback_field_type,
- std::vector<field> &the_fields) {
+ std::vector<field>& the_fields) {
size_t num_auto_detect_fields = 0;
+ std::vector<std::pair<size_t, size_t>> embed_json_field_indices;
- for(nlohmann::json & field_json: fields_json) {
- if(field_json.count(fields::embed) != 0) {
- if(!field_json[fields::embed].is_object()) {
- return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");
- }
-
- auto& embed_json = field_json[fields::embed];
-
- if(field_json[fields::embed].count(fields::from) == 0) {
- return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");
- }
-
- if(!field_json[fields::embed][fields::from].is_array()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");
- }
-
- if(field_json[fields::embed][fields::from].empty()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");
- }
-
- for(auto& embed_from_field : field_json[fields::embed][fields::from]) {
- if(!embed_from_field.is_string()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");
- }
-
- bool flag = false;
- for(const auto& field : fields_json) {
- if(field[fields::name] == embed_from_field) {
- if(field[fields::type] != field_types::STRING && field[fields::type] != field_types::STRING_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
- }
- flag = true;
- break;
- }
- }
- if(!flag) {
- for(const auto& field : the_fields) {
- if(field.name == embed_from_field) {
- if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
- }
- flag = true;
- break;
- }
- }
- }
-
- if(!flag) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
- }
- }
-
- if(field_json[fields::type] != field_types::FLOAT_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` is only allowed on a float array field.");
- }
-
- if(embed_json.count(fields::model_config) == 0) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "` not found.");
- }
+ for(size_t i = 0; i < fields_json.size(); i++) {
+ nlohmann::json& field_json = fields_json[i];
+ auto op = json_field_to_field(enable_nested_fields,
+ field_json, the_fields, fallback_field_type, num_auto_detect_fields);
+ if(!op.ok()) {
+ return op;
+ }
- auto& model_config = embed_json[fields::model_config];
+ if(!the_fields.empty() && !the_fields.back().embed.empty()) {
+ embed_json_field_indices.emplace_back(i, i);
+ }
+ }
- if(model_config.count(fields::model_name) == 0) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "`not found");
- }
+ const tsl::htrie_map<char, field> dummy_search_schema;
+ auto validation_op = field::validate_and_init_embed_fields(embed_json_field_indices, dummy_search_schema,
+ fields_json, the_fields);
+ if(!validation_op.ok()) {
+ return validation_op;
+ }
- if(!model_config[fields::model_name].is_string()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` must be a string.");
- }
- if(model_config[fields::model_name].get<std::string>().empty()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` cannot be empty.");
- }
+ if(num_auto_detect_fields > 1) {
+ return Option<bool>(400,"There can be only one field named `.*`.");
+ }
- if(model_config.count(fields::indexing_prefix) != 0) {
- if(!model_config[fields::indexing_prefix].is_string()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::indexing_prefix + "` must be a string.");
- }
- }
+ return Option<bool>(true);
+}
- if(model_config.count(fields::query_prefix) != 0) {
- if(!model_config[fields::query_prefix].is_string()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::query_prefix + "` must be a string.");
+Option<bool> field::validate_and_init_embed_fields(const std::vector<std::pair<size_t, size_t>>& embed_json_field_indices,
+ const tsl::htrie_map<char, field>& search_schema,
+ nlohmann::json& fields_json,
+ std::vector<field>& fields_vec) {
+
+ for(const auto& json_field_index: embed_json_field_indices) {
+ auto& field_json = fields_json[json_field_index.first];
+ const std::string err_msg = "Property `" + fields::embed + "." + fields::from +
+ "` can only refer to string or string array fields.";
+
+ LOG(INFO) << "field_json: " << field_json;
+
+ for(auto& field_name : field_json[fields::embed][fields::from].get<std::vector<std::string>>()) {
+ auto embed_field = std::find_if(fields_json.begin(), fields_json.end(), [&field_name](const nlohmann::json& x) {
+ return x["name"].get<std::string>() == field_name;
+ });
+
+ if(embed_field == fields_json.end()) {
+ const auto& embed_field2 = search_schema.find(field_name);
+ if (embed_field2 == search_schema.end()) {
+ return Option<bool>(400, err_msg);
+ } else if (embed_field2->type != field_types::STRING && embed_field2->type != field_types::STRING_ARRAY) {
+ return Option<bool>(400, err_msg);
}
+ } else if((*embed_field)[fields::type] != field_types::STRING &&
+ (*embed_field)[fields::type] != field_types::STRING_ARRAY) {
+ return Option<bool>(400, err_msg);
}
-
- size_t num_dim = 0;
- auto res = TextEmbedderManager::validate_and_init_model(model_config, num_dim);
- if(!res.ok()) {
- return Option<bool>(res.code(), res.error());
- }
-
- LOG(INFO) << "Model init done.";
- embed_json[fields::num_dim] = num_dim;
- field_json[fields::num_dim] = num_dim;
}
- auto op = json_field_to_field(enable_nested_fields,
- field_json, the_fields, fallback_field_type, num_auto_detect_fields);
- if(!op.ok()) {
- return op;
+ const auto& model_config = field_json[fields::embed][fields::model_config];
+ size_t num_dim = 0;
+ auto res = TextEmbedderManager::validate_and_init_model(model_config, num_dim);
+ if(!res.ok()) {
+ return Option<bool>(res.code(), res.error());
}
- }
- if(num_auto_detect_fields > 1) {
- return Option<bool>(400,"There can be only one field named `.*`.");
+ LOG(INFO) << "Model init done.";
+ field_json[fields::num_dim] = num_dim;
+ fields_vec[json_field_index.second].num_dim = num_dim;
}
return Option<bool>(true);
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 15e193a3..e76bb45a 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -1688,14 +1688,17 @@ TEST_F(CollectionAllFieldsTest, WrongDataTypeForEmbedFrom) {
field_json["embed"]["model_config"] = nlohmann::json::object();
field_json["embed"]["model_config"]["model_name"] = "ts/e5-small";
- std::vector<field> fields;
- std::string fallback_field_type;
- auto arr = nlohmann::json::array();
- arr.push_back(field_json);
- field_json["name"] = "age";
- field_json["type"] = "int32";
- arr.push_back(field_json);
- auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);
- ASSERT_FALSE(field_op.ok());
- ASSERT_EQ("Property `embed.from` can only refer to string or string array fields.", field_op.error());
-}
\ No newline at end of file
+ nlohmann::json schema = R"({
+ "name": "obj_coll",
+ "fields": [
+ {"name": "age", "type": "int32"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["age"],
+ "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ auto obj_coll_op = collectionManager.create_collection(schema);
+
+ ASSERT_FALSE(obj_coll_op.ok());
+ ASSERT_EQ("Property `embed.from` can only refer to string or string array fields.", obj_coll_op.error());
+}
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index f081a3f1..61f53e7a 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -1633,42 +1633,102 @@ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {
TEST_F(CollectionSchemaChangeTest, DropAndReindexEmbeddingField) {
nlohmann::json schema = R"({
- "name": "objects",
- "fields": [
- {"name": "name", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}
- ]
- })"_json;
+ "name": "objects",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- auto op = collectionManager.create_collection(schema);
-
- ASSERT_TRUE(op.ok());
+ auto create_op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(create_op.ok());
+ auto coll = create_op.get();
// drop the embedding field and reindex
- nlohmann::json schema_without_embedding = R"({
- "fields": [
- {"name": "embedding", "drop": true},
- {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}
- ]
- })"_json;
+ nlohmann::json alter_schema = R"({
+ "fields": [
+ {"name": "embedding", "drop": true},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
- auto update_op = op.get()->alter(schema_without_embedding);
-
+ auto update_op = coll->alter(alter_schema);
ASSERT_TRUE(update_op.ok());
- auto embedding_fields_map = op.get()->get_embedding_fields();
+ auto embedding_fields_map = coll->get_embedding_fields();
ASSERT_EQ(1, embedding_fields_map.size());
// try adding a document
nlohmann::json doc;
doc["name"] = "hello";
- auto add_op = op.get()->add(doc.dump());
-
+ auto add_op = coll->add(doc.dump());
ASSERT_TRUE(add_op.ok());
auto added_doc = add_op.get();
+ ASSERT_EQ(384, added_doc["embedding"].get<std::vector<float>>().size());
+
+ // alter with bad schema
+ alter_schema = R"({
+ "fields": [
+ {"name": "embedding", "drop": true},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["namez"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ update_op = coll->alter(alter_schema);
+ ASSERT_FALSE(update_op.ok());
+ ASSERT_EQ("Property `embed.from` can only refer to string or string array fields.", update_op.error());
+ // alter with bad model name
+ alter_schema = R"({
+ "fields": [
+ {"name": "embedding", "drop": true},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/x5-small"}}}
+ ]
+ })"_json;
+
+ update_op = coll->alter(alter_schema);
+ ASSERT_FALSE(update_op.ok());
+ ASSERT_EQ("Model not found", update_op.error());
+
+ // should still be able to add doc after aborted alter
+ add_op = coll->add(doc.dump());
+ ASSERT_TRUE(add_op.ok());
+ added_doc = add_op.get();
ASSERT_EQ(384, added_doc["embedding"].get<std::vector<float>>().size());
-}
\ No newline at end of file
+}
+
+TEST_F(CollectionSchemaChangeTest, EmbeddingFieldAlterDropTest) {
+ nlohmann::json schema = R"({
+ "name": "objects",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+
+ auto op = collectionManager.create_collection(schema);
+ ASSERT_TRUE(op.ok());
+ Collection* coll = op.get();
+
+ auto& vec_index = coll->_get_index()->_get_vector_index();
+ ASSERT_EQ(1, vec_index.size());
+ ASSERT_EQ(1, vec_index.count("embedding"));
+
+
+ nlohmann::json schema_change = R"({
+ "fields": [
+ {"name": "embedding", "drop": true}
+ ]
+ })"_json;
+
+ auto schema_change_op = coll->alter(schema_change);
+
+ ASSERT_TRUE(schema_change_op.ok());
+ ASSERT_EQ(0, vec_index.size());
+ ASSERT_EQ(0, vec_index.count("embedding"));
+}
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 66cd9223..2337d967 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -921,36 +921,3 @@ TEST_F(CollectionVectorTest, DistanceThresholdTest) {
ASSERT_FLOAT_EQ(0.8, results_op.get()["hits"][0]["document"]["vec"].get<std::vector<float>>()[2]);
}
-
-TEST_F(CollectionVectorTest, EmbeddingFieldAlterDropTest) {
- nlohmann::json schema = R"({
- "name": "objects",
- "fields": [
- {"name": "name", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed":{"from": ["name"], "model_config": {"model_name": "ts/e5-small"}}}
- ]
- })"_json;
-
- TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
-
- auto op = collectionManager.create_collection(schema);
- ASSERT_TRUE(op.ok());
- Collection* coll = op.get();
-
- auto& vec_index = coll->_get_index()->_get_vector_index();
- ASSERT_EQ(1, vec_index.size());
- ASSERT_EQ(1, vec_index.count("embedding"));
-
-
- nlohmann::json schema_change = R"({
- "fields": [
- {"name": "embedding", "drop": true}
- ]
- })"_json;
-
- auto schema_change_op = coll->alter(schema_change);
-
- ASSERT_TRUE(schema_change_op.ok());
- ASSERT_EQ(0, vec_index.size());
- ASSERT_EQ(0, vec_index.count("embedding"));
-}
\ No newline at end of file