Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor embedding model init.

Kishore Nallan39738349652f
diff --git a/include/field.h b/include/field.h
index 76982a68..470789f1 100644
--- a/include/field.h
+++ b/include/field.h
@@ -52,7 +52,6 @@ namespace fields {
static const std::string reference = "reference";
static const std::string embed = "embed";
static const std::string from = "from";
- static const std::string embed_from = "embed_from";
static const std::string model_name = "model_name";
// Some models require additional parameters to be passed to the model during indexing/querying
@@ -422,80 +421,7 @@ struct field {
static Option<bool> json_fields_to_fields(bool enable_nested_fields,
nlohmann::json& fields_json,
std::string& fallback_field_type,
- std::vector<field>& the_fields) {
-
- size_t num_auto_detect_fields = 0;
-
- for(nlohmann::json & field_json: fields_json) {
- if(field_json.count(fields::embed) != 0) {
-
- if(!field_json[fields::embed].is_object()) {
- return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");
- }
-
- if(field_json[fields::embed].count(fields::from) == 0) {
- return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");
- }
-
- if(!field_json[fields::embed][fields::from].is_array()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");
- }
-
- if(field_json[fields::embed][fields::from].empty()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");
- }
-
- for(auto& embed_from_field : field_json[fields::embed][fields::from]) {
- if(!embed_from_field.is_string()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");
- }
- }
-
- if(field_json[fields::type] != field_types::FLOAT_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` is only allowed on a float array field.");
- }
-
- for(auto& embed_from_field : field_json[fields::embed][fields::from]) {
- bool flag = false;
- for(const auto& field : fields_json) {
- if(field[fields::name] == embed_from_field) {
- if(field[fields::type] != field_types::STRING && field[fields::type] != field_types::STRING_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
- }
- flag = true;
- break;
- }
- }
- if(!flag) {
- for(const auto& field : the_fields) {
- if(field.name == embed_from_field) {
- if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
- }
- flag = true;
- break;
- }
- }
- }
- if(!flag) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
- }
- }
- }
-
- auto op = json_field_to_field(enable_nested_fields,
- field_json, the_fields, fallback_field_type, num_auto_detect_fields);
- if(!op.ok()) {
- return op;
- }
- }
-
- if(num_auto_detect_fields > 1) {
- return Option<bool>(400,"There can be only one field named `.*`.");
- }
-
- return Option<bool>(true);
- }
+ std::vector<field>& the_fields);
static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,
const field& the_field, const std::string& flat_name,
diff --git a/include/text_embedder.h b/include/text_embedder.h
index eeefea64..d19a3446 100644
--- a/include/text_embedder.h
+++ b/include/text_embedder.h
@@ -22,7 +22,7 @@ class TextEmbedder {
bool is_remote() {
return remote_embedder_ != nullptr;
}
- static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);
+ Option<bool> validate(size_t& num_dims);
private:
std::unique_ptr<Ort::Session> session_;
Ort::Env env_;
@@ -33,7 +33,5 @@ class TextEmbedder {
std::string vocab_file_name;
static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);
std::string output_tensor_name;
- static Option<bool> validate_remote_model(const nlohmann::json& model_config, unsigned int& num_dims);
- static Option<bool> validate_local_or_public_model(const nlohmann::json& model_config, unsigned int& num_dims);
std::mutex mutex_;
};
diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.h
index 18a58824..b9158305 100644
--- a/include/text_embedder_manager.h
+++ b/include/text_embedder_manager.h
@@ -36,6 +36,8 @@ public:
TextEmbedderManager& operator=(const TextEmbedderManager&) = delete;
Option<TextEmbedder*> get_text_embedder(const nlohmann::json& model_config);
+ Option<bool> init_text_embedder(const nlohmann::json& model_config, size_t& num_dim);
+
void delete_text_embedder(const std::string& model_path);
void delete_all_text_embedders();
@@ -61,10 +63,15 @@ public:
static const std::string get_model_namespace(const std::string& model_name);
static const std::string get_model_subdir(const std::string& model_name);
static const bool check_md5(const std::string& file_path, const std::string& target_md5);
- Option<bool> download_public_model(const std::string& model_name);
+ Option<bool> download_public_model(const text_embedding_model& model);
+
+ Option<bool> init_public_model(const std::string& model_name);
+ bool is_public_model(const std::string& model_name);
+ static bool is_remote_model(const std::string& model_name);
- const bool is_public_model(const std::string& model_name);
- static const bool is_remote_model(const std::string& model_name);
+ static Option<bool> validate_and_init_remote_model(const nlohmann::json& model_config, size_t& num_dims);
+ static Option<bool> validate_and_init_local_model(const nlohmann::json& model_config, size_t& num_dims);
+ static Option<bool> validate_and_init_model(const nlohmann::json& model_config, size_t& num_dims);
private:
TextEmbedderManager() = default;
@@ -73,8 +80,6 @@ private:
std::unordered_map<std::string, text_embedding_model> public_models;
std::mutex text_embedders_mutex;
-
static Option<std::string> get_namespace(const std::string& model_name);
-
};
diff --git a/include/text_embedder_remote.h b/include/text_embedder_remote.h
index f212713d..75eb13e2 100644
--- a/include/text_embedder_remote.h
+++ b/include/text_embedder_remote.h
@@ -47,7 +47,7 @@ class OpenAIEmbedder : public RemoteEmbedder {
static constexpr char* OPENAI_CREATE_EMBEDDING = "https://api.openai.com/v1/embeddings";
public:
OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key);
- static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);
+ static Option<bool> is_model_valid(const nlohmann::json& model_config, size_t& num_dims);
embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_try = 2) override;
std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) override;
nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) override;
@@ -64,7 +64,7 @@ class GoogleEmbedder : public RemoteEmbedder {
public:
GoogleEmbedder(const std::string& google_api_key);
- static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);
+ static Option<bool> is_model_valid(const nlohmann::json& model_config, size_t& num_dims);
embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_try = 2) override;
std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) override;
nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) override;
@@ -91,7 +91,7 @@ class GCPEmbedder : public RemoteEmbedder {
public:
GCPEmbedder(const std::string& project_id, const std::string& model_name, const std::string& access_token,
const std::string& refresh_token, const std::string& client_id, const std::string& client_secret);
- static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);
+ static Option<bool> is_model_valid(const nlohmann::json& model_config, size_t& num_dims);
embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_try = 2) override;
std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) override;
nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) override;
diff --git a/src/collection.cpp b/src/collection.cpp
index 8036da34..4713b127 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -261,6 +261,9 @@ nlohmann::json Collection::get_summary_json() const {
field_json[fields::reference] = coll_field.reference;
}
+ if(!coll_field.embed.empty()) {
+ field_json[fields::embed] = coll_field.embed;
+ }
fields_arr.push_back(field_json);
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 1c7e912b..95762e85 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -76,6 +76,20 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
}
}
+ if(field_obj.count(fields::embed) != 0 && !field_obj[fields::embed].empty()) {
+ size_t num_dim = 0;
+ auto& model_config = field_obj[fields::embed][fields::model_config];
+
+ auto res = TextEmbedderManager::validate_and_init_model(model_config, num_dim);
+ if(!res.ok()) {
+ const std::string& model_name = model_config["model_name"].get<std::string>();
+ LOG(ERROR) << "Error initializing model: " << model_name << ", error: " << res.error();
+ continue;
+ }
+
+ LOG(INFO) << "Model init done.";
+ }
+
field f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],
field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],
-1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],
diff --git a/src/field.cpp b/src/field.cpp
index 344347b4..5386378b 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -673,55 +673,6 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
}
}
- if(field_json.count(fields::embed) != 0) {
- // If the model path is not specified, use the default model and set the number of dimensions to 384 (number of dimensions of the default model)
- field_json[fields::num_dim] = static_cast<unsigned int>(384);
-
- auto& embed_json = field_json[fields::embed];
-
- if(embed_json.count(fields::from) == 0) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` not found.");
- }
-
- if(embed_json.count(fields::model_config) == 0) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "` not found.");
- }
-
- auto& model_config = embed_json[fields::model_config];
-
- if(model_config.count(fields::model_name) == 0) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "`not found");
- }
-
- unsigned int num_dim = 0;
- if(!model_config[fields::model_name].is_string()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` must be a string.");
- }
- if(model_config[fields::model_name].get<std::string>().empty()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` cannot be empty.");
- }
-
- if(model_config.count(fields::indexing_prefix) != 0) {
- if(!model_config[fields::indexing_prefix].is_string()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::indexing_prefix + "` must be a string.");
- }
- }
-
- if(model_config.count(fields::query_prefix) != 0) {
- if(!model_config[fields::query_prefix].is_string()) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::query_prefix + "` must be a string.");
- }
- }
-
- auto res = TextEmbedder::is_model_valid(model_config, num_dim);
- if(!res.ok()) {
- return Option<bool>(res.code(), res.error());
- }
- field_json[fields::num_dim] = num_dim;
- } else {
- field_json[fields::embed] = nlohmann::json::object();
- }
-
auto DEFAULT_VEC_DIST_METRIC = magic_enum::enum_name(vector_distance_type_t::cosine);
if(field_json.count(fields::num_dim) == 0) {
@@ -1047,6 +998,120 @@ void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {
}
}
+Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::json &fields_json, string &fallback_field_type,
+ std::vector<field> &the_fields) {
+ size_t num_auto_detect_fields = 0;
+
+ for(nlohmann::json & field_json: fields_json) {
+ if(field_json.count(fields::embed) != 0) {
+ if(!field_json[fields::embed].is_object()) {
+ return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");
+ }
+
+ auto& embed_json = field_json[fields::embed];
+
+ if(field_json[fields::embed].count(fields::from) == 0) {
+ return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");
+ }
+
+ if(!field_json[fields::embed][fields::from].is_array()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");
+ }
+
+ if(field_json[fields::embed][fields::from].empty()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");
+ }
+
+ for(auto& embed_from_field : field_json[fields::embed][fields::from]) {
+ if(!embed_from_field.is_string()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");
+ }
+
+ bool flag = false;
+ for(const auto& field : fields_json) {
+ if(field[fields::name] == embed_from_field) {
+ if(field[fields::type] != field_types::STRING && field[fields::type] != field_types::STRING_ARRAY) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
+ }
+ flag = true;
+ break;
+ }
+ }
+ if(!flag) {
+ for(const auto& field : the_fields) {
+ if(field.name == embed_from_field) {
+ if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
+ }
+ flag = true;
+ break;
+ }
+ }
+ }
+
+ if(!flag) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");
+ }
+ }
+
+ if(field_json[fields::type] != field_types::FLOAT_ARRAY) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` is only allowed on a float array field.");
+ }
+
+ if(embed_json.count(fields::model_config) == 0) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "` not found.");
+ }
+
+ auto& model_config = embed_json[fields::model_config];
+
+ if(model_config.count(fields::model_name) == 0) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "`not found");
+ }
+
+ if(!model_config[fields::model_name].is_string()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` must be a string.");
+ }
+ if(model_config[fields::model_name].get<std::string>().empty()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` cannot be empty.");
+ }
+
+ if(model_config.count(fields::indexing_prefix) != 0) {
+ if(!model_config[fields::indexing_prefix].is_string()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::indexing_prefix + "` must be a string.");
+ }
+ }
+
+ if(model_config.count(fields::query_prefix) != 0) {
+ if(!model_config[fields::query_prefix].is_string()) {
+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::query_prefix + "` must be a string.");
+ }
+ }
+
+ size_t num_dim = 0;
+ auto res = TextEmbedderManager::validate_and_init_model(model_config, num_dim);
+ if(!res.ok()) {
+ return Option<bool>(res.code(), res.error());
+ }
+
+ LOG(INFO) << "Model init done.";
+ embed_json[fields::num_dim] = num_dim;
+ field_json[fields::num_dim] = num_dim;
+ }
+
+ auto op = json_field_to_field(enable_nested_fields,
+ field_json, the_fields, fallback_field_type, num_auto_detect_fields);
+ if(!op.ok()) {
+ return op;
+ }
+ }
+
+ if(num_auto_detect_fields > 1) {
+ return Option<bool>(400,"There can be only one field named `.*`.");
+ }
+
+ return Option<bool>(true);
+}
+
void filter_result_t::and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {
auto lenA = a.count, lenB = b.count;
if (lenA == 0 || lenB == 0) {
diff --git a/src/text_embedder.cpp b/src/text_embedder.cpp
index af84e7ee..1f0752ff 100644
--- a/src/text_embedder.cpp
+++ b/src/text_embedder.cpp
@@ -221,115 +221,72 @@ std::vector<embedding_res_t> TextEmbedder::batch_embed(const std::vector<std::st
return outputs;
}
-TextEmbedder::~TextEmbedder() {
-}
-
-Option<bool> TextEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {
- auto model_name = model_config["model_name"].get<std::string>();
+TextEmbedder::~TextEmbedder() { }
- if(TextEmbedderManager::is_remote_model(model_name)) {
- return validate_remote_model(model_config, num_dims);
- } else {
- return validate_local_or_public_model(model_config, num_dims);
+batch_encoded_input_t TextEmbedder::batch_encode(const std::vector<std::string>& inputs) {
+ batch_encoded_input_t encoded_inputs;
+ for(auto& input : inputs) {
+ auto encoded_input = tokenizer_->Encode(input);
+ encoded_inputs.input_ids.push_back(encoded_input.input_ids);
+ encoded_inputs.attention_mask.push_back(encoded_input.attention_mask);
+ encoded_inputs.token_type_ids.push_back(encoded_input.token_type_ids);
}
-}
-Option<bool> TextEmbedder::validate_remote_model(const nlohmann::json& model_config, unsigned int& num_dims) {
- auto model_name = model_config["model_name"].get<std::string>();
- auto model_namespace = TextEmbedderManager::get_model_namespace(model_name);
-
- if(model_namespace == "openai") {
- return OpenAIEmbedder::is_model_valid(model_config, num_dims);
- } else if(model_namespace == "google") {
- return GoogleEmbedder::is_model_valid(model_config, num_dims);
- } else if(model_namespace == "gcp") {
- return GCPEmbedder::is_model_valid(model_config, num_dims);
+ // Pad inputs
+ size_t max_input_len = 0;
+ for(auto& input_ids : encoded_inputs.input_ids) {
+ if(input_ids.size() > max_input_len) {
+ max_input_len = input_ids.size();
+ }
}
- return Option<bool>(400, "Invalid model namespace");
-}
-
-Option<bool> TextEmbedder::validate_local_or_public_model(const nlohmann::json& model_config, unsigned int& num_dims) {
- auto model_name = model_config["model_name"].get<std::string>();
- LOG(INFO) << "Validating model: " << model_name;
-
- if(TextEmbedderManager::get_instance().is_public_model(model_name)) {
- auto res = TextEmbedderManager::get_instance().download_public_model(model_name);
- if(!res.ok()) {
- LOG(ERROR) << res.error();
- return Option<bool>(400, res.error());
- }
+ for(auto& input_ids : encoded_inputs.input_ids) {
+ input_ids.resize(max_input_len, 0);
}
- Ort::SessionOptions session_options;
- Ort::Env env;
- std::string abs_path = TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::get_model_name_without_namespace(model_name));
-
- if(!std::filesystem::exists(abs_path)) {
- LOG(ERROR) << "Model file not found: " << abs_path;
- return Option<bool>(400, "Model file not found");
+ for(auto& attention_mask : encoded_inputs.attention_mask) {
+ attention_mask.resize(max_input_len, 0);
}
- if(!TextEmbedderManager::get_instance().is_public_model(model_name)) {
- if(!std::filesystem::exists(TextEmbedderManager::get_absolute_config_path(model_name))) {
- LOG(ERROR) << "Config file not found: " << TextEmbedderManager::get_absolute_config_path(model_name);
- return Option<bool>(400, "Config file not found");
- }
- std::ifstream config_file(TextEmbedderManager::get_absolute_config_path(model_name));
- nlohmann::json config;
- config_file >> config;
- if(config["model_type"].is_null() || config["vocab_file_name"].is_null()) {
- LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);
- return Option<bool>(400, "Invalid config file");
- }
-
- if(!config["model_type"].is_string() || !config["vocab_file_name"].is_string()) {
- LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);
- return Option<bool>(400, "Invalid config file");
- }
-
- if(!std::filesystem::exists(TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>())) {
- LOG(ERROR) << "Vocab file not found: " << TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>();
- return Option<bool>(400, "Vocab file not found");
- }
-
- if(config["model_type"].get<std::string>() != "bert" && config["model_type"].get<std::string>() != "xlm_roberta" && config["model_type"].get<std::string>() != "distilbert") {
- LOG(ERROR) << "Invalid model type: " << config["model_type"].get<std::string>();
- return Option<bool>(400, "Invalid model type");
- }
+ for(auto& token_type_ids : encoded_inputs.token_type_ids) {
+ token_type_ids.resize(max_input_len, 0);
}
- Ort::Session session(env, abs_path.c_str(), session_options);
- if(session.GetInputCount() != 3 && session.GetInputCount() != 2) {
+ return encoded_inputs;
+}
+
+Option<bool> TextEmbedder::validate(size_t& num_dims) {
+ if(session_->GetInputCount() != 3 && session_->GetInputCount() != 2) {
LOG(ERROR) << "Invalid model: input count is not 3 or 2";
return Option<bool>(400, "Invalid model: input count is not 3 or 2");
}
+
Ort::AllocatorWithDefaultOptions allocator;
- auto input_ids_name = session.GetInputNameAllocated(0, allocator);
+ auto input_ids_name = session_->GetInputNameAllocated(0, allocator);
if (std::strcmp(input_ids_name.get(), "input_ids") != 0) {
LOG(ERROR) << "Invalid model: input_ids tensor not found";
return Option<bool>(400, "Invalid model: input_ids tensor not found");
}
- auto attention_mask_name = session.GetInputNameAllocated(1, allocator);
+ auto attention_mask_name = session_->GetInputNameAllocated(1, allocator);
if (std::strcmp(attention_mask_name.get(), "attention_mask") != 0) {
LOG(ERROR) << "Invalid model: attention_mask tensor not found";
return Option<bool>(400, "Invalid model: attention_mask tensor not found");
}
-
- if(session.GetInputCount() == 3) {
- auto token_type_ids_name = session.GetInputNameAllocated(2, allocator);
+ if(session_->GetInputCount() == 3) {
+ auto token_type_ids_name = session_->GetInputNameAllocated(2, allocator);
if (std::strcmp(token_type_ids_name.get(), "token_type_ids") != 0) {
LOG(ERROR) << "Invalid model: token_type_ids tensor not found";
return Option<bool>(400, "Invalid model: token_type_ids tensor not found");
}
}
- auto output_tensor_count = session.GetOutputCount();
+ auto output_tensor_count = session_->GetOutputCount();
bool found_output_tensor = false;
+
for (size_t i = 0; i < output_tensor_count; i++) {
- auto shape = session.GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();
+ auto shape = session_->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();
if (shape.size() == 3 && shape[0] == -1 && shape[1] == -1 && shape[2] > 0) {
num_dims = shape[2];
found_output_tensor = true;
@@ -344,36 +301,3 @@ Option<bool> TextEmbedder::validate_local_or_public_model(const nlohmann::json&
return Option<bool>(true);
}
-
-
-batch_encoded_input_t TextEmbedder::batch_encode(const std::vector<std::string>& inputs) {
- batch_encoded_input_t encoded_inputs;
- for(auto& input : inputs) {
- auto encoded_input = tokenizer_->Encode(input);
- encoded_inputs.input_ids.push_back(encoded_input.input_ids);
- encoded_inputs.attention_mask.push_back(encoded_input.attention_mask);
- encoded_inputs.token_type_ids.push_back(encoded_input.token_type_ids);
- }
-
- // Pad inputs
- size_t max_input_len = 0;
- for(auto& input_ids : encoded_inputs.input_ids) {
- if(input_ids.size() > max_input_len) {
- max_input_len = input_ids.size();
- }
- }
-
- for(auto& input_ids : encoded_inputs.input_ids) {
- input_ids.resize(max_input_len, 0);
- }
-
- for(auto& attention_mask : encoded_inputs.attention_mask) {
- attention_mask.resize(max_input_len, 0);
- }
-
- for(auto& token_type_ids : encoded_inputs.token_type_ids) {
- token_type_ids.resize(max_input_len, 0);
- }
-
- return encoded_inputs;
-}
\ No newline at end of file
diff --git a/src/text_embedder_manager.cpp b/src/text_embedder_manager.cpp
index e7e0cf51..372833e3 100644
--- a/src/text_embedder_manager.cpp
+++ b/src/text_embedder_manager.cpp
@@ -6,24 +6,134 @@ TextEmbedderManager& TextEmbedderManager::get_instance() {
return instance;
}
-Option<TextEmbedder*>TextEmbedderManager::get_text_embedder(const nlohmann::json& model_config) {
+Option<bool> TextEmbedderManager::validate_and_init_model(const nlohmann::json& model_config, size_t& num_dims) {
+ const std::string& model_name = model_config["model_name"].get<std::string>();
+
+ if(is_remote_model(model_name)) {
+ LOG(INFO) << "Validating and initializing remote model: " << model_name;
+ return validate_and_init_remote_model(model_config, num_dims);
+ } else {
+ LOG(INFO) << "Validating and initializing local model: " << model_name;
+ return validate_and_init_local_model(model_config, num_dims);
+ }
+}
+
+Option<bool> TextEmbedderManager::validate_and_init_remote_model(const nlohmann::json& model_config,
+ size_t& num_dims) {
+ const std::string& model_name = model_config["model_name"].get<std::string>();
+ auto model_namespace = TextEmbedderManager::get_model_namespace(model_name);
+
+ if(model_namespace == "openai") {
+ auto op = OpenAIEmbedder::is_model_valid(model_config, num_dims);
+ if(!op.ok()) {
+ return op;
+ }
+ } else if(model_namespace == "google") {
+ auto op = GoogleEmbedder::is_model_valid(model_config, num_dims);
+ if(!op.ok()) {
+ return op;
+ }
+ } else if(model_namespace == "gcp") {
+ auto op = GCPEmbedder::is_model_valid(model_config, num_dims);
+ if(!op.ok()) {
+ return op;
+ }
+ } else {
+ return Option<bool>(400, "Invalid model namespace");
+ }
+
+ return TextEmbedderManager::get_instance().init_text_embedder(model_config, num_dims);
+}
+
+Option<bool> TextEmbedderManager::validate_and_init_local_model(const nlohmann::json& model_config, size_t& num_dims) {
+ const std::string& model_name = model_config["model_name"].get<std::string>();
+ Option<bool> public_model_op = TextEmbedderManager::get_instance().init_public_model(model_name);
+
+ if(!public_model_op.ok()) {
+ return public_model_op;
+ }
+
+ Ort::SessionOptions session_options;
+ Ort::Env env;
+ std::string abs_path = TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::get_model_name_without_namespace(model_name));
+
+ if(!std::filesystem::exists(abs_path)) {
+ LOG(ERROR) << "Model file not found: " << abs_path;
+ return Option<bool>(400, "Model file not found");
+ }
+
+ bool is_public_model = public_model_op.get();
+
+ if(!is_public_model) {
+ if(!std::filesystem::exists(TextEmbedderManager::get_absolute_config_path(model_name))) {
+ LOG(ERROR) << "Config file not found: " << TextEmbedderManager::get_absolute_config_path(model_name);
+ return Option<bool>(400, "Config file not found");
+ }
+ std::ifstream config_file(TextEmbedderManager::get_absolute_config_path(model_name));
+ nlohmann::json config;
+ config_file >> config;
+ if(config["model_type"].is_null() || config["vocab_file_name"].is_null()) {
+ LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);
+ return Option<bool>(400, "Invalid config file");
+ }
+
+ if(!config["model_type"].is_string() || !config["vocab_file_name"].is_string()) {
+ LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);
+ return Option<bool>(400, "Invalid config file");
+ }
+
+ if(!std::filesystem::exists(TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>())) {
+ LOG(ERROR) << "Vocab file not found: " << TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>();
+ return Option<bool>(400, "Vocab file not found");
+ }
+
+ if(config["model_type"].get<std::string>() != "bert" && config["model_type"].get<std::string>() != "xlm_roberta" && config["model_type"].get<std::string>() != "distilbert") {
+ LOG(ERROR) << "Invalid model type: " << config["model_type"].get<std::string>();
+ return Option<bool>(400, "Invalid model type");
+ }
+ }
+
+ return TextEmbedderManager::get_instance().init_text_embedder(model_config, num_dims);
+}
+
+Option<bool> TextEmbedderManager::init_text_embedder(const nlohmann::json& model_config, size_t& num_dim) {
std::unique_lock<std::mutex> lock(text_embedders_mutex);
const std::string& model_name = model_config.at("model_name");
- if(text_embedders[model_name] == nullptr) {
- if(!is_remote_model(model_name)) {
- if(is_public_model(model_name)) {
- // download the model if it doesn't exist
- auto res = download_public_model(model_name);
- if(!res.ok()) {
- return Option<TextEmbedder*>(res.code(), res.error());
- }
- }
- text_embedders[model_name] = std::make_shared<TextEmbedder>(get_model_name_without_namespace(model_name));
+ auto text_embedder_it = text_embedders.find(model_name);
+ if(text_embedder_it == text_embedders.end()) {
+ if(is_remote_model(model_name)) {
+ text_embedders.emplace(model_name, std::make_shared<TextEmbedder>(model_config));
} else {
- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_config);
+ const std::shared_ptr<TextEmbedder>& embedder = std::make_shared<TextEmbedder>(
+ get_model_name_without_namespace(model_name));
+
+ auto validate_op = embedder->validate(num_dim);
+ if(!validate_op.ok()) {
+ return validate_op;
+ }
+
+ text_embedders.emplace(model_name, embedder);
+ }
+ } else {
+ auto validate_op = text_embedder_it->second->validate(num_dim);
+ if(!validate_op.ok()) {
+ return validate_op;
}
}
- return Option<TextEmbedder*>(text_embedders[model_name].get());
+
+ return Option<bool>(true);
+}
+
+Option<TextEmbedder*> TextEmbedderManager::get_text_embedder(const nlohmann::json& model_config) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
+ const std::string& model_name = model_config.at("model_name");
+ auto text_embedder_it = text_embedders.find(model_name);
+
+ if(text_embedder_it == text_embedders.end()) {
+ return Option<TextEmbedder*>(404, "Text embedder was not found.");
+ }
+
+ return Option<TextEmbedder*>(text_embedder_it->second.get());
}
void TextEmbedderManager::delete_text_embedder(const std::string& model_path) {
@@ -56,6 +166,7 @@ const TokenizerType TextEmbedderManager::get_tokenizer_type(const nlohmann::json
const std::string TextEmbedderManager::get_indexing_prefix(const nlohmann::json& model_config) {
std::string val;
if(is_public_model(model_config["model_name"].get<std::string>())) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
val = public_models[model_config["model_name"].get<std::string>()].indexing_prefix;
} else {
val = model_config.count("indexing_prefix") == 0 ? "" : model_config["indexing_prefix"].get<std::string>();
@@ -70,6 +181,7 @@ const std::string TextEmbedderManager::get_indexing_prefix(const nlohmann::json&
const std::string TextEmbedderManager::get_query_prefix(const nlohmann::json& model_config) {
std::string val;
if(is_public_model(model_config["model_name"].get<std::string>())) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
val = public_models[model_config["model_name"].get<std::string>()].query_prefix;
} else {
val = model_config.count("query_prefix") == 0 ? "" : model_config["query_prefix"].get<std::string>();
@@ -122,14 +234,14 @@ const bool TextEmbedderManager::check_md5(const std::string& file_path, const st
}
return res.str() == target_md5;
}
-Option<bool> TextEmbedderManager::download_public_model(const std::string& model_name) {
+
+Option<bool> TextEmbedderManager::download_public_model(const text_embedding_model& model) {
HttpClient& httpClient = HttpClient::get_instance();
- auto model = public_models[model_name];
- auto actual_model_name = get_model_name_without_namespace(model_name);
+ auto actual_model_name = get_model_name_without_namespace(model.model_name);
if(!check_md5(get_absolute_model_path(actual_model_name), model.model_md5)) {
long res = httpClient.download_file(get_model_url(model), get_absolute_model_path(actual_model_name));
if(res != 200) {
- LOG(INFO) << "Failed to download public model " << model_name << ": " << res;
+ LOG(INFO) << "Failed to download public model: " << model.model_name;
return Option<bool>(400, "Failed to download model file");
}
}
@@ -137,7 +249,7 @@ Option<bool> TextEmbedderManager::download_public_model(const std::string& model
if(!check_md5(get_absolute_vocab_path(actual_model_name, model.vocab_file_name), model.vocab_md5)) {
long res = httpClient.download_file(get_vocab_url(model), get_absolute_vocab_path(actual_model_name, model.vocab_file_name));
if(res != 200) {
- LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;
+ LOG(INFO) << "Failed to download default vocab for model: " << model.model_name;
return Option<bool>(400, "Failed to download vocab file");
}
}
@@ -145,27 +257,44 @@ Option<bool> TextEmbedderManager::download_public_model(const std::string& model
return Option<bool>(true);
}
-const bool TextEmbedderManager::is_public_model(const std::string& model_name) {
+Option<bool> TextEmbedderManager::init_public_model(const std::string& model_name) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
if(public_models.find(model_name) != public_models.end()) {
- return true;
+ // model has already been initialized
+ return Option<bool>(true);
}
auto model_namespace = get_namespace(model_name);
if(!model_namespace.ok() || model_namespace.get() != "ts") {
- return false;
+ // not a public model
+ return Option<bool>(false);
}
auto actual_model_name = get_model_name_without_namespace(model_name);
auto model_config_op = get_public_model_config(actual_model_name);
- if(model_config_op.ok()) {
- auto config = model_config_op.get();
- config["model_name"] = actual_model_name;
- public_models[model_name] = text_embedding_model(config);
- return true;
+ if(!model_config_op.ok()) {
+ return Option<bool>(model_config_op.code(), model_config_op.error());
}
- return false;
+ auto config = model_config_op.get();
+ config["model_name"] = actual_model_name;
+
+ auto model = text_embedding_model(config);
+
+ auto download_op = TextEmbedderManager::get_instance().download_public_model(model);
+ if (!download_op.ok()) {
+ LOG(ERROR) << download_op.error();
+ return Option<bool>(400, download_op.error());
+ }
+
+ public_models.emplace(model_name, text_embedding_model(config));
+ return Option<bool>(true);;
+}
+
+bool TextEmbedderManager::is_public_model(const std::string& model_name) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
+ return public_models.find(model_name) != public_models.end();
}
const std::string TextEmbedderManager::get_model_subdir(const std::string& model_name) {
@@ -264,7 +393,7 @@ const std::string TextEmbedderManager::get_model_namespace(const std::string& mo
}
}
-const bool TextEmbedderManager::is_remote_model(const std::string& model_name) {
+bool TextEmbedderManager::is_remote_model(const std::string& model_name) {
auto model_namespace = get_namespace(model_name);
return model_namespace.ok() && (model_namespace.get() == "openai" || model_namespace.get() == "google" || model_namespace.get() == "gcp");
-}
\ No newline at end of file
+}
diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cpp
index b24a3b98..dab13102 100644
--- a/src/text_embedder_remote.cpp
+++ b/src/text_embedder_remote.cpp
@@ -57,7 +57,7 @@ OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::
}
-Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {
+Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config, size_t& num_dims) {
auto validate_properties = validate_string_properties(model_config, {"model_name", "api_key"});
if (!validate_properties.ok()) {
@@ -260,7 +260,7 @@ GoogleEmbedder::GoogleEmbedder(const std::string& google_api_key) : google_api_k
}
-Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {
+Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config, size_t& num_dims) {
auto validate_properties = validate_string_properties(model_config, {"model_name", "api_key"});
if (!validate_properties.ok()) {
@@ -380,7 +380,7 @@ GCPEmbedder::GCPEmbedder(const std::string& project_id, const std::string& model
this->model_name = TextEmbedderManager::get_model_name_without_namespace(model_name);
}
-Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {
+Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, size_t& num_dims) {
auto validate_properties = validate_string_properties(model_config, {"model_name", "project_id", "access_token", "refresh_token", "client_id", "client_secret"});
if (!validate_properties.ok()) {
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index c3817ad5..15e193a3 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -1650,17 +1650,18 @@ TEST_F(CollectionAllFieldsTest, ModelParametersWithoutEmbedFrom) {
ASSERT_EQ("Property `embed` must contain a `from` property.", field_op.error());
}
-
TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {
-
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
-
- field embedding = field("embedding", field_types::FLOAT_ARRAY, false);
- embedding.embed["from"].push_back("name");
- embedding.embed["model_config"]["model_name"] = "ts/e5-small";
- std::vector<field> fields = {field("name", field_types::STRING, false),
- embedding};
- auto obj_coll_op = collectionManager.create_collection("obj_coll", 1, fields, "", 0, field_types::AUTO);
+ nlohmann::json schema = R"({
+ "name": "obj_coll",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"],
+ "model_config": {"model_name": "ts/e5-small"}}}
+ ]
+ })"_json;
+
+ auto obj_coll_op = collectionManager.create_collection(schema);
ASSERT_TRUE(obj_coll_op.ok());
Collection* obj_coll = obj_coll_op.get();
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index d27a78df..a1863b20 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -4816,16 +4816,14 @@ TEST_F(CollectionTest, WildcardSearchWithEmbeddingField) {
}
TEST_F(CollectionTest, CreateModelDirIfNotExists) {
- system("mkdir -p /tmp/typesense_test/models");
- system("rm -rf /tmp/typesense_test/models");
- TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
+ system("mkdir -p /tmp/typesense_test/new_models_dir");
+ system("rm -rf /tmp/typesense_test/new_models_dir");
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/new_models_dir");
// check if model dir is created
- ASSERT_TRUE(std::filesystem::exists("/tmp/typesense_test/models"));
+ ASSERT_TRUE(std::filesystem::exists("/tmp/typesense_test/new_models_dir"));
}
-
-
TEST_F(CollectionTest, EmbedStringArrayField) {
nlohmann::json schema = R"({
"name": "objects",