Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor embedding model init.
Kishore Nallan39738349652f
diff --git a/include/field.h b/include/field.hindex 76982a68..470789f1 100644--- a/include/field.h+++ b/include/field.h@@ -52,7 +52,6 @@ namespace fields {static const std::string reference = "reference";static const std::string embed = "embed";static const std::string from = "from";- static const std::string embed_from = "embed_from";static const std::string model_name = "model_name";// Some models require additional parameters to be passed to the model during indexing/querying@@ -422,80 +421,7 @@ struct field {static Option<bool> json_fields_to_fields(bool enable_nested_fields,nlohmann::json& fields_json,std::string& fallback_field_type,- std::vector<field>& the_fields) {-- size_t num_auto_detect_fields = 0;-- for(nlohmann::json & field_json: fields_json) {- if(field_json.count(fields::embed) != 0) {-- if(!field_json[fields::embed].is_object()) {- return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");- }-- if(field_json[fields::embed].count(fields::from) == 0) {- return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");- }-- if(!field_json[fields::embed][fields::from].is_array()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");- }-- if(field_json[fields::embed][fields::from].empty()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");- }-- for(auto& embed_from_field : field_json[fields::embed][fields::from]) {- if(!embed_from_field.is_string()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");- }- }-- if(field_json[fields::type] != field_types::FLOAT_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` is only allowed on a float array field.");- }-- for(auto& embed_from_field : field_json[fields::embed][fields::from]) {- bool flag = false;- for(const auto& field : fields_json) {- if(field[fields::name] == embed_from_field) {- if(field[fields::type] != field_types::STRING && field[fields::type] != field_types::STRING_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");- }- flag = true;- break;- }- }- if(!flag) {- for(const auto& field : the_fields) {- if(field.name == embed_from_field) {- if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");- }- flag = true;- break;- }- }- }- if(!flag) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");- }- }- }-- auto op = json_field_to_field(enable_nested_fields,- field_json, the_fields, fallback_field_type, num_auto_detect_fields);- if(!op.ok()) {- return op;- }- }-- if(num_auto_detect_fields > 1) {- return Option<bool>(400,"There can be only one field named `.*`.");- }-- return Option<bool>(true);- }+ std::vector<field>& the_fields);static bool flatten_obj(nlohmann::json& doc, nlohmann::json& value, bool has_array, bool has_obj_array,const field& the_field, const std::string& flat_name,diff --git a/include/text_embedder.h b/include/text_embedder.hindex eeefea64..d19a3446 100644--- a/include/text_embedder.h+++ b/include/text_embedder.h@@ -22,7 +22,7 @@ class TextEmbedder {bool is_remote() {return remote_embedder_ != nullptr;}- static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);+ Option<bool> validate(size_t& num_dims);private:std::unique_ptr<Ort::Session> session_;Ort::Env env_;@@ -33,7 +33,5 @@ class TextEmbedder {std::string vocab_file_name;static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);std::string output_tensor_name;- static Option<bool> validate_remote_model(const nlohmann::json& model_config, unsigned int& num_dims);- static Option<bool> validate_local_or_public_model(const nlohmann::json& model_config, unsigned int& num_dims);std::mutex mutex_;};diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.hindex 18a58824..b9158305 100644--- a/include/text_embedder_manager.h+++ b/include/text_embedder_manager.h@@ -36,6 +36,8 @@ public:TextEmbedderManager& operator=(const TextEmbedderManager&) = delete;Option<TextEmbedder*> get_text_embedder(const nlohmann::json& model_config);+ Option<bool> init_text_embedder(const nlohmann::json& model_config, size_t& num_dim);+void delete_text_embedder(const std::string& model_path);void delete_all_text_embedders();@@ -61,10 +63,15 @@ public:static const std::string get_model_namespace(const std::string& model_name);static const std::string get_model_subdir(const std::string& model_name);static const bool check_md5(const std::string& file_path, const std::string& target_md5);- Option<bool> download_public_model(const std::string& model_name);+ Option<bool> download_public_model(const text_embedding_model& model);++ Option<bool> init_public_model(const std::string& model_name);+ bool is_public_model(const std::string& model_name);+ static bool is_remote_model(const std::string& model_name);- const bool is_public_model(const std::string& model_name);- static const bool is_remote_model(const std::string& model_name);+ static Option<bool> validate_and_init_remote_model(const nlohmann::json& model_config, size_t& num_dims);+ static Option<bool> validate_and_init_local_model(const nlohmann::json& model_config, size_t& num_dims);+ static Option<bool> validate_and_init_model(const nlohmann::json& model_config, size_t& num_dims);private:TextEmbedderManager() = default;@@ -73,8 +80,6 @@ private:std::unordered_map<std::string, text_embedding_model> public_models;std::mutex text_embedders_mutex;-static Option<std::string> get_namespace(const std::string& model_name);-};diff --git a/include/text_embedder_remote.h b/include/text_embedder_remote.hindex f212713d..75eb13e2 100644--- a/include/text_embedder_remote.h+++ b/include/text_embedder_remote.h@@ -47,7 +47,7 @@ class OpenAIEmbedder : public RemoteEmbedder {static constexpr char* OPENAI_CREATE_EMBEDDING = "https://api.openai.com/v1/embeddings";public:OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key);- static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);+ static Option<bool> is_model_valid(const nlohmann::json& model_config, size_t& num_dims);embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_try = 2) override;std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) override;nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) override;@@ -64,7 +64,7 @@ class GoogleEmbedder : public RemoteEmbedder {public:GoogleEmbedder(const std::string& google_api_key);- static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);+ static Option<bool> is_model_valid(const nlohmann::json& model_config, size_t& num_dims);embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_try = 2) override;std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) override;nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) override;@@ -91,7 +91,7 @@ class GCPEmbedder : public RemoteEmbedder {public:GCPEmbedder(const std::string& project_id, const std::string& model_name, const std::string& access_token,const std::string& refresh_token, const std::string& client_id, const std::string& client_secret);- static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);+ static Option<bool> is_model_valid(const nlohmann::json& model_config, size_t& num_dims);embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_try = 2) override;std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) override;nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) override;diff --git a/src/collection.cpp b/src/collection.cppindex 8036da34..4713b127 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -261,6 +261,9 @@ nlohmann::json Collection::get_summary_json() const {field_json[fields::reference] = coll_field.reference;}+ if(!coll_field.embed.empty()) {+ field_json[fields::embed] = coll_field.embed;+ }fields_arr.push_back(field_json);}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 1c7e912b..95762e85 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -76,6 +76,20 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection}}+ if(field_obj.count(fields::embed) != 0 && !field_obj[fields::embed].empty()) {+ size_t num_dim = 0;+ auto& model_config = field_obj[fields::embed][fields::model_config];++ auto res = TextEmbedderManager::validate_and_init_model(model_config, num_dim);+ if(!res.ok()) {+ const std::string& model_name = model_config["model_name"].get<std::string>();+ LOG(ERROR) << "Error initializing model: " << model_name << ", error: " << res.error();+ continue;+ }++ LOG(INFO) << "Model init done.";+ }+field f(field_obj[fields::name], field_obj[fields::type], field_obj[fields::facet],field_obj[fields::optional], field_obj[fields::index], field_obj[fields::locale],-1, field_obj[fields::infix], field_obj[fields::nested], field_obj[fields::nested_array],diff --git a/src/field.cpp b/src/field.cppindex 344347b4..5386378b 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -673,55 +673,6 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}}- if(field_json.count(fields::embed) != 0) {- // If the model path is not specified, use the default model and set the number of dimensions to 384 (number of dimensions of the default model)- field_json[fields::num_dim] = static_cast<unsigned int>(384);-- auto& embed_json = field_json[fields::embed];-- if(embed_json.count(fields::from) == 0) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` not found.");- }-- if(embed_json.count(fields::model_config) == 0) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "` not found.");- }-- auto& model_config = embed_json[fields::model_config];-- if(model_config.count(fields::model_name) == 0) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "`not found");- }-- unsigned int num_dim = 0;- if(!model_config[fields::model_name].is_string()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` must be a string.");- }- if(model_config[fields::model_name].get<std::string>().empty()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` cannot be empty.");- }-- if(model_config.count(fields::indexing_prefix) != 0) {- if(!model_config[fields::indexing_prefix].is_string()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::indexing_prefix + "` must be a string.");- }- }-- if(model_config.count(fields::query_prefix) != 0) {- if(!model_config[fields::query_prefix].is_string()) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::query_prefix + "` must be a string.");- }- }-- auto res = TextEmbedder::is_model_valid(model_config, num_dim);- if(!res.ok()) {- return Option<bool>(res.code(), res.error());- }- field_json[fields::num_dim] = num_dim;- } else {- field_json[fields::embed] = nlohmann::json::object();- }-auto DEFAULT_VEC_DIST_METRIC = magic_enum::enum_name(vector_distance_type_t::cosine);if(field_json.count(fields::num_dim) == 0) {@@ -1047,6 +998,120 @@ void field::compact_nested_fields(tsl::htrie_map<char, field>& nested_fields) {}}+Option<bool> field::json_fields_to_fields(bool enable_nested_fields, nlohmann::json &fields_json, string &fallback_field_type,+ std::vector<field> &the_fields) {+ size_t num_auto_detect_fields = 0;++ for(nlohmann::json & field_json: fields_json) {+ if(field_json.count(fields::embed) != 0) {+ if(!field_json[fields::embed].is_object()) {+ return Option<bool>(400, "Property `" + fields::embed + "` must be an object.");+ }++ auto& embed_json = field_json[fields::embed];++ if(field_json[fields::embed].count(fields::from) == 0) {+ return Option<bool>(400, "Property `" + fields::embed + "` must contain a `" + fields::from + "` property.");+ }++ if(!field_json[fields::embed][fields::from].is_array()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must be an array.");+ }++ if(field_json[fields::embed][fields::from].empty()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must have at least one element.");+ }++ for(auto& embed_from_field : field_json[fields::embed][fields::from]) {+ if(!embed_from_field.is_string()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` must contain only field names as strings.");+ }++ bool flag = false;+ for(const auto& field : fields_json) {+ if(field[fields::name] == embed_from_field) {+ if(field[fields::type] != field_types::STRING && field[fields::type] != field_types::STRING_ARRAY) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");+ }+ flag = true;+ break;+ }+ }+ if(!flag) {+ for(const auto& field : the_fields) {+ if(field.name == embed_from_field) {+ if(field.type != field_types::STRING && field.type != field_types::STRING_ARRAY) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");+ }+ flag = true;+ break;+ }+ }+ }++ if(!flag) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` can only refer to string or string array fields.");+ }+ }++ if(field_json[fields::type] != field_types::FLOAT_ARRAY) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::from + "` is only allowed on a float array field.");+ }++ if(embed_json.count(fields::model_config) == 0) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "` not found.");+ }++ auto& model_config = embed_json[fields::model_config];++ if(model_config.count(fields::model_name) == 0) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "`not found");+ }++ if(!model_config[fields::model_name].is_string()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` must be a string.");+ }+ if(model_config[fields::model_name].get<std::string>().empty()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` cannot be empty.");+ }++ if(model_config.count(fields::indexing_prefix) != 0) {+ if(!model_config[fields::indexing_prefix].is_string()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::indexing_prefix + "` must be a string.");+ }+ }++ if(model_config.count(fields::query_prefix) != 0) {+ if(!model_config[fields::query_prefix].is_string()) {+ return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::query_prefix + "` must be a string.");+ }+ }++ size_t num_dim = 0;+ auto res = TextEmbedderManager::validate_and_init_model(model_config, num_dim);+ if(!res.ok()) {+ return Option<bool>(res.code(), res.error());+ }++ LOG(INFO) << "Model init done.";+ embed_json[fields::num_dim] = num_dim;+ field_json[fields::num_dim] = num_dim;+ }++ auto op = json_field_to_field(enable_nested_fields,+ field_json, the_fields, fallback_field_type, num_auto_detect_fields);+ if(!op.ok()) {+ return op;+ }+ }++ if(num_auto_detect_fields > 1) {+ return Option<bool>(400,"There can be only one field named `.*`.");+ }++ return Option<bool>(true);+}+void filter_result_t::and_filter_results(const filter_result_t& a, const filter_result_t& b, filter_result_t& result) {auto lenA = a.count, lenB = b.count;if (lenA == 0 || lenB == 0) {diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex af84e7ee..1f0752ff 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -221,115 +221,72 @@ std::vector<embedding_res_t> TextEmbedder::batch_embed(const std::vector<std::streturn outputs;}-TextEmbedder::~TextEmbedder() {-}--Option<bool> TextEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {- auto model_name = model_config["model_name"].get<std::string>();+TextEmbedder::~TextEmbedder() { }- if(TextEmbedderManager::is_remote_model(model_name)) {- return validate_remote_model(model_config, num_dims);- } else {- return validate_local_or_public_model(model_config, num_dims);+batch_encoded_input_t TextEmbedder::batch_encode(const std::vector<std::string>& inputs) {+ batch_encoded_input_t encoded_inputs;+ for(auto& input : inputs) {+ auto encoded_input = tokenizer_->Encode(input);+ encoded_inputs.input_ids.push_back(encoded_input.input_ids);+ encoded_inputs.attention_mask.push_back(encoded_input.attention_mask);+ encoded_inputs.token_type_ids.push_back(encoded_input.token_type_ids);}-}-Option<bool> TextEmbedder::validate_remote_model(const nlohmann::json& model_config, unsigned int& num_dims) {- auto model_name = model_config["model_name"].get<std::string>();- auto model_namespace = TextEmbedderManager::get_model_namespace(model_name);-- if(model_namespace == "openai") {- return OpenAIEmbedder::is_model_valid(model_config, num_dims);- } else if(model_namespace == "google") {- return GoogleEmbedder::is_model_valid(model_config, num_dims);- } else if(model_namespace == "gcp") {- return GCPEmbedder::is_model_valid(model_config, num_dims);+ // Pad inputs+ size_t max_input_len = 0;+ for(auto& input_ids : encoded_inputs.input_ids) {+ if(input_ids.size() > max_input_len) {+ max_input_len = input_ids.size();+ }}- return Option<bool>(400, "Invalid model namespace");-}--Option<bool> TextEmbedder::validate_local_or_public_model(const nlohmann::json& model_config, unsigned int& num_dims) {- auto model_name = model_config["model_name"].get<std::string>();- LOG(INFO) << "Validating model: " << model_name;-- if(TextEmbedderManager::get_instance().is_public_model(model_name)) {- auto res = TextEmbedderManager::get_instance().download_public_model(model_name);- if(!res.ok()) {- LOG(ERROR) << res.error();- return Option<bool>(400, res.error());- }+ for(auto& input_ids : encoded_inputs.input_ids) {+ input_ids.resize(max_input_len, 0);}- Ort::SessionOptions session_options;- Ort::Env env;- std::string abs_path = TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::get_model_name_without_namespace(model_name));-- if(!std::filesystem::exists(abs_path)) {- LOG(ERROR) << "Model file not found: " << abs_path;- return Option<bool>(400, "Model file not found");+ for(auto& attention_mask : encoded_inputs.attention_mask) {+ attention_mask.resize(max_input_len, 0);}- if(!TextEmbedderManager::get_instance().is_public_model(model_name)) {- if(!std::filesystem::exists(TextEmbedderManager::get_absolute_config_path(model_name))) {- LOG(ERROR) << "Config file not found: " << TextEmbedderManager::get_absolute_config_path(model_name);- return Option<bool>(400, "Config file not found");- }- std::ifstream config_file(TextEmbedderManager::get_absolute_config_path(model_name));- nlohmann::json config;- config_file >> config;- if(config["model_type"].is_null() || config["vocab_file_name"].is_null()) {- LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);- return Option<bool>(400, "Invalid config file");- }-- if(!config["model_type"].is_string() || !config["vocab_file_name"].is_string()) {- LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);- return Option<bool>(400, "Invalid config file");- }-- if(!std::filesystem::exists(TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>())) {- LOG(ERROR) << "Vocab file not found: " << TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>();- return Option<bool>(400, "Vocab file not found");- }-- if(config["model_type"].get<std::string>() != "bert" && config["model_type"].get<std::string>() != "xlm_roberta" && config["model_type"].get<std::string>() != "distilbert") {- LOG(ERROR) << "Invalid model type: " << config["model_type"].get<std::string>();- return Option<bool>(400, "Invalid model type");- }+ for(auto& token_type_ids : encoded_inputs.token_type_ids) {+ token_type_ids.resize(max_input_len, 0);}- Ort::Session session(env, abs_path.c_str(), session_options);- if(session.GetInputCount() != 3 && session.GetInputCount() != 2) {+ return encoded_inputs;+}++Option<bool> TextEmbedder::validate(size_t& num_dims) {+ if(session_->GetInputCount() != 3 && session_->GetInputCount() != 2) {LOG(ERROR) << "Invalid model: input count is not 3 or 2";return Option<bool>(400, "Invalid model: input count is not 3 or 2");}+Ort::AllocatorWithDefaultOptions allocator;- auto input_ids_name = session.GetInputNameAllocated(0, allocator);+ auto input_ids_name = session_->GetInputNameAllocated(0, allocator);if (std::strcmp(input_ids_name.get(), "input_ids") != 0) {LOG(ERROR) << "Invalid model: input_ids tensor not found";return Option<bool>(400, "Invalid model: input_ids tensor not found");}- auto attention_mask_name = session.GetInputNameAllocated(1, allocator);+ auto attention_mask_name = session_->GetInputNameAllocated(1, allocator);if (std::strcmp(attention_mask_name.get(), "attention_mask") != 0) {LOG(ERROR) << "Invalid model: attention_mask tensor not found";return Option<bool>(400, "Invalid model: attention_mask tensor not found");}-- if(session.GetInputCount() == 3) {- auto token_type_ids_name = session.GetInputNameAllocated(2, allocator);+ if(session_->GetInputCount() == 3) {+ auto token_type_ids_name = session_->GetInputNameAllocated(2, allocator);if (std::strcmp(token_type_ids_name.get(), "token_type_ids") != 0) {LOG(ERROR) << "Invalid model: token_type_ids tensor not found";return Option<bool>(400, "Invalid model: token_type_ids tensor not found");}}- auto output_tensor_count = session.GetOutputCount();+ auto output_tensor_count = session_->GetOutputCount();bool found_output_tensor = false;+for (size_t i = 0; i < output_tensor_count; i++) {- auto shape = session.GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();+ auto shape = session_->GetOutputTypeInfo(i).GetTensorTypeAndShapeInfo().GetShape();if (shape.size() == 3 && shape[0] == -1 && shape[1] == -1 && shape[2] > 0) {num_dims = shape[2];found_output_tensor = true;@@ -344,36 +301,3 @@ Option<bool> TextEmbedder::validate_local_or_public_model(const nlohmann::json&return Option<bool>(true);}---batch_encoded_input_t TextEmbedder::batch_encode(const std::vector<std::string>& inputs) {- batch_encoded_input_t encoded_inputs;- for(auto& input : inputs) {- auto encoded_input = tokenizer_->Encode(input);- encoded_inputs.input_ids.push_back(encoded_input.input_ids);- encoded_inputs.attention_mask.push_back(encoded_input.attention_mask);- encoded_inputs.token_type_ids.push_back(encoded_input.token_type_ids);- }-- // Pad inputs- size_t max_input_len = 0;- for(auto& input_ids : encoded_inputs.input_ids) {- if(input_ids.size() > max_input_len) {- max_input_len = input_ids.size();- }- }-- for(auto& input_ids : encoded_inputs.input_ids) {- input_ids.resize(max_input_len, 0);- }-- for(auto& attention_mask : encoded_inputs.attention_mask) {- attention_mask.resize(max_input_len, 0);- }-- for(auto& token_type_ids : encoded_inputs.token_type_ids) {- token_type_ids.resize(max_input_len, 0);- }-- return encoded_inputs;-}\ No newline at end of filediff --git a/src/text_embedder_manager.cpp b/src/text_embedder_manager.cppindex e7e0cf51..372833e3 100644--- a/src/text_embedder_manager.cpp+++ b/src/text_embedder_manager.cpp@@ -6,24 +6,134 @@ TextEmbedderManager& TextEmbedderManager::get_instance() {return instance;}-Option<TextEmbedder*>TextEmbedderManager::get_text_embedder(const nlohmann::json& model_config) {+Option<bool> TextEmbedderManager::validate_and_init_model(const nlohmann::json& model_config, size_t& num_dims) {+ const std::string& model_name = model_config["model_name"].get<std::string>();++ if(is_remote_model(model_name)) {+ LOG(INFO) << "Validating and initializing remote model: " << model_name;+ return validate_and_init_remote_model(model_config, num_dims);+ } else {+ LOG(INFO) << "Validating and initializing local model: " << model_name;+ return validate_and_init_local_model(model_config, num_dims);+ }+}++Option<bool> TextEmbedderManager::validate_and_init_remote_model(const nlohmann::json& model_config,+ size_t& num_dims) {+ const std::string& model_name = model_config["model_name"].get<std::string>();+ auto model_namespace = TextEmbedderManager::get_model_namespace(model_name);++ if(model_namespace == "openai") {+ auto op = OpenAIEmbedder::is_model_valid(model_config, num_dims);+ if(!op.ok()) {+ return op;+ }+ } else if(model_namespace == "google") {+ auto op = GoogleEmbedder::is_model_valid(model_config, num_dims);+ if(!op.ok()) {+ return op;+ }+ } else if(model_namespace == "gcp") {+ auto op = GCPEmbedder::is_model_valid(model_config, num_dims);+ if(!op.ok()) {+ return op;+ }+ } else {+ return Option<bool>(400, "Invalid model namespace");+ }++ return TextEmbedderManager::get_instance().init_text_embedder(model_config, num_dims);+}++Option<bool> TextEmbedderManager::validate_and_init_local_model(const nlohmann::json& model_config, size_t& num_dims) {+ const std::string& model_name = model_config["model_name"].get<std::string>();+ Option<bool> public_model_op = TextEmbedderManager::get_instance().init_public_model(model_name);++ if(!public_model_op.ok()) {+ return public_model_op;+ }++ Ort::SessionOptions session_options;+ Ort::Env env;+ std::string abs_path = TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::get_model_name_without_namespace(model_name));++ if(!std::filesystem::exists(abs_path)) {+ LOG(ERROR) << "Model file not found: " << abs_path;+ return Option<bool>(400, "Model file not found");+ }++ bool is_public_model = public_model_op.get();++ if(!is_public_model) {+ if(!std::filesystem::exists(TextEmbedderManager::get_absolute_config_path(model_name))) {+ LOG(ERROR) << "Config file not found: " << TextEmbedderManager::get_absolute_config_path(model_name);+ return Option<bool>(400, "Config file not found");+ }+ std::ifstream config_file(TextEmbedderManager::get_absolute_config_path(model_name));+ nlohmann::json config;+ config_file >> config;+ if(config["model_type"].is_null() || config["vocab_file_name"].is_null()) {+ LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);+ return Option<bool>(400, "Invalid config file");+ }++ if(!config["model_type"].is_string() || !config["vocab_file_name"].is_string()) {+ LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);+ return Option<bool>(400, "Invalid config file");+ }++ if(!std::filesystem::exists(TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>())) {+ LOG(ERROR) << "Vocab file not found: " << TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>();+ return Option<bool>(400, "Vocab file not found");+ }++ if(config["model_type"].get<std::string>() != "bert" && config["model_type"].get<std::string>() != "xlm_roberta" && config["model_type"].get<std::string>() != "distilbert") {+ LOG(ERROR) << "Invalid model type: " << config["model_type"].get<std::string>();+ return Option<bool>(400, "Invalid model type");+ }+ }++ return TextEmbedderManager::get_instance().init_text_embedder(model_config, num_dims);+}++Option<bool> TextEmbedderManager::init_text_embedder(const nlohmann::json& model_config, size_t& num_dim) {std::unique_lock<std::mutex> lock(text_embedders_mutex);const std::string& model_name = model_config.at("model_name");- if(text_embedders[model_name] == nullptr) {- if(!is_remote_model(model_name)) {- if(is_public_model(model_name)) {- // download the model if it doesn't exist- auto res = download_public_model(model_name);- if(!res.ok()) {- return Option<TextEmbedder*>(res.code(), res.error());- }- }- text_embedders[model_name] = std::make_shared<TextEmbedder>(get_model_name_without_namespace(model_name));+ auto text_embedder_it = text_embedders.find(model_name);+ if(text_embedder_it == text_embedders.end()) {+ if(is_remote_model(model_name)) {+ text_embedders.emplace(model_name, std::make_shared<TextEmbedder>(model_config));} else {- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_config);+ const std::shared_ptr<TextEmbedder>& embedder = std::make_shared<TextEmbedder>(+ get_model_name_without_namespace(model_name));++ auto validate_op = embedder->validate(num_dim);+ if(!validate_op.ok()) {+ return validate_op;+ }++ text_embedders.emplace(model_name, embedder);+ }+ } else {+ auto validate_op = text_embedder_it->second->validate(num_dim);+ if(!validate_op.ok()) {+ return validate_op;}}- return Option<TextEmbedder*>(text_embedders[model_name].get());++ return Option<bool>(true);+}++Option<TextEmbedder*> TextEmbedderManager::get_text_embedder(const nlohmann::json& model_config) {+ std::unique_lock<std::mutex> lock(text_embedders_mutex);+ const std::string& model_name = model_config.at("model_name");+ auto text_embedder_it = text_embedders.find(model_name);++ if(text_embedder_it == text_embedders.end()) {+ return Option<TextEmbedder*>(404, "Text embedder was not found.");+ }++ return Option<TextEmbedder*>(text_embedder_it->second.get());}void TextEmbedderManager::delete_text_embedder(const std::string& model_path) {@@ -56,6 +166,7 @@ const TokenizerType TextEmbedderManager::get_tokenizer_type(const nlohmann::jsonconst std::string TextEmbedderManager::get_indexing_prefix(const nlohmann::json& model_config) {std::string val;if(is_public_model(model_config["model_name"].get<std::string>())) {+ std::unique_lock<std::mutex> lock(text_embedders_mutex);val = public_models[model_config["model_name"].get<std::string>()].indexing_prefix;} else {val = model_config.count("indexing_prefix") == 0 ? "" : model_config["indexing_prefix"].get<std::string>();@@ -70,6 +181,7 @@ const std::string TextEmbedderManager::get_indexing_prefix(const nlohmann::json&const std::string TextEmbedderManager::get_query_prefix(const nlohmann::json& model_config) {std::string val;if(is_public_model(model_config["model_name"].get<std::string>())) {+ std::unique_lock<std::mutex> lock(text_embedders_mutex);val = public_models[model_config["model_name"].get<std::string>()].query_prefix;} else {val = model_config.count("query_prefix") == 0 ? "" : model_config["query_prefix"].get<std::string>();@@ -122,14 +234,14 @@ const bool TextEmbedderManager::check_md5(const std::string& file_path, const st}return res.str() == target_md5;}-Option<bool> TextEmbedderManager::download_public_model(const std::string& model_name) {++Option<bool> TextEmbedderManager::download_public_model(const text_embedding_model& model) {HttpClient& httpClient = HttpClient::get_instance();- auto model = public_models[model_name];- auto actual_model_name = get_model_name_without_namespace(model_name);+ auto actual_model_name = get_model_name_without_namespace(model.model_name);if(!check_md5(get_absolute_model_path(actual_model_name), model.model_md5)) {long res = httpClient.download_file(get_model_url(model), get_absolute_model_path(actual_model_name));if(res != 200) {- LOG(INFO) << "Failed to download public model " << model_name << ": " << res;+ LOG(INFO) << "Failed to download public model: " << model.model_name;return Option<bool>(400, "Failed to download model file");}}@@ -137,7 +249,7 @@ Option<bool> TextEmbedderManager::download_public_model(const std::string& modelif(!check_md5(get_absolute_vocab_path(actual_model_name, model.vocab_file_name), model.vocab_md5)) {long res = httpClient.download_file(get_vocab_url(model), get_absolute_vocab_path(actual_model_name, model.vocab_file_name));if(res != 200) {- LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;+ LOG(INFO) << "Failed to download default vocab for model: " << model.model_name;return Option<bool>(400, "Failed to download vocab file");}}@@ -145,27 +257,44 @@ Option<bool> TextEmbedderManager::download_public_model(const std::string& modelreturn Option<bool>(true);}-const bool TextEmbedderManager::is_public_model(const std::string& model_name) {+Option<bool> TextEmbedderManager::init_public_model(const std::string& model_name) {+ std::unique_lock<std::mutex> lock(text_embedders_mutex);if(public_models.find(model_name) != public_models.end()) {- return true;+ // model has already been initialized+ return Option<bool>(true);}auto model_namespace = get_namespace(model_name);if(!model_namespace.ok() || model_namespace.get() != "ts") {- return false;+ // not a public model+ return Option<bool>(false);}auto actual_model_name = get_model_name_without_namespace(model_name);auto model_config_op = get_public_model_config(actual_model_name);- if(model_config_op.ok()) {- auto config = model_config_op.get();- config["model_name"] = actual_model_name;- public_models[model_name] = text_embedding_model(config);- return true;+ if(!model_config_op.ok()) {+ return Option<bool>(model_config_op.code(), model_config_op.error());}- return false;+ auto config = model_config_op.get();+ config["model_name"] = actual_model_name;++ auto model = text_embedding_model(config);++ auto download_op = TextEmbedderManager::get_instance().download_public_model(model);+ if (!download_op.ok()) {+ LOG(ERROR) << download_op.error();+ return Option<bool>(400, download_op.error());+ }++ public_models.emplace(model_name, text_embedding_model(config));+ return Option<bool>(true);;+}++bool TextEmbedderManager::is_public_model(const std::string& model_name) {+ std::unique_lock<std::mutex> lock(text_embedders_mutex);+ return public_models.find(model_name) != public_models.end();}const std::string TextEmbedderManager::get_model_subdir(const std::string& model_name) {@@ -264,7 +393,7 @@ const std::string TextEmbedderManager::get_model_namespace(const std::string& mo}}-const bool TextEmbedderManager::is_remote_model(const std::string& model_name) {+bool TextEmbedderManager::is_remote_model(const std::string& model_name) {auto model_namespace = get_namespace(model_name);return model_namespace.ok() && (model_namespace.get() == "openai" || model_namespace.get() == "google" || model_namespace.get() == "gcp");-}\ No newline at end of file+}diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cppindex b24a3b98..dab13102 100644--- a/src/text_embedder_remote.cpp+++ b/src/text_embedder_remote.cpp@@ -57,7 +57,7 @@ OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::}-Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {+Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config, size_t& num_dims) {auto validate_properties = validate_string_properties(model_config, {"model_name", "api_key"});if (!validate_properties.ok()) {@@ -260,7 +260,7 @@ GoogleEmbedder::GoogleEmbedder(const std::string& google_api_key) : google_api_k}-Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {+Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config, size_t& num_dims) {auto validate_properties = validate_string_properties(model_config, {"model_name", "api_key"});if (!validate_properties.ok()) {@@ -380,7 +380,7 @@ GCPEmbedder::GCPEmbedder(const std::string& project_id, const std::string& modelthis->model_name = TextEmbedderManager::get_model_name_without_namespace(model_name);}-Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {+Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, size_t& num_dims) {auto validate_properties = validate_string_properties(model_config, {"model_name", "project_id", "access_token", "refresh_token", "client_id", "client_secret"});if (!validate_properties.ok()) {diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex c3817ad5..15e193a3 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1650,17 +1650,18 @@ TEST_F(CollectionAllFieldsTest, ModelParametersWithoutEmbedFrom) {ASSERT_EQ("Property `embed` must contain a `from` property.", field_op.error());}-TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {-TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");-- field embedding = field("embedding", field_types::FLOAT_ARRAY, false);- embedding.embed["from"].push_back("name");- embedding.embed["model_config"]["model_name"] = "ts/e5-small";- std::vector<field> fields = {field("name", field_types::STRING, false),- embedding};- auto obj_coll_op = collectionManager.create_collection("obj_coll", 1, fields, "", 0, field_types::AUTO);+ nlohmann::json schema = R"({+ "name": "obj_coll",+ "fields": [+ {"name": "name", "type": "string"},+ {"name": "embedding", "type":"float[]", "embed":{"from": ["name"],+ "model_config": {"model_name": "ts/e5-small"}}}+ ]+ })"_json;++ auto obj_coll_op = collectionManager.create_collection(schema);ASSERT_TRUE(obj_coll_op.ok());Collection* obj_coll = obj_coll_op.get();diff --git a/test/collection_test.cpp b/test/collection_test.cppindex d27a78df..a1863b20 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -4816,16 +4816,14 @@ TEST_F(CollectionTest, WildcardSearchWithEmbeddingField) {}TEST_F(CollectionTest, CreateModelDirIfNotExists) {- system("mkdir -p /tmp/typesense_test/models");- system("rm -rf /tmp/typesense_test/models");- TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");+ system("mkdir -p /tmp/typesense_test/new_models_dir");+ system("rm -rf /tmp/typesense_test/new_models_dir");+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/new_models_dir");// check if model dir is created- ASSERT_TRUE(std::filesystem::exists("/tmp/typesense_test/models"));+ ASSERT_TRUE(std::filesystem::exists("/tmp/typesense_test/new_models_dir"));}--TEST_F(CollectionTest, EmbedStringArrayField) {nlohmann::json schema = R"({"name": "objects",