Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Review changes
ozanarmagan5bcbb2832e14
diff --git a/include/field.h b/include/field.hindex 0de91ceb..a79b6896 100644--- a/include/field.h+++ b/include/field.h@@ -428,11 +428,7 @@ struct field {size_t num_auto_detect_fields = 0;for(nlohmann::json & field_json: fields_json) {-if(field_json.count(fields::embed_from) != 0) {- // if(TextEmbedderManager::model_dir.empty()) {- // return Option<bool>(400, "Text embedding is not enabled. Please set `model-dir` at startup.");- // }if(!field_json[fields::embed_from].is_array()) {return Option<bool>(400, "Property `" + fields::embed_from + "` must be an array.");diff --git a/include/index.h b/include/index.hindex a80dff23..281091c2 100644--- a/include/index.h+++ b/include/index.h@@ -533,12 +533,7 @@ private:const std::string& token, uint32_t seq_id);void initialize_facet_indexes(const field& facet_field);---- static Option<bool> embed_fields(nlohmann::json& document,- const tsl::htrie_map<char, field>& embedding_fields,- const tsl::htrie_map<char, field> & search_schema);+static Option<bool> batch_embed_fields(std::vector<nlohmann::json*>& documents,const tsl::htrie_map<char, field>& embedding_fields,const tsl::htrie_map<char, field> & search_schema);diff --git a/include/text_embedder.h b/include/text_embedder.hindex 3b585b71..855154a7 100644--- a/include/text_embedder.h+++ b/include/text_embedder.h@@ -10,11 +10,12 @@class TextEmbedder {public:- TextEmbedder(const std::string& model_path, TokenizerType tokenizer_type = TokenizerType::bert);+ TextEmbedder(const std::string& model_path);TextEmbedder(const std::string& openai_model_path, const std::string& api_key);~TextEmbedder();Option<std::vector<float>> Embed(const std::string& text);Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs);+ const std::string& get_vocab_file_name() const;bool is_openai() {return !api_key.empty();@@ -27,6 +28,7 @@ class TextEmbedder {Ort::Env env_;encoded_input_t Encode(const std::string& text);std::unique_ptr<TextEmbeddingTokenizer> tokenizer_;+ std::string vocab_file_name;static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);std::string output_tensor_name;std::string api_key;diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.hindex 7789dec2..3a81e6cc 100644--- a/include/text_embedder_manager.h+++ b/include/text_embedder_manager.h@@ -1,4 +1,5 @@#pragma once+#include <memory>#include <filesystem>#include <mutex>@@ -9,244 +10,68 @@#include "http_client.h"#include "text_embedder.h"-struct text_embedding_model {- std::string model_url;+ std::string model_name;std::string model_md5;- std::string vocab_url;+ std::string vocab_file_name;std::string vocab_md5;TokenizerType tokenizer_type;std::string indexing_prefix = "";std::string query_prefix = "";-};+ text_embedding_model(const nlohmann::json& json);++ text_embedding_model() = default;+};-// singleton class+// Singleton classclass TextEmbedderManager {public:- static TextEmbedderManager& get_instance() {- static TextEmbedderManager instance;- return instance;- }-+ static TextEmbedderManager& get_instance();+TextEmbedderManager(TextEmbedderManager&&) = delete;TextEmbedderManager& operator=(TextEmbedderManager&&) = delete;TextEmbedderManager(const TextEmbedderManager&) = delete;TextEmbedderManager& operator=(const TextEmbedderManager&) = delete;- TextEmbedder* get_text_embedder(const nlohmann::json& model_parameters) {- std::unique_lock<std::mutex> lock(text_embedders_mutex);- LOG(INFO) << "Getting text embedder for model: " << model_parameters.dump();- const std::string& model_name = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_NAME : model_parameters.at("model_name");- if(text_embedders[model_name] == nullptr) {- if(model_parameters.count("api_key") == 0) {- TokenizerType tokenizer_type;- if(is_public_model(model_name)) {- tokenizer_type = public_models[model_name].tokenizer_type;- // download the model if it doesn't exist- download_public_model(model_name);- } else {- tokenizer_type = get_tokenizer_type(model_parameters);- }- LOG(INFO) << "Creating text embedder for model: " << model_name;- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, tokenizer_type);- } else {- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("api_key").get<std::string>());- }- }- return text_embedders[model_name].get();- }-- void delete_text_embedder(const std::string& model_path) {- std::unique_lock<std::mutex> lock(text_embedders_mutex);- if (text_embedders.find(model_path) != text_embedders.end()) {- text_embedders.erase(model_path);- }- }-- void delete_all_text_embedders() {- std::unique_lock<std::mutex> lock(text_embedders_mutex);- text_embedders.clear();- }-- static const TokenizerType get_tokenizer_type(const nlohmann::json& model_parameters) {- if(model_parameters.count("tokenizer_type") == 0) {- return TokenizerType::bert;- } else {- std::string tokenizer_type = model_parameters.at("model_type").get<std::string>();- if(tokenizer_type == "distilBert") {- return TokenizerType::distilBert;- } else if(tokenizer_type == "xlm-roberta") {- return TokenizerType::xlm_roberta;- } else {- return TokenizerType::bert;- }- }- }-- const std::string get_indexing_prefix(const nlohmann::json& model_parameters) {- std::string val;- if(is_public_model(model_parameters["model_name"].get<std::string>())) {- val = public_models[model_parameters["model_name"].get<std::string>()].indexing_prefix;- } else {- val = model_parameters.count("indexing_prefix") == 0 ? "" : model_parameters["indexing_prefix"].get<std::string>();- }- if(!val.empty()) {- val += " ";- }-- return val;- }-- const std::string get_query_prefix(const nlohmann::json& model_parameters) {- std::string val;- if(is_public_model(model_parameters["model_name"].get<std::string>())) {- val = public_models[model_parameters["model_name"].get<std::string>()].query_prefix;- } else {- val = model_parameters.count("query_prefix") == 0 ? "" : model_parameters["query_prefix"].get<std::string>();- }- if(!val.empty()) {- val += " ";- }-- return val;- }-- static void set_model_dir(const std::string& dir) {- // create the directory if it doesn't exist- if(!std::filesystem::exists(dir)) {- std::filesystem::create_directories(dir);- }- model_dir = dir;- }-- static const std::string& get_model_dir() {- return model_dir;- }-- ~TextEmbedderManager() {- delete_all_text_embedders();- }-- static constexpr char* DEFAULT_MODEL_NAME = "ts-e5-small";- static constexpr char* DEFAULT_MODEL_INDEXING_PREFIX = "passage:";- static constexpr char* DEFAULT_MODEL_QUERY_PREFIX = "query:";+ TextEmbedder* get_text_embedder(const nlohmann::json& model_parameters);+ void delete_text_embedder(const std::string& model_path);+ void delete_all_text_embedders();++ static const TokenizerType get_tokenizer_type(const nlohmann::json& model_parameters);+ const std::string get_indexing_prefix(const nlohmann::json& model_parameters);+ const std::string get_query_prefix(const nlohmann::json& model_parameters);+ static void set_model_dir(const std::string& dir);+ static const std::string& get_model_dir();++ ~TextEmbedderManager();++ inline static const std::string MODELS_REPO_URL = "https://models.typesense.org/public/";+ inline static const std::string MODEL_CONFIG_FILE = "config.json";inline static std::string model_dir = "";- inline static const std::string get_absolute_model_path(const std::string& model_name) {- return get_model_subdir(model_name) + "/model.onnx";- }- inline static const std::string get_absolute_vocab_path(const std::string& model_name) {- return get_model_subdir(model_name) + "/vocab.txt";- }- inline static const std::string get_absolute_sentencepiece_model_path(const std::string& model_name) {- auto model_subdir = get_model_subdir(model_name);- // look for any file that ends with .model- for (const auto & entry : std::filesystem::directory_iterator(model_subdir)) {- if(entry.path().extension() == ".model") {- return entry.path().string();- }- }-- return model_subdir + "/sentencepiece.model";- }- inline static const bool check_md5(const std::string& file_path, const std::string& target_md5) {- std::ifstream stream(file_path);- if (stream.fail()) {- return false;- }- unsigned char md5[MD5_DIGEST_LENGTH];- std::stringstream ss,res;- ss << stream.rdbuf();- MD5((unsigned char*)ss.str().c_str(), ss.str().length(), md5);- for (int i = 0; i < MD5_DIGEST_LENGTH; i++) {- res << std::hex << (int)md5[i];- }- return res.str() == target_md5;- }- void download_public_model(const std::string& model_name) {- HttpClient& httpClient = HttpClient::get_instance();- auto model = public_models[model_name];- if(!check_md5(get_absolute_model_path(model_name), model.model_md5)) {- LOG(INFO) << "Downloading public model: " << model_name;- long res = httpClient.download_file(model.model_url, get_absolute_model_path(model_name));- if(res != 200) {- LOG(INFO) << "Failed to download public model " << model_name << ": " << res;- }- }-- if(model.tokenizer_type == TokenizerType::bert || model.tokenizer_type == TokenizerType::distilBert) {- if(!check_md5(get_absolute_vocab_path(model_name), model.vocab_md5)) {- LOG(INFO) << "Downloading default vocab for model: " << model_name;- long res = httpClient.download_file(model.vocab_url, get_absolute_vocab_path(model_name));- if(res != 200) {- LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;- }- }- } else {- LOG(INFO) << "Downloading default sentencepiece model for model: " << model_name;- if(!check_md5(get_absolute_sentencepiece_model_path(model_name), model.vocab_md5)) {- LOG(INFO) << "Downloading default sentencepiece model for model: " << model_name;- long res = httpClient.download_file(model.vocab_url, get_absolute_sentencepiece_model_path(model_name));- if(res != 200) {- LOG(INFO) << "Failed to download default sentencepiece model " << model_name << ": " << res;- }- }- }- }-- const bool is_public_model(const std::string& model_name) {- return public_models.find(model_name) != public_models.end();- }++ static const std::string get_absolute_model_path(const std::string& model_name);+ static const std::string get_absolute_vocab_path(const std::string& model_name, const std::string& vocab_file_name);+ static const std::string get_absolute_config_path(const std::string& model_name);+ static const std::string get_model_url(const text_embedding_model& model);+ static const std::string get_vocab_url(const text_embedding_model& model);+ static Option<nlohmann::json> get_public_model_config(const std::string& model_name);+ static const std::string get_model_name_without_namespace(const std::string& model_name);+ static const std::string get_model_subdir(const std::string& model_name);+ static const bool check_md5(const std::string& file_path, const std::string& target_md5);+ void download_public_model(const std::string& model_name);++ const bool is_public_model(const std::string& model_name);+private:- TextEmbedderManager() {- public_models[DEFAULT_MODEL_NAME] = text_embedding_model{- "https://models.typesense.org/public/e5-small/model.onnx",- "3d421dc72859a72368c106415cdebf2",- "https://models.typesense.org/public/e5-small/vocab.txt",- "6480d5d8528ce344256daf115d4965e",- TokenizerType::bert,- "passage:",- "query:"};- public_models["ts-all-MiniLM-L12-v2"] = text_embedding_model{- "https://models.typesense.org/public/all-miniLM-L12-v2/model.onnx",- "6d196b8f7a8d8abcfb08afcac1704302",- "https://models.typesense.org/public/all-miniLM-L12-v2/vocab.txt",- "9d2131a9a433502abeb512978d452ecc",- TokenizerType::bert};- public_models["ts-distiluse-base-multilingual-cased-v2"] = text_embedding_model{- "https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/model.onnx",- "91f535cd7ca87359b9bfd1d87f9e452e",- "https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/vocab.txt",- "0f05e1b7420dea2db8ade63eb5b80f7a",- TokenizerType::distilBert};- public_models["ts-paraphrase-multilingual-mpnet-base-v2"] = text_embedding_model{- "https://models.typesense.org/public/paraphrase-multilingual-mpnet-base-v2/model.onnx",- "728d3db98e1b7a691a731644867382c5",- "https://models.typesense.org/public/paraphrase-multilingual-mpnet-base-v2/sentencepiece.bpe.model",- "bf25eb5120ad92ef5c7d8596b5dc4046",- TokenizerType::xlm_roberta};- }+ TextEmbedderManager() = default;+std::unordered_map<std::string, std::shared_ptr<TextEmbedder>> text_embedders;std::unordered_map<std::string, text_embedding_model> public_models;std::mutex text_embedders_mutex;- static const std::string get_model_subdir(const std::string& model_name) {- if(model_dir.back() != '/') {- // create subdir <model_name> if it doesn't exist- if(!std::filesystem::exists(model_dir + "/" + model_name)) {- std::filesystem::create_directories(model_dir + "/" + model_name);- }- return model_dir + "/" + model_name;- } else {- // create subdir <model_name> if it doesn't exist- if(!std::filesystem::exists(model_dir + model_name)) {- std::filesystem::create_directories(model_dir + model_name);- }- return model_dir + model_name;- }- }-};--++ Option<std::string> get_namespace(const std::string& model_name);+};diff --git a/include/text_embedder_tokenizer.h b/include/text_embedder_tokenizer.hindex d0d66098..bd41717a 100644--- a/include/text_embedder_tokenizer.h+++ b/include/text_embedder_tokenizer.h@@ -8,7 +8,7 @@enum class TokenizerType {bert,- distilBert,+ distilbert,xlm_roberta};@@ -35,9 +35,9 @@ class BertTokenizerWrapper : public TextEmbeddingTokenizer {encoded_input_t Encode(const std::string& text) override;};-class DistilBertTokenizer : public BertTokenizerWrapper {+class DistilbertTokenizer : public BertTokenizerWrapper {public:- DistilBertTokenizer(const std::string& vocab_path);+ DistilbertTokenizer(const std::string& vocab_path);encoded_input_t Encode(const std::string& text) override;};diff --git a/src/field.cpp b/src/field.cppindex 92909b9a..f9c9c363 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -679,43 +679,49 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsoif(field_json.count(fields::embed_from) != 0) {// If the model path is not specified, use the default model and set the number of dimensions to 384 (number of dimensions of the default model)field_json[fields::num_dim] = static_cast<unsigned int>(384);- if(field_json.count(fields::model_parameters) != 0) {- auto model_parameters = field_json[fields::model_parameters];- if(model_parameters.count(fields::model_name) != 0) {- unsigned int num_dim = 0;- if(!model_parameters[fields::model_name].is_string()) {- return Option<bool>(400, "Property `" + fields::model_name + "` must be a string.");- }- if(model_parameters[fields::model_name].get<std::string>().empty()) {- return Option<bool>(400, "Property `" + fields::model_name + "` must be a non-empty string.");- }- if(model_parameters.count(fields::indexing_prefix) != 0) {- if(!model_parameters[fields::indexing_prefix].is_string()) {- return Option<bool>(400, "Property `" + fields::indexing_prefix + "` must be a string.");- }- }+ if(field_json.count(fields::model_parameters) == 0) {+ return Option<bool>(400, "Property `" + fields::model_parameters + "` must be specified with `" + fields::embed_from + "`.");+ }- if(model_parameters.count(fields::query_prefix) != 0) {- if(!model_parameters[fields::query_prefix].is_string()) {- return Option<bool>(400, "Property `" + fields::query_prefix + "` must be a string.");- }- }+ auto& model_parameters = field_json[fields::model_parameters];++ if(model_parameters.count(fields::model_name) == 0) {+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::model_name + "` must be specified with `" + fields::embed_from + "`.");+ }- if(model_parameters.count("api_key") != 0) {- auto res = TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), model_parameters[fields::api_key].get<std::string>(), num_dim);- if(res.ok()) {- field_json[fields::num_dim] = num_dim;- } else {- return Option<bool>(res.code(), res.error());- }- } else {- if(TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), num_dim)) {- field_json[fields::num_dim] = num_dim;- } else {- return Option<bool>(400, "Property `" + fields::model_name + "` must be a valid model path.");- }- }+ unsigned int num_dim = 0;+ if(!model_parameters[fields::model_name].is_string()) {+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::model_name + "` must be a string.");+ }+ if(model_parameters[fields::model_name].get<std::string>().empty()) {+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::model_name + "` cannot be empty.");+ }++ if(model_parameters.count(fields::indexing_prefix) != 0) {+ if(!model_parameters[fields::indexing_prefix].is_string()) {+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::indexing_prefix + "` must be a string.");+ }+ }++ if(model_parameters.count(fields::query_prefix) != 0) {+ if(!model_parameters[fields::query_prefix].is_string()) {+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::query_prefix + "` must be a string.");+ }+ }++ if(model_parameters.count("api_key") != 0) {+ auto res = TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), model_parameters[fields::api_key].get<std::string>(), num_dim);+ if(res.ok()) {+ field_json[fields::num_dim] = num_dim;+ } else {+ return Option<bool>(res.code(), res.error());+ }+ } else {+ if(TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), num_dim)) {+ field_json[fields::num_dim] = num_dim;+ } else {+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::model_name + "` is invalid.");}}} else {diff --git a/src/http_client.cpp b/src/http_client.cppindex 652f0971..0829400f 100644--- a/src/http_client.cpp+++ b/src/http_client.cpp@@ -101,6 +101,9 @@ long HttpClient::get_response(const std::string &url, std::string &response,chunk = curl_slist_append(chunk, header_str.c_str());}+ // follow redirects+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);+return perform_curl(curl, res_headers, chunk);}diff --git a/src/index.cpp b/src/index.cppindex e54312ad..c18faadf 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -455,10 +455,8 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iteindex_rec.del_doc);scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);docs_to_embed.push_back(&index_rec.new_doc);- //embed_fields(index_rec.new_doc, embedding_fields, search_schema);} else {docs_to_embed.push_back(&index_rec.doc);- //embed_fields(index_rec.doc, embedding_fields, search_schema);}compute_token_offsets_facets(index_rec, search_schema, token_separators, symbols_to_index);@@ -6344,33 +6342,6 @@ bool Index::common_results_exist(std::vector<art_leaf*>& leaves, bool must_matchreturn phrase_exists;}-Option<bool> Index::embed_fields(nlohmann::json& document,- const tsl::htrie_map<char, field>& embedding_fields,- const tsl::htrie_map<char, field> & search_schema) {- for(const auto& field : embedding_fields) {- std::string text_to_embed = TextEmbedderManager::get_instance().get_indexing_prefix(field.model_parameters);- for(const auto& field_name : field.embed_from) {- auto field_it = search_schema.find(field_name);- if(field_it.value().type == field_types::STRING) {- text_to_embed += document[field_name].get<std::string>() + " ";- } else if(field_it.value().type == field_types::STRING_ARRAY) {- for(const auto& val : document[field_name]) {- text_to_embed += val.get<std::string>() + " ";- }- }- }- TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();- auto embedder = embedder_manager.get_text_embedder(field.model_parameters);- auto embedding_op = embedder->Embed(text_to_embed);-- if(!embedding_op.ok()) {- return Option<bool>(400, embedding_op.error());- }-- document[field.name] = embedding_op.get();- }- return Option<bool>(true);-}Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,const tsl::htrie_map<char, field>& embedding_fields,diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex 871a679a..c884fdf5 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -6,18 +6,24 @@#include <sstream>#include <filesystem>-TextEmbedder::TextEmbedder(const std::string& model_name, TokenizerType tokenizer_type) {+TextEmbedder::TextEmbedder(const std::string& model_name) {// create environmentOrt::SessionOptions session_options;std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_name);LOG(INFO) << "Loading model from: " << abs_path;session_ = std::make_unique<Ort::Session>(env_, abs_path.c_str(), session_options);+ std::ifstream config_file(TextEmbedderManager::get_absolute_config_path(model_name));+ nlohmann::json config;+ config_file >> config;+ TokenizerType tokenizer_type = TextEmbedderManager::get_tokenizer_type(config);+ auto vocab_path = TextEmbedderManager::get_absolute_vocab_path(model_name, config["vocab_file_name"].get<std::string>());if(tokenizer_type == TokenizerType::bert) {- auto vocab_path = TextEmbedderManager::get_absolute_vocab_path(model_name);tokenizer_ = std::make_unique<BertTokenizerWrapper>(vocab_path);- } else {- auto spiece_model_path = TextEmbedderManager::get_absolute_sentencepiece_model_path(model_name);- tokenizer_ = std::make_unique<XLMRobertaTokenizer>(spiece_model_path);+ } else if(tokenizer_type == TokenizerType::distilbert) {+ tokenizer_ = std::make_unique<DistilbertTokenizer>(vocab_path);+ }+ else if(tokenizer_type == TokenizerType::xlm_roberta) {+ tokenizer_ = std::make_unique<XLMRobertaTokenizer>(vocab_path);}auto output_tensor_count = session_->GetOutputCount();for (size_t i = 0; i < output_tensor_count; i++) {@@ -67,6 +73,7 @@ Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {}return Option<std::vector<float>>(nlohmann::json::parse(res)["data"][0]["embedding"].get<std::vector<float>>());} else {+ LOG(INFO) << "Embedding text: " << text;auto encoded_input = tokenizer_->Encode(text);// create input tensor object from data valuesOrt::AllocatorWithDefaultOptions allocator;@@ -159,13 +166,32 @@ bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& nOrt::SessionOptions session_options;Ort::Env env;- std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_name);+ std::string abs_path = TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::get_model_name_without_namespace(model_name));if(!std::filesystem::exists(abs_path)) {LOG(ERROR) << "Model file not found: " << abs_path;return false;}+ if(!TextEmbedderManager::get_instance().is_public_model(model_name)) {+ if(!std::filesystem::exists(TextEmbedderManager::get_absolute_config_path(model_name))) {+ LOG(ERROR) << "Config file not found: " << TextEmbedderManager::get_absolute_config_path(model_name);+ return false;+ }+ std::ifstream config_file(TextEmbedderManager::get_absolute_config_path(model_name));+ nlohmann::json config;+ config_file >> config;+ if(config["model_type"].is_null() || config["vocab_file_name"].is_null()) {+ LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);+ return false;+ }++ if(!std::filesystem::exists(TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>())) {+ LOG(ERROR) << "Vocab file not found: " << TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>();+ return false;+ }+ }+Ort::Session session(env, abs_path.c_str(), session_options);if(session.GetInputCount() != 3 && session.GetInputCount() != 2) {LOG(ERROR) << "Invalid model: input count is not 3 or 2";diff --git a/src/text_embedder_manager.cpp b/src/text_embedder_manager.cppnew file mode 100644index 00000000..01cadc09--- /dev/null+++ b/src/text_embedder_manager.cpp@@ -0,0 +1,251 @@+#include "text_embedder_manager.h"+++TextEmbedderManager& TextEmbedderManager::get_instance() {+ static TextEmbedderManager instance;+ return instance;+}++TextEmbedder* TextEmbedderManager::get_text_embedder(const nlohmann::json& model_parameters) {+ std::unique_lock<std::mutex> lock(text_embedders_mutex);+ const std::string& model_name = model_parameters.at("model_name");+ if(text_embedders[model_name] == nullptr) {+ if(model_parameters.count("api_key") == 0) {+ if(is_public_model(model_name)) {+ // download the model if it doesn't exist+ download_public_model(model_name);+ }+ text_embedders[model_name] = std::make_shared<TextEmbedder>(get_model_name_without_namespace(model_name));+ } else {+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("api_key").get<std::string>());+ }+ }+ return text_embedders[model_name].get();+}++void TextEmbedderManager::delete_text_embedder(const std::string& model_path) {+ std::unique_lock<std::mutex> lock(text_embedders_mutex);+ if (text_embedders.find(model_path) != text_embedders.end()) {+ text_embedders.erase(model_path);+ }+}++void TextEmbedderManager::delete_all_text_embedders() {+ std::unique_lock<std::mutex> lock(text_embedders_mutex);+ text_embedders.clear();+}++const TokenizerType TextEmbedderManager::get_tokenizer_type(const nlohmann::json& model_parameters) {+ if(model_parameters.find("model_type") == model_parameters.end()) {+ return TokenizerType::bert;+ } else {+ std::string tokenizer_type = model_parameters.at("model_type").get<std::string>();+ if(tokenizer_type == "distilbert") {+ return TokenizerType::distilbert;+ } else if(tokenizer_type == "xlm_roberta") {+ return TokenizerType::xlm_roberta;+ } else {+ return TokenizerType::bert;+ }+ }+}++const std::string TextEmbedderManager::get_indexing_prefix(const nlohmann::json& model_parameters) {+ std::string val;+ if(is_public_model(model_parameters["model_name"].get<std::string>())) {+ val = public_models[model_parameters["model_name"].get<std::string>()].indexing_prefix;+ } else {+ val = model_parameters.count("indexing_prefix") == 0 ? "" : model_parameters["indexing_prefix"].get<std::string>();+ }+ if(!val.empty()) {+ val += " ";+ }++ return val;+}++const std::string TextEmbedderManager::get_query_prefix(const nlohmann::json& model_parameters) {+ std::string val;+ if(is_public_model(model_parameters["model_name"].get<std::string>())) {+ val = public_models[model_parameters["model_name"].get<std::string>()].query_prefix;+ } else {+ val = model_parameters.count("query_prefix") == 0 ? "" : model_parameters["query_prefix"].get<std::string>();+ }+ if(!val.empty()) {+ val += " ";+ }++ return val;+}++void TextEmbedderManager::set_model_dir(const std::string& dir) {+ // create the directory if it doesn't exist+ if(!std::filesystem::exists(dir)) {+ std::filesystem::create_directories(dir);+ }+ model_dir = dir;+}++const std::string& TextEmbedderManager::get_model_dir() {+ return model_dir;+}++TextEmbedderManager::~TextEmbedderManager() {+ delete_all_text_embedders();+}++const std::string TextEmbedderManager::get_absolute_model_path(const std::string& model_name) {+ return get_model_subdir(model_name) + "/model.onnx";+}+const std::string TextEmbedderManager::get_absolute_vocab_path(const std::string& model_name, const std::string& vocab_file_name) {+ return get_model_subdir(model_name) + "/" + vocab_file_name;+}++const std::string TextEmbedderManager::get_absolute_config_path(const std::string& model_name) {+ return get_model_subdir(model_name) + "/config.json";+}++const bool TextEmbedderManager::check_md5(const std::string& file_path, const std::string& target_md5) {+ std::ifstream stream(file_path);+ if (stream.fail()) {+ return false;+ }+ unsigned char md5[MD5_DIGEST_LENGTH];+ std::stringstream ss,res;+ ss << stream.rdbuf();+ MD5((unsigned char*)ss.str().c_str(), ss.str().length(), md5);+ for (int i = 0; i < MD5_DIGEST_LENGTH; i++) {+ res << std::hex << (int)md5[i];+ }+ return res.str() == target_md5;+}+void TextEmbedderManager::download_public_model(const std::string& model_name) {+ HttpClient& httpClient = HttpClient::get_instance();+ auto model = public_models[model_name];+ auto actual_model_name = get_model_name_without_namespace(model_name);+ if(!check_md5(get_absolute_model_path(actual_model_name), model.model_md5)) {+ long res = httpClient.download_file(get_model_url(model), get_absolute_model_path(actual_model_name));+ if(res != 200) {+ LOG(INFO) << "Failed to download public model " << model_name << ": " << res;+ }+ }++ if(!check_md5(get_absolute_vocab_path(actual_model_name, model.vocab_file_name), model.vocab_md5)) {+ long res = httpClient.download_file(get_vocab_url(model), get_absolute_vocab_path(actual_model_name, model.vocab_file_name));+ if(res != 200) {+ LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;+ }+ }++}++const bool TextEmbedderManager::is_public_model(const std::string& model_name) {+ if(public_models.find(model_name) != public_models.end()) {+ return true;+ }++ auto model_namespace = get_namespace(model_name);+ if(!model_namespace.ok() || model_namespace.get() != "ts") {+ return false;+ }++ auto actual_model_name = get_model_name_without_namespace(model_name);+ auto model_config_op = get_public_model_config(actual_model_name);++ if(model_config_op.ok()) {+ auto config = model_config_op.get();+ config["model_name"] = actual_model_name;+ public_models[model_name] = text_embedding_model(config);+ return true;+ }++ return false;+}++const std::string TextEmbedderManager::get_model_subdir(const std::string& model_name) {+ if(model_dir.back() != '/') {+ // create subdir <model_name> if it doesn't exist+ if(!std::filesystem::exists(model_dir + "/" + model_name)) {+ std::filesystem::create_directories(model_dir + "/" + model_name);+ }+ return model_dir + "/" + model_name;+ } else {+ // create subdir <model_name> if it doesn't exist+ if(!std::filesystem::exists(model_dir + model_name)) {+ std::filesystem::create_directories(model_dir + model_name);+ }+ return model_dir + model_name;+ }+}++Option<std::string> TextEmbedderManager::get_namespace(const std::string& model_name) {+ // <namespace>/<model_name> if / is present in model_name+ if(model_name.find("/") != std::string::npos) {+ return Option<std::string>(model_name.substr(0, model_name.find("/")));+ } else {+ return Option<std::string>(404, "Namespace not found");+ }+}++const std::string TextEmbedderManager::get_model_name_without_namespace(const std::string& model_name) {+ // <namespace>/<model_name> if / is present in model_name+ if(model_name.find("/") != std::string::npos) {+ return model_name.substr(model_name.find("/") + 1);+ } else {+ return model_name;+ }+}++text_embedding_model::text_embedding_model(const nlohmann::json& json) {+ model_name = json.at("model_name").get<std::string>();+ model_md5 = json.at("model_md5").get<std::string>();+ vocab_file_name = json.at("vocab_file_name").get<std::string>();+ vocab_md5 = json.at("vocab_md5").get<std::string>();+ tokenizer_type = TextEmbedderManager::get_tokenizer_type(json);+ if(json.count("indexing_prefix") != 0) {+ indexing_prefix = json.at("indexing_prefix").get<std::string>();+ }+ if(json.count("query_prefix") != 0) {+ query_prefix = json.at("query_prefix").get<std::string>();+ }+}+++Option<nlohmann::json> TextEmbedderManager::get_public_model_config(const std::string& model_name) {+ // check cache first+ if(std::filesystem::exists(get_absolute_config_path(model_name))) {+ std::ifstream config_file(get_absolute_config_path(model_name));+ nlohmann::json config;+ config_file >> config;+ config_file.close();+ return Option<nlohmann::json>(config);+ }++ auto actual_model_name = get_model_name_without_namespace(model_name);+ HttpClient& httpClient = HttpClient::get_instance();+ std::unordered_map<std::string, std::string> headers;+ headers["Accept"] = "application/json";+ std::map<std::string, std::string> response_headers;+ std::string response_body;+ long res = httpClient.get_response(MODELS_REPO_URL + actual_model_name + "/" + MODEL_CONFIG_FILE, response_body, response_headers, headers);++ if(res == 200 || res == 302) {+ // cache the config file+ auto config_file_path = get_absolute_config_path(actual_model_name);+ std::ofstream config_file(config_file_path);+ config_file << response_body;+ config_file.close();++ return Option<nlohmann::json>(nlohmann::json::parse(response_body));+ }++ return Option<nlohmann::json>(404, "Model not found");+}++const std::string TextEmbedderManager::get_model_url(const text_embedding_model& model) {+ return MODELS_REPO_URL + model.model_name + "/model.onnx";+}++const std::string TextEmbedderManager::get_vocab_url(const text_embedding_model& model) {+ return MODELS_REPO_URL + model.model_name + "/" + model.vocab_file_name;+}\ No newline at end of filediff --git a/src/text_embedder_tokenizer.cpp b/src/text_embedder_tokenizer.cppindex f3dee97c..100f8e0a 100644--- a/src/text_embedder_tokenizer.cpp+++ b/src/text_embedder_tokenizer.cpp@@ -28,10 +28,10 @@ encoded_input_t BertTokenizerWrapper::Encode(const std::string& text) {-DistilBertTokenizer::DistilBertTokenizer(const std::string& vocab_path) : BertTokenizerWrapper(vocab_path) {}+DistilbertTokenizer::DistilbertTokenizer(const std::string& vocab_path) : BertTokenizerWrapper(vocab_path) {}-encoded_input_t DistilBertTokenizer::Encode(const std::string& text) {+encoded_input_t DistilbertTokenizer::Encode(const std::string& text) {auto encoded = bert_tokenizer_->Encode(bert_tokenizer_->Tokenize(ustring(text)));auto input_ids = bert_tokenizer_->AddSpecialToken(encoded);auto attention_mask = std::vector<int64_t>(input_ids.size(), 1);diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex ba1768ba..488dce4c 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1593,11 +1593,13 @@ TEST_F(CollectionAllFieldsTest, FieldNameMatchingRegexpShouldNotBeIndexedInNonAu}TEST_F(CollectionAllFieldsTest, EmbedFromFieldJSONInvalidField) {- TextEmbedderManager::model_dir = "/tmp/models";+ TextEmbedderManager::set_model_dir("/tmp/typensense_test/models");nlohmann::json field_json;field_json["name"] = "embedding";field_json["type"] = "float[]";field_json["embed_from"] = {"name"};+ field_json["model_parameters"] = nlohmann::json::object();+ field_json["model_parameters"]["model_name"] = "ts/e5-small";std::vector<field> fields;std::string fallback_field_type;@@ -1629,11 +1631,13 @@ TEST_F(CollectionAllFieldsTest, EmbedFromFieldJSONInvalidField) {// }TEST_F(CollectionAllFieldsTest, EmbedFromNotArray) {- TextEmbedderManager::model_dir = "/tmp/models";+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");nlohmann::json field_json;field_json["name"] = "embedding";field_json["type"] = "float[]";field_json["embed_from"] = "name";+ field_json["model_parameters"] = nlohmann::json::object();+ field_json["model_parameters"]["model_name"] = "ts/e5-small";std::vector<field> fields;std::string fallback_field_type;@@ -1647,11 +1651,12 @@ TEST_F(CollectionAllFieldsTest, EmbedFromNotArray) {}TEST_F(CollectionAllFieldsTest, ModelParametersWithoutEmbedFrom) {- TextEmbedderManager::model_dir = "/tmp/models";+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");nlohmann::json field_json;field_json["name"] = "embedding";field_json["type"] = "float[]";- field_json["model_parameters"] = {{"model_name", "bert-base-uncased"}};+ field_json["model_parameters"] = nlohmann::json::object();+ field_json["model_parameters"]["model_name"] = "ts/e5-small";std::vector<field> fields;std::string fallback_field_type;@@ -1666,10 +1671,11 @@ TEST_F(CollectionAllFieldsTest, ModelParametersWithoutEmbedFrom) {TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {- TextEmbedderManager::model_dir = "/tmp/typesense_test/models";+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");field embedding = field("embedding", field_types::FLOAT_ARRAY, false);embedding.embed_from.push_back("name");+ embedding.model_parameters["model_name"] = "ts/e5-small";std::vector<field> fields = {field("name", field_types::STRING, false),embedding};auto obj_coll_op = collectionManager.create_collection("obj_coll", 1, fields, "", 0, field_types::AUTO);@@ -1690,11 +1696,13 @@ TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {}TEST_F(CollectionAllFieldsTest, WrongDataTypeForEmbedFrom) {- TextEmbedderManager::model_dir = "/tmp/models";+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");nlohmann::json field_json;field_json["name"] = "embedding";field_json["type"] = "float[]";field_json["embed_from"] = {"age"};+ field_json["model_parameters"] = nlohmann::json::object();+ field_json["model_parameters"]["model_name"] = "ts/e5-small";std::vector<field> fields;std::string fallback_field_type;diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex 42bab54a..ebccb569 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -1464,7 +1464,7 @@ TEST_F(CollectionSchemaChangeTest, UpdateSchemaWithNewEmbeddingField) {nlohmann::json update_schema = R"({"fields": [- {"name": "embedding", "type":"float[]", "embed_from": ["names"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["names"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -1489,7 +1489,7 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {"fields": [{"name": "names", "type": "string[]"},{"name": "category", "type":"string"},- {"name": "embedding", "type":"float[]", "embed_from": ["names","category"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["names","category"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -1537,7 +1537,7 @@ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {"name": "objects","fields": [{"name": "name", "type": "string"},- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 1346fbc4..c7d7d25b 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -4617,7 +4617,7 @@ TEST_F(CollectionTest, SemanticSearchTest) {"name": "objects","fields": [{"name": "name", "type": "string"},- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -4651,7 +4651,7 @@ TEST_F(CollectionTest, InvalidSemanticSearch) {"name": "objects","fields": [{"name": "name", "type": "string"},- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -4680,7 +4680,7 @@ TEST_F(CollectionTest, HybridSearch) {"name": "objects","fields": [{"name": "name", "type": "string"},- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -4741,7 +4741,7 @@ TEST_F(CollectionTest, HybridSearchRankFusionTest) {"name": "objects","fields": [{"name": "name", "type": "string"},- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -4814,7 +4814,7 @@ TEST_F(CollectionTest, WildcardSearchWithEmbeddingField) {"name": "objects","fields": [{"name": "name", "type": "string"},- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -4847,7 +4847,7 @@ TEST_F(CollectionTest, EmbedStringArrayField) {"name": "objects","fields": [{"name": "names", "type": "string[]"},- {"name": "embedding", "type":"float[]", "embed_from": ["names"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["names"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -4872,7 +4872,7 @@ TEST_F(CollectionTest, MissingFieldForEmbedding) {"fields": [{"name": "names", "type": "string[]"},{"name": "category", "type": "string", "optional": true},- {"name": "embedding", "type":"float[]", "embed_from": ["names", "category"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["names", "category"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -4898,7 +4898,7 @@ TEST_F(CollectionTest, WrongTypeForEmbedding) {"name": "objects","fields": [{"name": "category", "type": "string"},- {"name": "embedding", "type":"float[]", "embed_from": ["category"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["category"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -4921,7 +4921,7 @@ TEST_F(CollectionTest, WrongTypeOfElementForEmbeddingInStringArray) {"name": "objects","fields": [{"name": "category", "type": "string[]"},- {"name": "embedding", "type":"float[]", "embed_from": ["category"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["category"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -4944,7 +4944,7 @@ TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {"name": "objects","fields": [{"name": "name", "type": "string"},- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 3df3bde5..2334fd0a 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -683,7 +683,7 @@ TEST_F(CollectionVectorTest, HybridSearchWithExplicitVector) {"name": "coll1","fields": [{"name": "name", "type": "string"},- {"name": "vec", "type": "float[]", "embed_from": ["name"]}+ {"name": "vec", "type": "float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;@@ -781,7 +781,7 @@ TEST_F(CollectionVectorTest, EmbeddingFieldVectorIndexTest) {"name": "objects","fields": [{"name": "name", "type": "string"},- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}]})"_json;