Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Review Changes
ozanarmagana571d64d5695
diff --git a/include/field.h b/include/field.hindex 5fbb386e..0de91ceb 100644--- a/include/field.h+++ b/include/field.h@@ -52,7 +52,12 @@ namespace fields {static const std::string reference = "reference";static const std::string embed_from = "embed_from";static const std::string model_name = "model_name";- static const std::string openai_api_key = "openai_api_key";++ // Some models require additional parameters to be passed to the model during indexing/querying+ // For e.g. e5-small model requires prefix "passage:" for indexing and "query:" for querying+ static const std::string indexing_prefix = "indexing_prefix";+ static const std::string query_prefix = "query_prefix";+ static const std::string api_key = "api_key";static const std::string model_parameters = "model_parameters";}@@ -326,10 +331,6 @@ struct field {field_val[fields::embed_from] = field.embed_from;if(!field.model_parameters.empty()) {field_val[fields::model_parameters] = field.model_parameters;- // hide openai api key- if(field_val[fields::model_parameters].count("openai_api_key") != 0) {- field_val[fields::model_parameters]["openai_api_key"] = "<hidden>";- }}}fields_json.push_back(field_val);@@ -429,9 +430,9 @@ struct field {for(nlohmann::json & field_json: fields_json) {if(field_json.count(fields::embed_from) != 0) {- if(TextEmbedderManager::model_dir.empty()) {- return Option<bool>(400, "Text embedding is not enabled. Please set `model-dir` at startup.");- }+ // if(TextEmbedderManager::model_dir.empty()) {+ // return Option<bool>(400, "Text embedding is not enabled. Please set `model-dir` at startup.");+ // }if(!field_json[fields::embed_from].is_array()) {return Option<bool>(400, "Property `" + fields::embed_from + "` must be an array.");diff --git a/include/text_embedder.h b/include/text_embedder.hindex 869644f8..3ca0d3d0 100644--- a/include/text_embedder.h+++ b/include/text_embedder.h@@ -15,17 +15,17 @@ struct encoded_input_t {class TextEmbedder {public:TextEmbedder(const std::string& model_path);- TextEmbedder(const std::string& openai_model_path, const std::string& openai_api_key);+ TextEmbedder(const std::string& openai_model_path, const std::string& api_key);~TextEmbedder();Option<std::vector<float>> Embed(const std::string& text);Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs);bool is_openai() {- return !openai_api_key.empty();+ return !api_key.empty();}static bool is_model_valid(const std::string& model_path, unsigned int& num_dims);- static Option<bool> is_model_valid(const std::string openai_model_path, const std::string openai_api_key, unsigned int& num_dims);+ static Option<bool> is_model_valid(const std::string openai_model_path, const std::string api_key, unsigned int& num_dims);private:std::unique_ptr<Ort::Session> session_;Ort::Env env_;@@ -33,7 +33,7 @@ class TextEmbedder {std::unique_ptr<BertTokenizer> tokenizer_;static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);std::string output_tensor_name;- std::string openai_api_key;+ std::string api_key;std::string openai_model_path;static constexpr char* OPENAI_LIST_MODELS = "https://api.openai.com/v1/models";static constexpr char* OPENAI_CREATE_EMBEDDING = "https://api.openai.com/v1/embeddings";diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.hindex f19bc8d9..dfad1b10 100644--- a/include/text_embedder_manager.h+++ b/include/text_embedder_manager.h@@ -1,6 +1,7 @@#pragma once#include <memory>#include <filesystem>+#include <mutex>#include <unordered_map>#include <openssl/md5.h>#include <fstream>@@ -9,6 +10,14 @@#include "text_embedder.h"+struct text_embedding_model {+ std::string model_url;+ std::string model_md5;+ std::string vocab_url;+ std::string vocab_md5;+};++// singleton classclass TextEmbedderManager {public:@@ -23,35 +32,52 @@ public:TextEmbedderManager& operator=(const TextEmbedderManager&) = delete;TextEmbedder* get_text_embedder(const nlohmann::json& model_parameters) {- if(model_parameters.count("model_name") == 0) {- if(text_embedders[DEFAULT_MODEL_NAME] == nullptr) {- text_embedders[DEFAULT_MODEL_NAME] = std::make_shared<TextEmbedder>(DEFAULT_MODEL_NAME);- }- return text_embedders[DEFAULT_MODEL_NAME].get();- } else {- const std::string& model_name = model_parameters.at("model_name");- if(text_embedders[model_name] == nullptr) {- if(model_parameters.count("openai_api_key") == 0) {- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name);- } else {- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("openai_api_key"));+ std::unique_lock<std::mutex> lock(text_embedders_mutex);+ const std::string& model_name = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_NAME : model_parameters.at("model_name");+ if(text_embedders[model_name] == nullptr) {+ if(model_parameters.count("api_key") == 0) {+ // download the model if it doesn't exist+ if(is_public_model(model_name)) {+ download_public_model(model_name);}+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name);+ } else {+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("api_key"));}- return text_embedders[model_name].get();}-+ return text_embedders[model_name].get();}void delete_text_embedder(const std::string& model_path) {+ std::unique_lock<std::mutex> lock(text_embedders_mutex);if (text_embedders.find(model_path) != text_embedders.end()) {text_embedders.erase(model_path);}}void delete_all_text_embedders() {+ std::unique_lock<std::mutex> lock(text_embedders_mutex);text_embedders.clear();}+ static const std::string get_indexing_prefix(const nlohmann::json& model_parameters) {+ std::string val = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_INDEXING_PREFIX : model_parameters["indexing_prefix"].get<std::string>();+ if(!val.empty()) {+ val += " ";+ }++ return val;+ }++ static const std::string get_query_prefix(const nlohmann::json& model_parameters) {+ std::string val = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_QUERY_PREFIX : model_parameters["query_prefix"].get<std::string>();+ if(!val.empty()) {+ val += " ";+ }++ return val;+ }+static void set_model_dir(const std::string& dir) {// create the directory if it doesn't existif(!std::filesystem::exists(dir)) {@@ -68,33 +94,38 @@ public:delete_all_text_embedders();}- static constexpr char* DEFAULT_MODEL_URL = "https://huggingface.co/typesense/models/resolve/main/e5-small/model.onnx";static constexpr char* DEFAULT_MODEL_NAME = "ts-e5-small";- static constexpr char* DEFAULT_VOCAB_URL = "https://huggingface.co/typesense/models/resolve/main/e5-small/vocab.txt";- static constexpr char* DEFAULT_VOCAB_NAME = "vocab.txt";- static constexpr char* DEFAULT_VOCAB_MD5 = "6480d5d8528ce344256daf115d4965e";- static constexpr char* DEFAULT_MODEL_MD5 = "3d421dc72859a72368c106415cdebf2";+ static constexpr char* DEFAULT_MODEL_INDEXING_PREFIX = "passage:";+ static constexpr char* DEFAULT_MODEL_QUERY_PREFIX = "query:";inline static std::string model_dir = "";inline static const std::string get_absolute_model_path(const std::string& model_name) {if(model_dir.back() != '/') {- if(model_name.front() != '/') {- return model_dir + "/" + model_name + ".onnx";- } else {- return model_dir + model_name + ".onnx";+ // create subdir <model_name> if it doesn't exist+ if(!std::filesystem::exists(model_dir + "/" + model_name)) {+ std::filesystem::create_directories(model_dir + "/" + model_name);}+ return model_dir + "/" + model_name + "/" + "model.onnx";} else {- if(model_name.front() != '/') {- return model_dir + model_name + ".onnx";- } else {- return model_dir + "/" + model_name + ".onnx";+ // create subdir <model_name> if it doesn't exist+ if(!std::filesystem::exists(model_dir + model_name)) {+ std::filesystem::create_directories(model_dir + model_name);}+ return model_dir + model_name + "/" + "model.onnx";}};- inline static const std::string get_absolute_vocab_path() {+ inline static const std::string get_absolute_vocab_path(const std::string& model_name) {if(model_dir.back() != '/') {- return model_dir + "/" + TextEmbedderManager::DEFAULT_VOCAB_NAME;+ // create subdir <model_name> if it doesn't exist+ if(!std::filesystem::exists(model_dir + "/" + model_name)) {+ std::filesystem::create_directories(model_dir + "/" + model_name);+ }+ return model_dir + "/" + model_name + "/" + "vocab.txt";} else {- return model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME;+ // create subdir <model_name> if it doesn't exist+ if(!std::filesystem::exists(model_dir + model_name)) {+ std::filesystem::create_directories(model_dir + model_name);+ }+ return model_dir + model_name + "/" + "vocab.txt";}}inline static const bool check_md5(const std::string& file_path, const std::string& target_md5) {@@ -111,26 +142,49 @@ public:}return res.str() == target_md5;}- inline static void download_default_model() {+ void download_public_model(const std::string& model_name) {HttpClient& httpClient = HttpClient::get_instance();- if(!check_md5(get_absolute_model_path(DEFAULT_MODEL_NAME), DEFAULT_MODEL_MD5)) {- LOG(INFO) << "Downloading default model";- long res = httpClient.download_file(DEFAULT_MODEL_URL, get_absolute_model_path(DEFAULT_MODEL_NAME));+ auto model = public_models[model_name];+ if(!check_md5(get_absolute_model_path(model_name), model.model_md5)) {+ LOG(INFO) << "Downloading public model: " << model_name;+ long res = httpClient.download_file(model.model_url, get_absolute_model_path(model_name));if(res != 200) {- LOG(INFO) << "Failed to download default model: " << res;+ LOG(INFO) << "Failed to download public model " << model_name << ": " << res;}}- if(!check_md5(get_absolute_vocab_path(), DEFAULT_VOCAB_MD5)) {- LOG(INFO) << "Downloading default vocab";- long res = httpClient.download_file(DEFAULT_VOCAB_URL, get_absolute_vocab_path());+ if(!check_md5(get_absolute_vocab_path(model_name), model.vocab_md5)) {+ LOG(INFO) << "Downloading default vocab for model: " << model_name;+ long res = httpClient.download_file(model.vocab_url, get_absolute_vocab_path(model_name));if(res != 200) {- LOG(INFO) << "Failed to download default vocab: " << res;+ LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;}}}++ const bool is_public_model(const std::string& model_name) {+ return public_models.find(model_name) != public_models.end();+ }private:- TextEmbedderManager() = default;+ TextEmbedderManager() {+ public_models[DEFAULT_MODEL_NAME] = text_embedding_model{+ "https://models.typesense.org/public/e5-small/model.onnx",+ "3d421dc72859a72368c106415cdebf2",+ "https://models.typesense.org/public/e5-small/vocab.txt",+ "6480d5d8528ce344256daf115d4965e"};+ public_models["ts-all-MiniLM-L12-v2"] = text_embedding_model{+ "https://models.typesense.org/public/all-miniLM-L12-v2/model.onnx",+ "6d196b8f7a8d8abcfb08afcac1704302",+ "https://models.typesense.org/public/all-miniLM-L12-v2/vocab.txt",+ "9d2131a9a433502abeb512978d452ecc"};+ public_models["ts-distiluse-base-multilingual-cased-v2"] = text_embedding_model{+ "https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/model.onnx",+ "91f535cd7ca87359b9bfd1d87f9e452e",+ "https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/vocab.txt",+ "0f05e1b7420dea2db8ade63eb5b80f7a"};+ }std::unordered_map<std::string, std::shared_ptr<TextEmbedder>> text_embedders;+ std::unordered_map<std::string, text_embedding_model> public_models;+ std::mutex text_embedders_mutex;};diff --git a/include/tsconfig.h b/include/tsconfig.hindex 57796930..cff6f85e 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -11,7 +11,6 @@ class Config {private:std::string data_dir;std::string log_dir;- std::string model_dir;std::string api_key;@@ -112,10 +111,6 @@ public:this->log_dir = log_dir;}- void set_model_dir(const std::string & model_dir) {- this->model_dir = model_dir;- }-void set_api_key(const std::string & api_key) {this->api_key = api_key;}@@ -183,9 +178,6 @@ public:return this->log_dir;}- std::string get_model_dir() const {- return this->model_dir;- }std::string get_api_key() const {return this->api_key;@@ -592,9 +584,6 @@ public:this->reset_peers_on_error = (reset_peers_on_error_str == "true");}- if(reader.Exists("server", "model-dir")) {- this->model_dir = reader.Get("server", "model-dir", "");- }}void load_config_cmd_args(cmdline::parser & options) {@@ -610,10 +599,6 @@ public:this->api_key = options.get<std::string>("api-key");}- if(options.exist("model-dir")) {- LOG(INFO) << "model-dir found";- this->model_dir = options.get<std::string>("model-dir");- }// @deprecatedif(options.exist("search-only-api-key")) {diff --git a/src/collection.cpp b/src/collection.cppindex 525cc753..a4723772 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -247,8 +247,8 @@ nlohmann::json Collection::get_summary_json() const {if(coll_field.model_parameters.size() > 0) {field_json[fields::model_parameters] = coll_field.model_parameters;// Hide OpenAI API key from the response.- if(field_json[fields::model_parameters].count(fields::openai_api_key) != 0) {- field_json[fields::model_parameters][fields::openai_api_key] = "<hidden>";+ if(field_json[fields::model_parameters].count(fields::api_key) != 0) {+ field_json[fields::model_parameters][fields::api_key] = "<hidden>";}}@@ -1177,10 +1177,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,return Option<nlohmann::json>(400, error);}- if(TextEmbedderManager::model_dir.empty()) {- std::string error = "Text embedding is not enabled. Please set `model-dir` at startup.";- return Option<nlohmann::json>(400, error);- }+ // if(TextEmbedderManager::model_dir.empty()) {+ // std::string error = "Text embedding is not enabled. Please set `model-dir` at startup.";+ // return Option<nlohmann::json>(400, error);+ // }if(raw_query == "*") {std::string error = "Wildcard query is not supported for embedding fields.";@@ -1190,7 +1190,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();auto embedder = embedder_manager.get_text_embedder(search_field.model_parameters);- std::string embed_query = "query: " + raw_query;+ std::string embed_query = embedder_manager.get_query_prefix(search_field.model_parameters) + raw_query;auto embedding_op = embedder->Embed(embed_query);if(!embedding_op.ok()) {return Option<nlohmann::json>(400, embedding_op.error());diff --git a/src/field.cpp b/src/field.cppindex ec9e8f7c..92909b9a 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -689,8 +689,21 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jsoif(model_parameters[fields::model_name].get<std::string>().empty()) {return Option<bool>(400, "Property `" + fields::model_name + "` must be a non-empty string.");}- if(model_parameters.count("openai_api_key") != 0) {- auto res = TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), model_parameters[fields::openai_api_key].get<std::string>(), num_dim);++ if(model_parameters.count(fields::indexing_prefix) != 0) {+ if(!model_parameters[fields::indexing_prefix].is_string()) {+ return Option<bool>(400, "Property `" + fields::indexing_prefix + "` must be a string.");+ }+ }++ if(model_parameters.count(fields::query_prefix) != 0) {+ if(!model_parameters[fields::query_prefix].is_string()) {+ return Option<bool>(400, "Property `" + fields::query_prefix + "` must be a string.");+ }+ }++ if(model_parameters.count("api_key") != 0) {+ auto res = TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), model_parameters[fields::api_key].get<std::string>(), num_dim);if(res.ok()) {field_json[fields::num_dim] = num_dim;} else {diff --git a/src/index.cpp b/src/index.cppindex 52b7d647..69b12609 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -6348,7 +6348,7 @@ Option<bool> Index::embed_fields(nlohmann::json& document,const tsl::htrie_map<char, field>& embedding_fields,const tsl::htrie_map<char, field> & search_schema) {for(const auto& field : embedding_fields) {- std::string text_to_embed = "passage: ";+ std::string text_to_embed = TextEmbedderManager::get_indexing_prefix(field.model_parameters);for(const auto& field_name : field.embed_from) {auto field_it = search_schema.find(field_name);if(field_it.value().type == field_types::STRING) {@@ -6378,7 +6378,7 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,for(const auto& field : embedding_fields) {std::vector<std::string> text_to_embed;for(auto& document : documents) {- std::string text = "passage: ";+ std::string text = TextEmbedderManager::get_indexing_prefix(field.model_parameters);for(const auto& field_name : field.embed_from) {auto field_it = search_schema.find(field_name);if(field_it.value().type == field_types::STRING) {diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex de2f4cc8..419f0a5b 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -6,13 +6,13 @@#include <sstream>#include <filesystem>-TextEmbedder::TextEmbedder(const std::string& model_path) {+TextEmbedder::TextEmbedder(const std::string& model_name) {// create environmentOrt::SessionOptions session_options;- std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_path);+ std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_name);LOG(INFO) << "Loading model from: " << abs_path;session_ = std::make_unique<Ort::Session>(env_, abs_path.c_str(), session_options);- std::ifstream stream(TextEmbedderManager::get_absolute_vocab_path());+ std::ifstream stream(TextEmbedderManager::get_absolute_vocab_path(model_name));std::stringstream ss;ss << stream.rdbuf();auto vocab_ = ss.str();@@ -29,7 +29,7 @@ TextEmbedder::TextEmbedder(const std::string& model_path) {}}-TextEmbedder::TextEmbedder(const std::string& openai_model_path, const std::string& openai_api_key) : openai_api_key(openai_api_key), openai_model_path(openai_model_path) {+TextEmbedder::TextEmbedder(const std::string& openai_model_path, const std::string& api_key) : api_key(api_key), openai_model_path(openai_model_path) {}@@ -68,7 +68,7 @@ Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {HttpClient& client = HttpClient::get_instance();std::unordered_map<std::string, std::string> headers;std::map<std::string, std::string> res_headers;- headers["Authorization"] = "Bearer " + openai_api_key;+ headers["Authorization"] = "Bearer " + api_key;headers["Content-Type"] = "application/json";std::string res;nlohmann::json req_body;@@ -88,18 +88,25 @@ Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);std::vector<Ort::Value> input_tensors;std::vector<std::vector<int64_t>> input_shapes;- std::vector<const char*> input_node_names = {"input_ids", "attention_mask", "token_type_ids"};+ std::vector<const char*> input_node_names = {"input_ids", "attention_mask"};+ // If model is DistilBERT, it has 2 inputs, else it has 3 inputs+ if(session_->GetInputCount() == 3) {+ input_node_names.push_back("token_type_ids");+ }input_shapes.push_back({1, static_cast<int64_t>(encoded_input.input_ids.size())});input_shapes.push_back({1, static_cast<int64_t>(encoded_input.attention_mask.size())});- input_shapes.push_back({1, static_cast<int64_t>(encoded_input.token_type_ids.size())});+ if(session_->GetInputCount() == 3) {+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.token_type_ids.size())});+ }input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.input_ids.data(), encoded_input.input_ids.size(), input_shapes[0].data(), input_shapes[0].size()));input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.attention_mask.data(), encoded_input.attention_mask.size(), input_shapes[1].data(), input_shapes[1].size()));- input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.token_type_ids.data(), encoded_input.token_type_ids.size(), input_shapes[2].data(), input_shapes[2].size()));+ if(session_->GetInputCount() == 3) {+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.token_type_ids.data(), encoded_input.token_type_ids.size(), input_shapes[2].data(), input_shapes[2].size()));+ }//LOG(INFO) << "Running model";// create output tensor objectstd::vector<const char*> output_node_names = {output_tensor_name.c_str()};- std::vector<int64_t> output_node_dims {1, static_cast<int64_t>(encoded_input.input_ids.size()), 384}; // batch_size x seq_length x hidden_sizeauto output_tensor = session_->Run(Ort::RunOptions{nullptr}, input_node_names.data(), input_tensors.data(), input_tensors.size(), output_node_names.data(), output_node_names.size());std::vector<std::vector<float>> output;float* data = output_tensor[0].GetTensorMutableData<float>();@@ -132,7 +139,7 @@ Option<std::vector<std::vector<float>>> TextEmbedder::batch_embed(const std::vec// remove "openai/" prefixreq_body["model"] = openai_model_path.substr(7);std::unordered_map<std::string, std::string> headers;- headers["Authorization"] = "Bearer " + openai_api_key;+ headers["Authorization"] = "Bearer " + api_key;headers["Content-Type"] = "application/json";std::map<std::string, std::string> res_headers;std::string res;@@ -157,11 +164,22 @@ TextEmbedder::~TextEmbedder() {}-bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& num_dims) {- LOG(INFO) << "Loading model: " << model_path;+bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& num_dims) {+ LOG(INFO) << "Loading model: " << model_name;++ if(TextEmbedderManager::get_instance().is_public_model(model_name)) {+ TextEmbedderManager::get_instance().download_public_model(model_name);+ }++ std::string vocab_path = TextEmbedderManager::get_absolute_vocab_path(model_name);+ if(!std::filesystem::exists(vocab_path)) {+ LOG(ERROR) << "Vocab file not found: " << vocab_path;+ return false;+ }+Ort::SessionOptions session_options;Ort::Env env;- std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_path);+ std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_name);if(!std::filesystem::exists(abs_path)) {LOG(ERROR) << "Model file not found: " << abs_path;@@ -169,8 +187,8 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& n}Ort::Session session(env, abs_path.c_str(), session_options);- if(session.GetInputCount() != 3) {- LOG(ERROR) << "Invalid model: input count is not 3";+ if(session.GetInputCount() != 3 && session.GetInputCount() != 2) {+ LOG(ERROR) << "Invalid model: input count is not 3 or 2";return false;}Ort::AllocatorWithDefaultOptions allocator;@@ -186,10 +204,15 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& nreturn false;}- auto token_type_ids_name = session.GetInputNameAllocated(2, allocator);- if (std::strcmp(token_type_ids_name.get(), "token_type_ids") != 0) {- LOG(ERROR) << "Invalid model: token_type_ids tensor not found";- return false;++ if(session.GetInputCount() == 3) {+ auto token_type_ids_name = session.GetInputNameAllocated(2, allocator);+ if (std::strcmp(token_type_ids_name.get(), "token_type_ids") != 0) {+ LOG(ERROR) << "Invalid model: token_type_ids tensor not found";+ return false;+ }+ } else {+ LOG(INFO) << "Model is DistilBERT";}auto output_tensor_count = session.GetOutputCount();@@ -212,15 +235,15 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& n}-Option<bool> TextEmbedder::is_model_valid(const std::string openai_model_path, const std::string openai_api_key, unsigned int& num_dims) {- if (openai_model_path.empty() || openai_api_key.empty() || openai_model_path.length() < 7) {+Option<bool> TextEmbedder::is_model_valid(const std::string openai_model_path, const std::string api_key, unsigned int& num_dims) {+ if (openai_model_path.empty() || api_key.empty() || openai_model_path.length() < 7) {return Option<bool>(400, "Invalid OpenAI model path or API key");}HttpClient& client = HttpClient::get_instance();std::unordered_map<std::string, std::string> headers;std::map<std::string, std::string> res_headers;- headers["Authorization"] = "Bearer " + openai_api_key;+ headers["Authorization"] = "Bearer " + api_key;std::string res;auto res_code = client.get_response(TextEmbedder::OPENAI_LIST_MODELS, res, res_headers, headers);if (res_code != 200) {@@ -243,7 +266,6 @@ Option<bool> TextEmbedder::is_model_valid(const std::string openai_model_path, creturn Option<bool>(400, "OpenAI model not found");}-// This part is hard coded for now. Because OpenAI API does not provide a way to get the output dimensions of the model.if(model_name.find("-ada-") != std::string::npos) {if(model_name.substr(model_name.length() - 3) == "002") {diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 0d9ccad7..a610ae45 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -55,7 +55,6 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.set_program_name("./typesense-server");options.add<std::string>("data-dir", 'd', "Directory where data will be stored.", true);- options.add<std::string>("model-dir", '\0', "Directory where text embedding models will be stored.", false, "");options.add<std::string>("api-key", 'a', "API key that allows all operations.", true);options.add<std::string>("search-only-api-key", 's', "[DEPRECATED: use API key management end-point] API key that allows only searches.", false);@@ -422,12 +421,7 @@ int run_server(const Config & config, const std::string & version, void (*masterif(!rate_limit_manager_init.ok()) {LOG(INFO) << "Failed to initialize rate limit manager: " << rate_limit_manager_init.error();}-- if(config.get_model_dir().size() > 0) {- LOG(INFO) << "Loading text embedding models from " << config.get_model_dir();- TextEmbedderManager::set_model_dir(config.get_model_dir());- TextEmbedderManager::download_default_model();- }+ TextEmbedderManager::set_model_dir(config.get_data_dir() + "/models");// first we start the peering servicediff --git a/src/validator.cpp b/src/validator.cppindex f66eaf9d..cb36571f 100644--- a/src/validator.cpp+++ b/src/validator.cpp@@ -594,9 +594,6 @@ Option<bool> validator_t::validate_embed_fields(const nlohmann::json& document,const tsl::htrie_map<char, field>& embedding_fields,const tsl::htrie_map<char, field> & search_schema,const bool& error_if_field_not_found) {- if(!embedding_fields.empty() && TextEmbedderManager::model_dir.empty()) {- return Option<bool>(400, "Text embedding is not enabled. Please set `model-dir` at startup.");- }for(const auto& field : embedding_fields) {for(const auto& field_name : field.embed_from) {auto schema_field_it = search_schema.find(field_name);diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cppindex 14e6e27b..ba1768ba 100644--- a/test/collection_all_fields_test.cpp+++ b/test/collection_all_fields_test.cpp@@ -1610,23 +1610,23 @@ TEST_F(CollectionAllFieldsTest, EmbedFromFieldJSONInvalidField) {ASSERT_EQ("Property `embed_from` can only refer to string or string array fields.", field_op.error());}-TEST_F(CollectionAllFieldsTest, EmbedFromFieldNoModelDir) {- TextEmbedderManager::model_dir = std::string();- nlohmann::json field_json;- field_json["name"] = "embedding";- field_json["type"] = "float[]";- field_json["embed_from"] = {"name"};+// TEST_F(CollectionAllFieldsTest, EmbedFromFieldNoModelDir) {+// TextEmbedderManager::model_dir = std::string();+// nlohmann::json field_json;+// field_json["name"] = "embedding";+// field_json["type"] = "float[]";+// field_json["embed_from"] = {"name"};- std::vector<field> fields;- std::string fallback_field_type;- auto arr = nlohmann::json::array();- arr.push_back(field_json);+// std::vector<field> fields;+// std::string fallback_field_type;+// auto arr = nlohmann::json::array();+// arr.push_back(field_json);- auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);+// auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);- ASSERT_FALSE(field_op.ok());- ASSERT_EQ("Text embedding is not enabled. Please set `model-dir` at startup.", field_op.error());-}+// ASSERT_FALSE(field_op.ok());+// ASSERT_EQ("Text embedding is not enabled. Please set `model-dir` at startup.", field_op.error());+// }TEST_F(CollectionAllFieldsTest, EmbedFromNotArray) {TextEmbedderManager::model_dir = "/tmp/models";@@ -1667,7 +1667,6 @@ TEST_F(CollectionAllFieldsTest, ModelParametersWithoutEmbedFrom) {TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {TextEmbedderManager::model_dir = "/tmp/typesense_test/models";- TextEmbedderManager::download_default_model();field embedding = field("embedding", field_types::FLOAT_ARRAY, false);embedding.embed_from.push_back("name");diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cppindex 88d8e940..42bab54a 100644--- a/test/collection_schema_change_test.cpp+++ b/test/collection_schema_change_test.cpp@@ -1456,7 +1456,6 @@ TEST_F(CollectionSchemaChangeTest, UpdateSchemaWithNewEmbeddingField) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -1495,7 +1494,6 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -1544,7 +1542,6 @@ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 218e199a..1346fbc4 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -4622,7 +4622,6 @@ TEST_F(CollectionTest, SemanticSearchTest) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4657,7 +4656,6 @@ TEST_F(CollectionTest, InvalidSemanticSearch) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);LOG(INFO) << "op.error(): " << op.error();@@ -4687,7 +4685,6 @@ TEST_F(CollectionTest, HybridSearch) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4721,7 +4718,7 @@ TEST_F(CollectionTest, HybridSearch) {// })"_json;// TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");-// TextEmbedderManager::download_default_model();+//// auto op = collectionManager.create_collection(schema);// ASSERT_TRUE(op.ok());@@ -4749,7 +4746,6 @@ TEST_F(CollectionTest, HybridSearchRankFusionTest) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4823,7 +4819,6 @@ TEST_F(CollectionTest, WildcardSearchWithEmbeddingField) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4857,7 +4852,6 @@ TEST_F(CollectionTest, EmbedStringArrayField) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4883,7 +4877,6 @@ TEST_F(CollectionTest, MissingFieldForEmbedding) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4910,7 +4903,6 @@ TEST_F(CollectionTest, WrongTypeForEmbedding) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4934,7 +4926,6 @@ TEST_F(CollectionTest, WrongTypeOfElementForEmbeddingInStringArray) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -4958,7 +4949,6 @@ TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());@@ -5005,21 +4995,21 @@ TEST_F(CollectionTest, DISABLED_CreateOpenAIEmbeddingField) {]})"_json;- if (std::getenv("OPENAI_API_KEY") == nullptr) {- LOG(INFO) << "Skipping test as OPENAI_API_KEY is not set.";+ if (std::getenv("api_key") == nullptr) {+ LOG(INFO) << "Skipping test as api_key is not set.";return;}- auto openai_api_key = std::string(std::getenv("OPENAI_API_KEY"));- schema["fields"][1]["model_parameters"]["openai_api_key"] = openai_api_key;+ auto api_key = std::string(std::getenv("api_key"));+ schema["fields"][1]["model_parameters"]["api_key"] = api_key;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());auto summary = op.get()->get_summary_json();ASSERT_EQ("openai/text-embedding-ada-002", summary["fields"][1]["model_parameters"]["model_name"]);ASSERT_EQ(1536, summary["fields"][1]["num_dim"]);- // make sure openai_api_key is <hidden>- ASSERT_EQ("<hidden>", summary["fields"][1]["model_parameters"]["openai_api_key"]);+ // make sure api_key is <hidden>+ ASSERT_EQ("<hidden>", summary["fields"][1]["model_parameters"]["api_key"]);nlohmann::json doc;doc["name"] = "butter";diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex c78e004e..3df3bde5 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -688,7 +688,6 @@ TEST_F(CollectionVectorTest, HybridSearchWithExplicitVector) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();Collection* coll1 = collectionManager.create_collection(schema).get();@@ -787,7 +786,6 @@ TEST_F(CollectionVectorTest, EmbeddingFieldVectorIndexTest) {})"_json;TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");- TextEmbedderManager::download_default_model();auto op = collectionManager.create_collection(schema);ASSERT_TRUE(op.ok());