Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Review Changes

ozanarmagana571d64d5695
diff --git a/include/field.h b/include/field.h
index 5fbb386e..0de91ceb 100644
--- a/include/field.h
+++ b/include/field.h
@@ -52,7 +52,12 @@ namespace fields {
static const std::string reference = "reference";
static const std::string embed_from = "embed_from";
static const std::string model_name = "model_name";
- static const std::string openai_api_key = "openai_api_key";
+
+ // Some models require additional parameters to be passed to the model during indexing/querying
+ // For e.g. e5-small model requires prefix "passage:" for indexing and "query:" for querying
+ static const std::string indexing_prefix = "indexing_prefix";
+ static const std::string query_prefix = "query_prefix";
+ static const std::string api_key = "api_key";
static const std::string model_parameters = "model_parameters";
}
@@ -326,10 +331,6 @@ struct field {
field_val[fields::embed_from] = field.embed_from;
if(!field.model_parameters.empty()) {
field_val[fields::model_parameters] = field.model_parameters;
- // hide openai api key
- if(field_val[fields::model_parameters].count("openai_api_key") != 0) {
- field_val[fields::model_parameters]["openai_api_key"] = "<hidden>";
- }
}
}
fields_json.push_back(field_val);
@@ -429,9 +430,9 @@ struct field {
for(nlohmann::json & field_json: fields_json) {
if(field_json.count(fields::embed_from) != 0) {
- if(TextEmbedderManager::model_dir.empty()) {
- return Option<bool>(400, "Text embedding is not enabled. Please set `model-dir` at startup.");
- }
+ // if(TextEmbedderManager::model_dir.empty()) {
+ // return Option<bool>(400, "Text embedding is not enabled. Please set `model-dir` at startup.");
+ // }
if(!field_json[fields::embed_from].is_array()) {
return Option<bool>(400, "Property `" + fields::embed_from + "` must be an array.");
diff --git a/include/text_embedder.h b/include/text_embedder.h
index 869644f8..3ca0d3d0 100644
--- a/include/text_embedder.h
+++ b/include/text_embedder.h
@@ -15,17 +15,17 @@ struct encoded_input_t {
class TextEmbedder {
public:
TextEmbedder(const std::string& model_path);
- TextEmbedder(const std::string& openai_model_path, const std::string& openai_api_key);
+ TextEmbedder(const std::string& openai_model_path, const std::string& api_key);
~TextEmbedder();
Option<std::vector<float>> Embed(const std::string& text);
Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs);
bool is_openai() {
- return !openai_api_key.empty();
+ return !api_key.empty();
}
static bool is_model_valid(const std::string& model_path, unsigned int& num_dims);
- static Option<bool> is_model_valid(const std::string openai_model_path, const std::string openai_api_key, unsigned int& num_dims);
+ static Option<bool> is_model_valid(const std::string openai_model_path, const std::string api_key, unsigned int& num_dims);
private:
std::unique_ptr<Ort::Session> session_;
Ort::Env env_;
@@ -33,7 +33,7 @@ class TextEmbedder {
std::unique_ptr<BertTokenizer> tokenizer_;
static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);
std::string output_tensor_name;
- std::string openai_api_key;
+ std::string api_key;
std::string openai_model_path;
static constexpr char* OPENAI_LIST_MODELS = "https://api.openai.com/v1/models";
static constexpr char* OPENAI_CREATE_EMBEDDING = "https://api.openai.com/v1/embeddings";
diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.h
index f19bc8d9..dfad1b10 100644
--- a/include/text_embedder_manager.h
+++ b/include/text_embedder_manager.h
@@ -1,6 +1,7 @@
#pragma once
#include <memory>
#include <filesystem>
+#include <mutex>
#include <unordered_map>
#include <openssl/md5.h>
#include <fstream>
@@ -9,6 +10,14 @@
#include "text_embedder.h"
+struct text_embedding_model {
+ std::string model_url;
+ std::string model_md5;
+ std::string vocab_url;
+ std::string vocab_md5;
+};
+
+
// singleton class
class TextEmbedderManager {
public:
@@ -23,35 +32,52 @@ public:
TextEmbedderManager& operator=(const TextEmbedderManager&) = delete;
TextEmbedder* get_text_embedder(const nlohmann::json& model_parameters) {
- if(model_parameters.count("model_name") == 0) {
- if(text_embedders[DEFAULT_MODEL_NAME] == nullptr) {
- text_embedders[DEFAULT_MODEL_NAME] = std::make_shared<TextEmbedder>(DEFAULT_MODEL_NAME);
- }
- return text_embedders[DEFAULT_MODEL_NAME].get();
- } else {
- const std::string& model_name = model_parameters.at("model_name");
- if(text_embedders[model_name] == nullptr) {
- if(model_parameters.count("openai_api_key") == 0) {
- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name);
- } else {
- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("openai_api_key"));
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
+ const std::string& model_name = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_NAME : model_parameters.at("model_name");
+ if(text_embedders[model_name] == nullptr) {
+ if(model_parameters.count("api_key") == 0) {
+ // download the model if it doesn't exist
+ if(is_public_model(model_name)) {
+ download_public_model(model_name);
}
+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name);
+ } else {
+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("api_key"));
}
- return text_embedders[model_name].get();
}
-
+ return text_embedders[model_name].get();
}
void delete_text_embedder(const std::string& model_path) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
if (text_embedders.find(model_path) != text_embedders.end()) {
text_embedders.erase(model_path);
}
}
void delete_all_text_embedders() {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
text_embedders.clear();
}
+ static const std::string get_indexing_prefix(const nlohmann::json& model_parameters) {
+ std::string val = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_INDEXING_PREFIX : model_parameters["indexing_prefix"].get<std::string>();
+ if(!val.empty()) {
+ val += " ";
+ }
+
+ return val;
+ }
+
+ static const std::string get_query_prefix(const nlohmann::json& model_parameters) {
+ std::string val = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_QUERY_PREFIX : model_parameters["query_prefix"].get<std::string>();
+ if(!val.empty()) {
+ val += " ";
+ }
+
+ return val;
+ }
+
static void set_model_dir(const std::string& dir) {
// create the directory if it doesn't exist
if(!std::filesystem::exists(dir)) {
@@ -68,33 +94,38 @@ public:
delete_all_text_embedders();
}
- static constexpr char* DEFAULT_MODEL_URL = "https://huggingface.co/typesense/models/resolve/main/e5-small/model.onnx";
static constexpr char* DEFAULT_MODEL_NAME = "ts-e5-small";
- static constexpr char* DEFAULT_VOCAB_URL = "https://huggingface.co/typesense/models/resolve/main/e5-small/vocab.txt";
- static constexpr char* DEFAULT_VOCAB_NAME = "vocab.txt";
- static constexpr char* DEFAULT_VOCAB_MD5 = "6480d5d8528ce344256daf115d4965e";
- static constexpr char* DEFAULT_MODEL_MD5 = "3d421dc72859a72368c106415cdebf2";
+ static constexpr char* DEFAULT_MODEL_INDEXING_PREFIX = "passage:";
+ static constexpr char* DEFAULT_MODEL_QUERY_PREFIX = "query:";
inline static std::string model_dir = "";
inline static const std::string get_absolute_model_path(const std::string& model_name) {
if(model_dir.back() != '/') {
- if(model_name.front() != '/') {
- return model_dir + "/" + model_name + ".onnx";
- } else {
- return model_dir + model_name + ".onnx";
+ // create subdir <model_name> if it doesn't exist
+ if(!std::filesystem::exists(model_dir + "/" + model_name)) {
+ std::filesystem::create_directories(model_dir + "/" + model_name);
}
+ return model_dir + "/" + model_name + "/" + "model.onnx";
} else {
- if(model_name.front() != '/') {
- return model_dir + model_name + ".onnx";
- } else {
- return model_dir + "/" + model_name + ".onnx";
+ // create subdir <model_name> if it doesn't exist
+ if(!std::filesystem::exists(model_dir + model_name)) {
+ std::filesystem::create_directories(model_dir + model_name);
}
+ return model_dir + model_name + "/" + "model.onnx";
}
};
- inline static const std::string get_absolute_vocab_path() {
+ inline static const std::string get_absolute_vocab_path(const std::string& model_name) {
if(model_dir.back() != '/') {
- return model_dir + "/" + TextEmbedderManager::DEFAULT_VOCAB_NAME;
+ // create subdir <model_name> if it doesn't exist
+ if(!std::filesystem::exists(model_dir + "/" + model_name)) {
+ std::filesystem::create_directories(model_dir + "/" + model_name);
+ }
+ return model_dir + "/" + model_name + "/" + "vocab.txt";
} else {
- return model_dir + TextEmbedderManager::DEFAULT_VOCAB_NAME;
+ // create subdir <model_name> if it doesn't exist
+ if(!std::filesystem::exists(model_dir + model_name)) {
+ std::filesystem::create_directories(model_dir + model_name);
+ }
+ return model_dir + model_name + "/" + "vocab.txt";
}
}
inline static const bool check_md5(const std::string& file_path, const std::string& target_md5) {
@@ -111,26 +142,49 @@ public:
}
return res.str() == target_md5;
}
- inline static void download_default_model() {
+ void download_public_model(const std::string& model_name) {
HttpClient& httpClient = HttpClient::get_instance();
- if(!check_md5(get_absolute_model_path(DEFAULT_MODEL_NAME), DEFAULT_MODEL_MD5)) {
- LOG(INFO) << "Downloading default model";
- long res = httpClient.download_file(DEFAULT_MODEL_URL, get_absolute_model_path(DEFAULT_MODEL_NAME));
+ auto model = public_models[model_name];
+ if(!check_md5(get_absolute_model_path(model_name), model.model_md5)) {
+ LOG(INFO) << "Downloading public model: " << model_name;
+ long res = httpClient.download_file(model.model_url, get_absolute_model_path(model_name));
if(res != 200) {
- LOG(INFO) << "Failed to download default model: " << res;
+ LOG(INFO) << "Failed to download public model " << model_name << ": " << res;
}
}
- if(!check_md5(get_absolute_vocab_path(), DEFAULT_VOCAB_MD5)) {
- LOG(INFO) << "Downloading default vocab";
- long res = httpClient.download_file(DEFAULT_VOCAB_URL, get_absolute_vocab_path());
+ if(!check_md5(get_absolute_vocab_path(model_name), model.vocab_md5)) {
+ LOG(INFO) << "Downloading default vocab for model: " << model_name;
+ long res = httpClient.download_file(model.vocab_url, get_absolute_vocab_path(model_name));
if(res != 200) {
- LOG(INFO) << "Failed to download default vocab: " << res;
+ LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;
}
}
}
+
+ const bool is_public_model(const std::string& model_name) {
+ return public_models.find(model_name) != public_models.end();
+ }
private:
- TextEmbedderManager() = default;
+ TextEmbedderManager() {
+ public_models[DEFAULT_MODEL_NAME] = text_embedding_model{
+ "https://models.typesense.org/public/e5-small/model.onnx",
+ "3d421dc72859a72368c106415cdebf2",
+ "https://models.typesense.org/public/e5-small/vocab.txt",
+ "6480d5d8528ce344256daf115d4965e"};
+ public_models["ts-all-MiniLM-L12-v2"] = text_embedding_model{
+ "https://models.typesense.org/public/all-miniLM-L12-v2/model.onnx",
+ "6d196b8f7a8d8abcfb08afcac1704302",
+ "https://models.typesense.org/public/all-miniLM-L12-v2/vocab.txt",
+ "9d2131a9a433502abeb512978d452ecc"};
+ public_models["ts-distiluse-base-multilingual-cased-v2"] = text_embedding_model{
+ "https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/model.onnx",
+ "91f535cd7ca87359b9bfd1d87f9e452e",
+ "https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/vocab.txt",
+ "0f05e1b7420dea2db8ade63eb5b80f7a"};
+ }
std::unordered_map<std::string, std::shared_ptr<TextEmbedder>> text_embedders;
+ std::unordered_map<std::string, text_embedding_model> public_models;
+ std::mutex text_embedders_mutex;
};
diff --git a/include/tsconfig.h b/include/tsconfig.h
index 57796930..cff6f85e 100644
--- a/include/tsconfig.h
+++ b/include/tsconfig.h
@@ -11,7 +11,6 @@ class Config {
private:
std::string data_dir;
std::string log_dir;
- std::string model_dir;
std::string api_key;
@@ -112,10 +111,6 @@ public:
this->log_dir = log_dir;
}
- void set_model_dir(const std::string & model_dir) {
- this->model_dir = model_dir;
- }
-
void set_api_key(const std::string & api_key) {
this->api_key = api_key;
}
@@ -183,9 +178,6 @@ public:
return this->log_dir;
}
- std::string get_model_dir() const {
- return this->model_dir;
- }
std::string get_api_key() const {
return this->api_key;
@@ -592,9 +584,6 @@ public:
this->reset_peers_on_error = (reset_peers_on_error_str == "true");
}
- if(reader.Exists("server", "model-dir")) {
- this->model_dir = reader.Get("server", "model-dir", "");
- }
}
void load_config_cmd_args(cmdline::parser & options) {
@@ -610,10 +599,6 @@ public:
this->api_key = options.get<std::string>("api-key");
}
- if(options.exist("model-dir")) {
- LOG(INFO) << "model-dir found";
- this->model_dir = options.get<std::string>("model-dir");
- }
// @deprecated
if(options.exist("search-only-api-key")) {
diff --git a/src/collection.cpp b/src/collection.cpp
index 525cc753..a4723772 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -247,8 +247,8 @@ nlohmann::json Collection::get_summary_json() const {
if(coll_field.model_parameters.size() > 0) {
field_json[fields::model_parameters] = coll_field.model_parameters;
// Hide OpenAI API key from the response.
- if(field_json[fields::model_parameters].count(fields::openai_api_key) != 0) {
- field_json[fields::model_parameters][fields::openai_api_key] = "<hidden>";
+ if(field_json[fields::model_parameters].count(fields::api_key) != 0) {
+ field_json[fields::model_parameters][fields::api_key] = "<hidden>";
}
}
@@ -1177,10 +1177,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
return Option<nlohmann::json>(400, error);
}
- if(TextEmbedderManager::model_dir.empty()) {
- std::string error = "Text embedding is not enabled. Please set `model-dir` at startup.";
- return Option<nlohmann::json>(400, error);
- }
+ // if(TextEmbedderManager::model_dir.empty()) {
+ // std::string error = "Text embedding is not enabled. Please set `model-dir` at startup.";
+ // return Option<nlohmann::json>(400, error);
+ // }
if(raw_query == "*") {
std::string error = "Wildcard query is not supported for embedding fields.";
@@ -1190,7 +1190,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();
auto embedder = embedder_manager.get_text_embedder(search_field.model_parameters);
- std::string embed_query = "query: " + raw_query;
+ std::string embed_query = embedder_manager.get_query_prefix(search_field.model_parameters) + raw_query;
auto embedding_op = embedder->Embed(embed_query);
if(!embedding_op.ok()) {
return Option<nlohmann::json>(400, embedding_op.error());
diff --git a/src/field.cpp b/src/field.cpp
index ec9e8f7c..92909b9a 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -689,8 +689,21 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
if(model_parameters[fields::model_name].get<std::string>().empty()) {
return Option<bool>(400, "Property `" + fields::model_name + "` must be a non-empty string.");
}
- if(model_parameters.count("openai_api_key") != 0) {
- auto res = TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), model_parameters[fields::openai_api_key].get<std::string>(), num_dim);
+
+ if(model_parameters.count(fields::indexing_prefix) != 0) {
+ if(!model_parameters[fields::indexing_prefix].is_string()) {
+ return Option<bool>(400, "Property `" + fields::indexing_prefix + "` must be a string.");
+ }
+ }
+
+ if(model_parameters.count(fields::query_prefix) != 0) {
+ if(!model_parameters[fields::query_prefix].is_string()) {
+ return Option<bool>(400, "Property `" + fields::query_prefix + "` must be a string.");
+ }
+ }
+
+ if(model_parameters.count("api_key") != 0) {
+ auto res = TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), model_parameters[fields::api_key].get<std::string>(), num_dim);
if(res.ok()) {
field_json[fields::num_dim] = num_dim;
} else {
diff --git a/src/index.cpp b/src/index.cpp
index 52b7d647..69b12609 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -6348,7 +6348,7 @@ Option<bool> Index::embed_fields(nlohmann::json& document,
const tsl::htrie_map<char, field>& embedding_fields,
const tsl::htrie_map<char, field> & search_schema) {
for(const auto& field : embedding_fields) {
- std::string text_to_embed = "passage: ";
+ std::string text_to_embed = TextEmbedderManager::get_indexing_prefix(field.model_parameters);
for(const auto& field_name : field.embed_from) {
auto field_it = search_schema.find(field_name);
if(field_it.value().type == field_types::STRING) {
@@ -6378,7 +6378,7 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,
for(const auto& field : embedding_fields) {
std::vector<std::string> text_to_embed;
for(auto& document : documents) {
- std::string text = "passage: ";
+ std::string text = TextEmbedderManager::get_indexing_prefix(field.model_parameters);
for(const auto& field_name : field.embed_from) {
auto field_it = search_schema.find(field_name);
if(field_it.value().type == field_types::STRING) {
diff --git a/src/text_embedder.cpp b/src/text_embedder.cpp
index de2f4cc8..419f0a5b 100644
--- a/src/text_embedder.cpp
+++ b/src/text_embedder.cpp
@@ -6,13 +6,13 @@
#include <sstream>
#include <filesystem>
-TextEmbedder::TextEmbedder(const std::string& model_path) {
+TextEmbedder::TextEmbedder(const std::string& model_name) {
// create environment
Ort::SessionOptions session_options;
- std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_path);
+ std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_name);
LOG(INFO) << "Loading model from: " << abs_path;
session_ = std::make_unique<Ort::Session>(env_, abs_path.c_str(), session_options);
- std::ifstream stream(TextEmbedderManager::get_absolute_vocab_path());
+ std::ifstream stream(TextEmbedderManager::get_absolute_vocab_path(model_name));
std::stringstream ss;
ss << stream.rdbuf();
auto vocab_ = ss.str();
@@ -29,7 +29,7 @@ TextEmbedder::TextEmbedder(const std::string& model_path) {
}
}
-TextEmbedder::TextEmbedder(const std::string& openai_model_path, const std::string& openai_api_key) : openai_api_key(openai_api_key), openai_model_path(openai_model_path) {
+TextEmbedder::TextEmbedder(const std::string& openai_model_path, const std::string& api_key) : api_key(api_key), openai_model_path(openai_model_path) {
}
@@ -68,7 +68,7 @@ Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {
HttpClient& client = HttpClient::get_instance();
std::unordered_map<std::string, std::string> headers;
std::map<std::string, std::string> res_headers;
- headers["Authorization"] = "Bearer " + openai_api_key;
+ headers["Authorization"] = "Bearer " + api_key;
headers["Content-Type"] = "application/json";
std::string res;
nlohmann::json req_body;
@@ -88,18 +88,25 @@ Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {
Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtAllocatorType::OrtArenaAllocator, OrtMemType::OrtMemTypeDefault);
std::vector<Ort::Value> input_tensors;
std::vector<std::vector<int64_t>> input_shapes;
- std::vector<const char*> input_node_names = {"input_ids", "attention_mask", "token_type_ids"};
+ std::vector<const char*> input_node_names = {"input_ids", "attention_mask"};
+ // If model is DistilBERT, it has 2 inputs, else it has 3 inputs
+ if(session_->GetInputCount() == 3) {
+ input_node_names.push_back("token_type_ids");
+ }
input_shapes.push_back({1, static_cast<int64_t>(encoded_input.input_ids.size())});
input_shapes.push_back({1, static_cast<int64_t>(encoded_input.attention_mask.size())});
- input_shapes.push_back({1, static_cast<int64_t>(encoded_input.token_type_ids.size())});
+ if(session_->GetInputCount() == 3) {
+ input_shapes.push_back({1, static_cast<int64_t>(encoded_input.token_type_ids.size())});
+ }
input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.input_ids.data(), encoded_input.input_ids.size(), input_shapes[0].data(), input_shapes[0].size()));
input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.attention_mask.data(), encoded_input.attention_mask.size(), input_shapes[1].data(), input_shapes[1].size()));
- input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.token_type_ids.data(), encoded_input.token_type_ids.size(), input_shapes[2].data(), input_shapes[2].size()));
+ if(session_->GetInputCount() == 3) {
+ input_tensors.push_back(Ort::Value::CreateTensor<int64_t>(memory_info, encoded_input.token_type_ids.data(), encoded_input.token_type_ids.size(), input_shapes[2].data(), input_shapes[2].size()));
+ }
//LOG(INFO) << "Running model";
// create output tensor object
std::vector<const char*> output_node_names = {output_tensor_name.c_str()};
- std::vector<int64_t> output_node_dims {1, static_cast<int64_t>(encoded_input.input_ids.size()), 384}; // batch_size x seq_length x hidden_size
auto output_tensor = session_->Run(Ort::RunOptions{nullptr}, input_node_names.data(), input_tensors.data(), input_tensors.size(), output_node_names.data(), output_node_names.size());
std::vector<std::vector<float>> output;
float* data = output_tensor[0].GetTensorMutableData<float>();
@@ -132,7 +139,7 @@ Option<std::vector<std::vector<float>>> TextEmbedder::batch_embed(const std::vec
// remove "openai/" prefix
req_body["model"] = openai_model_path.substr(7);
std::unordered_map<std::string, std::string> headers;
- headers["Authorization"] = "Bearer " + openai_api_key;
+ headers["Authorization"] = "Bearer " + api_key;
headers["Content-Type"] = "application/json";
std::map<std::string, std::string> res_headers;
std::string res;
@@ -157,11 +164,22 @@ TextEmbedder::~TextEmbedder() {
}
-bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& num_dims) {
- LOG(INFO) << "Loading model: " << model_path;
+bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& num_dims) {
+ LOG(INFO) << "Loading model: " << model_name;
+
+ if(TextEmbedderManager::get_instance().is_public_model(model_name)) {
+ TextEmbedderManager::get_instance().download_public_model(model_name);
+ }
+
+ std::string vocab_path = TextEmbedderManager::get_absolute_vocab_path(model_name);
+ if(!std::filesystem::exists(vocab_path)) {
+ LOG(ERROR) << "Vocab file not found: " << vocab_path;
+ return false;
+ }
+
Ort::SessionOptions session_options;
Ort::Env env;
- std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_path);
+ std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_name);
if(!std::filesystem::exists(abs_path)) {
LOG(ERROR) << "Model file not found: " << abs_path;
@@ -169,8 +187,8 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& n
}
Ort::Session session(env, abs_path.c_str(), session_options);
- if(session.GetInputCount() != 3) {
- LOG(ERROR) << "Invalid model: input count is not 3";
+ if(session.GetInputCount() != 3 && session.GetInputCount() != 2) {
+ LOG(ERROR) << "Invalid model: input count is not 3 or 2";
return false;
}
Ort::AllocatorWithDefaultOptions allocator;
@@ -186,10 +204,15 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& n
return false;
}
- auto token_type_ids_name = session.GetInputNameAllocated(2, allocator);
- if (std::strcmp(token_type_ids_name.get(), "token_type_ids") != 0) {
- LOG(ERROR) << "Invalid model: token_type_ids tensor not found";
- return false;
+
+ if(session.GetInputCount() == 3) {
+ auto token_type_ids_name = session.GetInputNameAllocated(2, allocator);
+ if (std::strcmp(token_type_ids_name.get(), "token_type_ids") != 0) {
+ LOG(ERROR) << "Invalid model: token_type_ids tensor not found";
+ return false;
+ }
+ } else {
+ LOG(INFO) << "Model is DistilBERT";
}
auto output_tensor_count = session.GetOutputCount();
@@ -212,15 +235,15 @@ bool TextEmbedder::is_model_valid(const std::string& model_path, unsigned int& n
}
-Option<bool> TextEmbedder::is_model_valid(const std::string openai_model_path, const std::string openai_api_key, unsigned int& num_dims) {
- if (openai_model_path.empty() || openai_api_key.empty() || openai_model_path.length() < 7) {
+Option<bool> TextEmbedder::is_model_valid(const std::string openai_model_path, const std::string api_key, unsigned int& num_dims) {
+ if (openai_model_path.empty() || api_key.empty() || openai_model_path.length() < 7) {
return Option<bool>(400, "Invalid OpenAI model path or API key");
}
HttpClient& client = HttpClient::get_instance();
std::unordered_map<std::string, std::string> headers;
std::map<std::string, std::string> res_headers;
- headers["Authorization"] = "Bearer " + openai_api_key;
+ headers["Authorization"] = "Bearer " + api_key;
std::string res;
auto res_code = client.get_response(TextEmbedder::OPENAI_LIST_MODELS, res, res_headers, headers);
if (res_code != 200) {
@@ -243,7 +266,6 @@ Option<bool> TextEmbedder::is_model_valid(const std::string openai_model_path, c
return Option<bool>(400, "OpenAI model not found");
}
-
// This part is hard coded for now. Because OpenAI API does not provide a way to get the output dimensions of the model.
if(model_name.find("-ada-") != std::string::npos) {
if(model_name.substr(model_name.length() - 3) == "002") {
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 0d9ccad7..a610ae45 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -55,7 +55,6 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.set_program_name("./typesense-server");
options.add<std::string>("data-dir", 'd', "Directory where data will be stored.", true);
- options.add<std::string>("model-dir", '\0', "Directory where text embedding models will be stored.", false, "");
options.add<std::string>("api-key", 'a', "API key that allows all operations.", true);
options.add<std::string>("search-only-api-key", 's', "[DEPRECATED: use API key management end-point] API key that allows only searches.", false);
@@ -422,12 +421,7 @@ int run_server(const Config & config, const std::string & version, void (*master
if(!rate_limit_manager_init.ok()) {
LOG(INFO) << "Failed to initialize rate limit manager: " << rate_limit_manager_init.error();
}
-
- if(config.get_model_dir().size() > 0) {
- LOG(INFO) << "Loading text embedding models from " << config.get_model_dir();
- TextEmbedderManager::set_model_dir(config.get_model_dir());
- TextEmbedderManager::download_default_model();
- }
+ TextEmbedderManager::set_model_dir(config.get_data_dir() + "/models");
// first we start the peering service
diff --git a/src/validator.cpp b/src/validator.cpp
index f66eaf9d..cb36571f 100644
--- a/src/validator.cpp
+++ b/src/validator.cpp
@@ -594,9 +594,6 @@ Option<bool> validator_t::validate_embed_fields(const nlohmann::json& document,
const tsl::htrie_map<char, field>& embedding_fields,
const tsl::htrie_map<char, field> & search_schema,
const bool& error_if_field_not_found) {
- if(!embedding_fields.empty() && TextEmbedderManager::model_dir.empty()) {
- return Option<bool>(400, "Text embedding is not enabled. Please set `model-dir` at startup.");
- }
for(const auto& field : embedding_fields) {
for(const auto& field_name : field.embed_from) {
auto schema_field_it = search_schema.find(field_name);
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index 14e6e27b..ba1768ba 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -1610,23 +1610,23 @@ TEST_F(CollectionAllFieldsTest, EmbedFromFieldJSONInvalidField) {
ASSERT_EQ("Property `embed_from` can only refer to string or string array fields.", field_op.error());
}
-TEST_F(CollectionAllFieldsTest, EmbedFromFieldNoModelDir) {
- TextEmbedderManager::model_dir = std::string();
- nlohmann::json field_json;
- field_json["name"] = "embedding";
- field_json["type"] = "float[]";
- field_json["embed_from"] = {"name"};
+// TEST_F(CollectionAllFieldsTest, EmbedFromFieldNoModelDir) {
+// TextEmbedderManager::model_dir = std::string();
+// nlohmann::json field_json;
+// field_json["name"] = "embedding";
+// field_json["type"] = "float[]";
+// field_json["embed_from"] = {"name"};
- std::vector<field> fields;
- std::string fallback_field_type;
- auto arr = nlohmann::json::array();
- arr.push_back(field_json);
+// std::vector<field> fields;
+// std::string fallback_field_type;
+// auto arr = nlohmann::json::array();
+// arr.push_back(field_json);
- auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);
+// auto field_op = field::json_fields_to_fields(false, arr, fallback_field_type, fields);
- ASSERT_FALSE(field_op.ok());
- ASSERT_EQ("Text embedding is not enabled. Please set `model-dir` at startup.", field_op.error());
-}
+// ASSERT_FALSE(field_op.ok());
+// ASSERT_EQ("Text embedding is not enabled. Please set `model-dir` at startup.", field_op.error());
+// }
TEST_F(CollectionAllFieldsTest, EmbedFromNotArray) {
TextEmbedderManager::model_dir = "/tmp/models";
@@ -1667,7 +1667,6 @@ TEST_F(CollectionAllFieldsTest, ModelParametersWithoutEmbedFrom) {
TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {
TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
- TextEmbedderManager::download_default_model();
field embedding = field("embedding", field_types::FLOAT_ARRAY, false);
embedding.embed_from.push_back("name");
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index 88d8e940..42bab54a 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -1456,7 +1456,6 @@ TEST_F(CollectionSchemaChangeTest, UpdateSchemaWithNewEmbeddingField) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -1495,7 +1494,6 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -1544,7 +1542,6 @@ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 218e199a..1346fbc4 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -4622,7 +4622,6 @@ TEST_F(CollectionTest, SemanticSearchTest) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4657,7 +4656,6 @@ TEST_F(CollectionTest, InvalidSemanticSearch) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
LOG(INFO) << "op.error(): " << op.error();
@@ -4687,7 +4685,6 @@ TEST_F(CollectionTest, HybridSearch) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4721,7 +4718,7 @@ TEST_F(CollectionTest, HybridSearch) {
// })"_json;
// TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
-// TextEmbedderManager::download_default_model();
+//
// auto op = collectionManager.create_collection(schema);
// ASSERT_TRUE(op.ok());
@@ -4749,7 +4746,6 @@ TEST_F(CollectionTest, HybridSearchRankFusionTest) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4823,7 +4819,6 @@ TEST_F(CollectionTest, WildcardSearchWithEmbeddingField) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4857,7 +4852,6 @@ TEST_F(CollectionTest, EmbedStringArrayField) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4883,7 +4877,6 @@ TEST_F(CollectionTest, MissingFieldForEmbedding) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4910,7 +4903,6 @@ TEST_F(CollectionTest, WrongTypeForEmbedding) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4934,7 +4926,6 @@ TEST_F(CollectionTest, WrongTypeOfElementForEmbeddingInStringArray) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -4958,7 +4949,6 @@ TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
@@ -5005,21 +4995,21 @@ TEST_F(CollectionTest, DISABLED_CreateOpenAIEmbeddingField) {
]
})"_json;
- if (std::getenv("OPENAI_API_KEY") == nullptr) {
- LOG(INFO) << "Skipping test as OPENAI_API_KEY is not set.";
+ if (std::getenv("api_key") == nullptr) {
+ LOG(INFO) << "Skipping test as api_key is not set.";
return;
}
- auto openai_api_key = std::string(std::getenv("OPENAI_API_KEY"));
- schema["fields"][1]["model_parameters"]["openai_api_key"] = openai_api_key;
+ auto api_key = std::string(std::getenv("api_key"));
+ schema["fields"][1]["model_parameters"]["api_key"] = api_key;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());
auto summary = op.get()->get_summary_json();
ASSERT_EQ("openai/text-embedding-ada-002", summary["fields"][1]["model_parameters"]["model_name"]);
ASSERT_EQ(1536, summary["fields"][1]["num_dim"]);
- // make sure openai_api_key is <hidden>
- ASSERT_EQ("<hidden>", summary["fields"][1]["model_parameters"]["openai_api_key"]);
+ // make sure api_key is <hidden>
+ ASSERT_EQ("<hidden>", summary["fields"][1]["model_parameters"]["api_key"]);
nlohmann::json doc;
doc["name"] = "butter";
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index c78e004e..3df3bde5 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -688,7 +688,6 @@ TEST_F(CollectionVectorTest, HybridSearchWithExplicitVector) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
Collection* coll1 = collectionManager.create_collection(schema).get();
@@ -787,7 +786,6 @@ TEST_F(CollectionVectorTest, EmbeddingFieldVectorIndexTest) {
})"_json;
TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
- TextEmbedderManager::download_default_model();
auto op = collectionManager.create_collection(schema);
ASSERT_TRUE(op.ok());