Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Review changes

ozanarmagan5bcbb2832e14
diff --git a/include/field.h b/include/field.h
index 0de91ceb..a79b6896 100644
--- a/include/field.h
+++ b/include/field.h
@@ -428,11 +428,7 @@ struct field {
size_t num_auto_detect_fields = 0;
for(nlohmann::json & field_json: fields_json) {
-
if(field_json.count(fields::embed_from) != 0) {
- // if(TextEmbedderManager::model_dir.empty()) {
- // return Option<bool>(400, "Text embedding is not enabled. Please set `model-dir` at startup.");
- // }
if(!field_json[fields::embed_from].is_array()) {
return Option<bool>(400, "Property `" + fields::embed_from + "` must be an array.");
diff --git a/include/index.h b/include/index.h
index a80dff23..281091c2 100644
--- a/include/index.h
+++ b/include/index.h
@@ -533,12 +533,7 @@ private:
const std::string& token, uint32_t seq_id);
void initialize_facet_indexes(const field& facet_field);
-
-
-
- static Option<bool> embed_fields(nlohmann::json& document,
- const tsl::htrie_map<char, field>& embedding_fields,
- const tsl::htrie_map<char, field> & search_schema);
+
static Option<bool> batch_embed_fields(std::vector<nlohmann::json*>& documents,
const tsl::htrie_map<char, field>& embedding_fields,
const tsl::htrie_map<char, field> & search_schema);
diff --git a/include/text_embedder.h b/include/text_embedder.h
index 3b585b71..855154a7 100644
--- a/include/text_embedder.h
+++ b/include/text_embedder.h
@@ -10,11 +10,12 @@
class TextEmbedder {
public:
- TextEmbedder(const std::string& model_path, TokenizerType tokenizer_type = TokenizerType::bert);
+ TextEmbedder(const std::string& model_path);
TextEmbedder(const std::string& openai_model_path, const std::string& api_key);
~TextEmbedder();
Option<std::vector<float>> Embed(const std::string& text);
Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs);
+ const std::string& get_vocab_file_name() const;
bool is_openai() {
return !api_key.empty();
@@ -27,6 +28,7 @@ class TextEmbedder {
Ort::Env env_;
encoded_input_t Encode(const std::string& text);
std::unique_ptr<TextEmbeddingTokenizer> tokenizer_;
+ std::string vocab_file_name;
static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);
std::string output_tensor_name;
std::string api_key;
diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.h
index 7789dec2..3a81e6cc 100644
--- a/include/text_embedder_manager.h
+++ b/include/text_embedder_manager.h
@@ -1,4 +1,5 @@
#pragma once
+
#include <memory>
#include <filesystem>
#include <mutex>
@@ -9,244 +10,68 @@
#include "http_client.h"
#include "text_embedder.h"
-
struct text_embedding_model {
- std::string model_url;
+ std::string model_name;
std::string model_md5;
- std::string vocab_url;
+ std::string vocab_file_name;
std::string vocab_md5;
TokenizerType tokenizer_type;
std::string indexing_prefix = "";
std::string query_prefix = "";
-};
+ text_embedding_model(const nlohmann::json& json);
+
+ text_embedding_model() = default;
+};
-// singleton class
+// Singleton class
class TextEmbedderManager {
public:
- static TextEmbedderManager& get_instance() {
- static TextEmbedderManager instance;
- return instance;
- }
-
+ static TextEmbedderManager& get_instance();
+
TextEmbedderManager(TextEmbedderManager&&) = delete;
TextEmbedderManager& operator=(TextEmbedderManager&&) = delete;
TextEmbedderManager(const TextEmbedderManager&) = delete;
TextEmbedderManager& operator=(const TextEmbedderManager&) = delete;
- TextEmbedder* get_text_embedder(const nlohmann::json& model_parameters) {
- std::unique_lock<std::mutex> lock(text_embedders_mutex);
- LOG(INFO) << "Getting text embedder for model: " << model_parameters.dump();
- const std::string& model_name = model_parameters.count("model_name") == 0 ? DEFAULT_MODEL_NAME : model_parameters.at("model_name");
- if(text_embedders[model_name] == nullptr) {
- if(model_parameters.count("api_key") == 0) {
- TokenizerType tokenizer_type;
- if(is_public_model(model_name)) {
- tokenizer_type = public_models[model_name].tokenizer_type;
- // download the model if it doesn't exist
- download_public_model(model_name);
- } else {
- tokenizer_type = get_tokenizer_type(model_parameters);
- }
- LOG(INFO) << "Creating text embedder for model: " << model_name;
- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, tokenizer_type);
- } else {
- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("api_key").get<std::string>());
- }
- }
- return text_embedders[model_name].get();
- }
-
- void delete_text_embedder(const std::string& model_path) {
- std::unique_lock<std::mutex> lock(text_embedders_mutex);
- if (text_embedders.find(model_path) != text_embedders.end()) {
- text_embedders.erase(model_path);
- }
- }
-
- void delete_all_text_embedders() {
- std::unique_lock<std::mutex> lock(text_embedders_mutex);
- text_embedders.clear();
- }
-
- static const TokenizerType get_tokenizer_type(const nlohmann::json& model_parameters) {
- if(model_parameters.count("tokenizer_type") == 0) {
- return TokenizerType::bert;
- } else {
- std::string tokenizer_type = model_parameters.at("model_type").get<std::string>();
- if(tokenizer_type == "distilBert") {
- return TokenizerType::distilBert;
- } else if(tokenizer_type == "xlm-roberta") {
- return TokenizerType::xlm_roberta;
- } else {
- return TokenizerType::bert;
- }
- }
- }
-
- const std::string get_indexing_prefix(const nlohmann::json& model_parameters) {
- std::string val;
- if(is_public_model(model_parameters["model_name"].get<std::string>())) {
- val = public_models[model_parameters["model_name"].get<std::string>()].indexing_prefix;
- } else {
- val = model_parameters.count("indexing_prefix") == 0 ? "" : model_parameters["indexing_prefix"].get<std::string>();
- }
- if(!val.empty()) {
- val += " ";
- }
-
- return val;
- }
-
- const std::string get_query_prefix(const nlohmann::json& model_parameters) {
- std::string val;
- if(is_public_model(model_parameters["model_name"].get<std::string>())) {
- val = public_models[model_parameters["model_name"].get<std::string>()].query_prefix;
- } else {
- val = model_parameters.count("query_prefix") == 0 ? "" : model_parameters["query_prefix"].get<std::string>();
- }
- if(!val.empty()) {
- val += " ";
- }
-
- return val;
- }
-
- static void set_model_dir(const std::string& dir) {
- // create the directory if it doesn't exist
- if(!std::filesystem::exists(dir)) {
- std::filesystem::create_directories(dir);
- }
- model_dir = dir;
- }
-
- static const std::string& get_model_dir() {
- return model_dir;
- }
-
- ~TextEmbedderManager() {
- delete_all_text_embedders();
- }
-
- static constexpr char* DEFAULT_MODEL_NAME = "ts-e5-small";
- static constexpr char* DEFAULT_MODEL_INDEXING_PREFIX = "passage:";
- static constexpr char* DEFAULT_MODEL_QUERY_PREFIX = "query:";
+ TextEmbedder* get_text_embedder(const nlohmann::json& model_parameters);
+ void delete_text_embedder(const std::string& model_path);
+ void delete_all_text_embedders();
+
+ static const TokenizerType get_tokenizer_type(const nlohmann::json& model_parameters);
+ const std::string get_indexing_prefix(const nlohmann::json& model_parameters);
+ const std::string get_query_prefix(const nlohmann::json& model_parameters);
+ static void set_model_dir(const std::string& dir);
+ static const std::string& get_model_dir();
+
+ ~TextEmbedderManager();
+
+ inline static const std::string MODELS_REPO_URL = "https://models.typesense.org/public/";
+ inline static const std::string MODEL_CONFIG_FILE = "config.json";
inline static std::string model_dir = "";
- inline static const std::string get_absolute_model_path(const std::string& model_name) {
- return get_model_subdir(model_name) + "/model.onnx";
- }
- inline static const std::string get_absolute_vocab_path(const std::string& model_name) {
- return get_model_subdir(model_name) + "/vocab.txt";
- }
- inline static const std::string get_absolute_sentencepiece_model_path(const std::string& model_name) {
- auto model_subdir = get_model_subdir(model_name);
- // look for any file that ends with .model
- for (const auto & entry : std::filesystem::directory_iterator(model_subdir)) {
- if(entry.path().extension() == ".model") {
- return entry.path().string();
- }
- }
-
- return model_subdir + "/sentencepiece.model";
- }
- inline static const bool check_md5(const std::string& file_path, const std::string& target_md5) {
- std::ifstream stream(file_path);
- if (stream.fail()) {
- return false;
- }
- unsigned char md5[MD5_DIGEST_LENGTH];
- std::stringstream ss,res;
- ss << stream.rdbuf();
- MD5((unsigned char*)ss.str().c_str(), ss.str().length(), md5);
- for (int i = 0; i < MD5_DIGEST_LENGTH; i++) {
- res << std::hex << (int)md5[i];
- }
- return res.str() == target_md5;
- }
- void download_public_model(const std::string& model_name) {
- HttpClient& httpClient = HttpClient::get_instance();
- auto model = public_models[model_name];
- if(!check_md5(get_absolute_model_path(model_name), model.model_md5)) {
- LOG(INFO) << "Downloading public model: " << model_name;
- long res = httpClient.download_file(model.model_url, get_absolute_model_path(model_name));
- if(res != 200) {
- LOG(INFO) << "Failed to download public model " << model_name << ": " << res;
- }
- }
-
- if(model.tokenizer_type == TokenizerType::bert || model.tokenizer_type == TokenizerType::distilBert) {
- if(!check_md5(get_absolute_vocab_path(model_name), model.vocab_md5)) {
- LOG(INFO) << "Downloading default vocab for model: " << model_name;
- long res = httpClient.download_file(model.vocab_url, get_absolute_vocab_path(model_name));
- if(res != 200) {
- LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;
- }
- }
- } else {
- LOG(INFO) << "Downloading default sentencepiece model for model: " << model_name;
- if(!check_md5(get_absolute_sentencepiece_model_path(model_name), model.vocab_md5)) {
- LOG(INFO) << "Downloading default sentencepiece model for model: " << model_name;
- long res = httpClient.download_file(model.vocab_url, get_absolute_sentencepiece_model_path(model_name));
- if(res != 200) {
- LOG(INFO) << "Failed to download default sentencepiece model " << model_name << ": " << res;
- }
- }
- }
- }
-
- const bool is_public_model(const std::string& model_name) {
- return public_models.find(model_name) != public_models.end();
- }
+
+ static const std::string get_absolute_model_path(const std::string& model_name);
+ static const std::string get_absolute_vocab_path(const std::string& model_name, const std::string& vocab_file_name);
+ static const std::string get_absolute_config_path(const std::string& model_name);
+ static const std::string get_model_url(const text_embedding_model& model);
+ static const std::string get_vocab_url(const text_embedding_model& model);
+ static Option<nlohmann::json> get_public_model_config(const std::string& model_name);
+ static const std::string get_model_name_without_namespace(const std::string& model_name);
+ static const std::string get_model_subdir(const std::string& model_name);
+ static const bool check_md5(const std::string& file_path, const std::string& target_md5);
+ void download_public_model(const std::string& model_name);
+
+ const bool is_public_model(const std::string& model_name);
+
private:
- TextEmbedderManager() {
- public_models[DEFAULT_MODEL_NAME] = text_embedding_model{
- "https://models.typesense.org/public/e5-small/model.onnx",
- "3d421dc72859a72368c106415cdebf2",
- "https://models.typesense.org/public/e5-small/vocab.txt",
- "6480d5d8528ce344256daf115d4965e",
- TokenizerType::bert,
- "passage:",
- "query:"};
- public_models["ts-all-MiniLM-L12-v2"] = text_embedding_model{
- "https://models.typesense.org/public/all-miniLM-L12-v2/model.onnx",
- "6d196b8f7a8d8abcfb08afcac1704302",
- "https://models.typesense.org/public/all-miniLM-L12-v2/vocab.txt",
- "9d2131a9a433502abeb512978d452ecc",
- TokenizerType::bert};
- public_models["ts-distiluse-base-multilingual-cased-v2"] = text_embedding_model{
- "https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/model.onnx",
- "91f535cd7ca87359b9bfd1d87f9e452e",
- "https://models.typesense.org/public/distiluse-base-multilingual-cased-v2/vocab.txt",
- "0f05e1b7420dea2db8ade63eb5b80f7a",
- TokenizerType::distilBert};
- public_models["ts-paraphrase-multilingual-mpnet-base-v2"] = text_embedding_model{
- "https://models.typesense.org/public/paraphrase-multilingual-mpnet-base-v2/model.onnx",
- "728d3db98e1b7a691a731644867382c5",
- "https://models.typesense.org/public/paraphrase-multilingual-mpnet-base-v2/sentencepiece.bpe.model",
- "bf25eb5120ad92ef5c7d8596b5dc4046",
- TokenizerType::xlm_roberta};
- }
+ TextEmbedderManager() = default;
+
std::unordered_map<std::string, std::shared_ptr<TextEmbedder>> text_embedders;
std::unordered_map<std::string, text_embedding_model> public_models;
std::mutex text_embedders_mutex;
- static const std::string get_model_subdir(const std::string& model_name) {
- if(model_dir.back() != '/') {
- // create subdir <model_name> if it doesn't exist
- if(!std::filesystem::exists(model_dir + "/" + model_name)) {
- std::filesystem::create_directories(model_dir + "/" + model_name);
- }
- return model_dir + "/" + model_name;
- } else {
- // create subdir <model_name> if it doesn't exist
- if(!std::filesystem::exists(model_dir + model_name)) {
- std::filesystem::create_directories(model_dir + model_name);
- }
- return model_dir + model_name;
- }
- }
-};
-
-
+
+ Option<std::string> get_namespace(const std::string& model_name);
+};
diff --git a/include/text_embedder_tokenizer.h b/include/text_embedder_tokenizer.h
index d0d66098..bd41717a 100644
--- a/include/text_embedder_tokenizer.h
+++ b/include/text_embedder_tokenizer.h
@@ -8,7 +8,7 @@
enum class TokenizerType {
bert,
- distilBert,
+ distilbert,
xlm_roberta
};
@@ -35,9 +35,9 @@ class BertTokenizerWrapper : public TextEmbeddingTokenizer {
encoded_input_t Encode(const std::string& text) override;
};
-class DistilBertTokenizer : public BertTokenizerWrapper {
+class DistilbertTokenizer : public BertTokenizerWrapper {
public:
- DistilBertTokenizer(const std::string& vocab_path);
+ DistilbertTokenizer(const std::string& vocab_path);
encoded_input_t Encode(const std::string& text) override;
};
diff --git a/src/field.cpp b/src/field.cpp
index 92909b9a..f9c9c363 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -679,43 +679,49 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
if(field_json.count(fields::embed_from) != 0) {
// If the model path is not specified, use the default model and set the number of dimensions to 384 (number of dimensions of the default model)
field_json[fields::num_dim] = static_cast<unsigned int>(384);
- if(field_json.count(fields::model_parameters) != 0) {
- auto model_parameters = field_json[fields::model_parameters];
- if(model_parameters.count(fields::model_name) != 0) {
- unsigned int num_dim = 0;
- if(!model_parameters[fields::model_name].is_string()) {
- return Option<bool>(400, "Property `" + fields::model_name + "` must be a string.");
- }
- if(model_parameters[fields::model_name].get<std::string>().empty()) {
- return Option<bool>(400, "Property `" + fields::model_name + "` must be a non-empty string.");
- }
- if(model_parameters.count(fields::indexing_prefix) != 0) {
- if(!model_parameters[fields::indexing_prefix].is_string()) {
- return Option<bool>(400, "Property `" + fields::indexing_prefix + "` must be a string.");
- }
- }
+ if(field_json.count(fields::model_parameters) == 0) {
+ return Option<bool>(400, "Property `" + fields::model_parameters + "` must be specified with `" + fields::embed_from + "`.");
+ }
- if(model_parameters.count(fields::query_prefix) != 0) {
- if(!model_parameters[fields::query_prefix].is_string()) {
- return Option<bool>(400, "Property `" + fields::query_prefix + "` must be a string.");
- }
- }
+ auto& model_parameters = field_json[fields::model_parameters];
+
+ if(model_parameters.count(fields::model_name) == 0) {
+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::model_name + "` must be specified with `" + fields::embed_from + "`.");
+ }
- if(model_parameters.count("api_key") != 0) {
- auto res = TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), model_parameters[fields::api_key].get<std::string>(), num_dim);
- if(res.ok()) {
- field_json[fields::num_dim] = num_dim;
- } else {
- return Option<bool>(res.code(), res.error());
- }
- } else {
- if(TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), num_dim)) {
- field_json[fields::num_dim] = num_dim;
- } else {
- return Option<bool>(400, "Property `" + fields::model_name + "` must be a valid model path.");
- }
- }
+ unsigned int num_dim = 0;
+ if(!model_parameters[fields::model_name].is_string()) {
+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::model_name + "` must be a string.");
+ }
+ if(model_parameters[fields::model_name].get<std::string>().empty()) {
+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::model_name + "` cannot be empty.");
+ }
+
+ if(model_parameters.count(fields::indexing_prefix) != 0) {
+ if(!model_parameters[fields::indexing_prefix].is_string()) {
+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::indexing_prefix + "` must be a string.");
+ }
+ }
+
+ if(model_parameters.count(fields::query_prefix) != 0) {
+ if(!model_parameters[fields::query_prefix].is_string()) {
+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::query_prefix + "` must be a string.");
+ }
+ }
+
+ if(model_parameters.count("api_key") != 0) {
+ auto res = TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), model_parameters[fields::api_key].get<std::string>(), num_dim);
+ if(res.ok()) {
+ field_json[fields::num_dim] = num_dim;
+ } else {
+ return Option<bool>(res.code(), res.error());
+ }
+ } else {
+ if(TextEmbedder::is_model_valid(model_parameters[fields::model_name].get<std::string>(), num_dim)) {
+ field_json[fields::num_dim] = num_dim;
+ } else {
+ return Option<bool>(400, "Property `" + fields::model_parameters + "." + fields::model_name + "` is invalid.");
}
}
} else {
diff --git a/src/http_client.cpp b/src/http_client.cpp
index 652f0971..0829400f 100644
--- a/src/http_client.cpp
+++ b/src/http_client.cpp
@@ -101,6 +101,9 @@ long HttpClient::get_response(const std::string &url, std::string &response,
chunk = curl_slist_append(chunk, header_str.c_str());
}
+ // follow redirects
+ curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
+
return perform_curl(curl, res_headers, chunk);
}
diff --git a/src/index.cpp b/src/index.cpp
index e54312ad..c18faadf 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -455,10 +455,8 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
index_rec.del_doc);
scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);
docs_to_embed.push_back(&index_rec.new_doc);
- //embed_fields(index_rec.new_doc, embedding_fields, search_schema);
} else {
docs_to_embed.push_back(&index_rec.doc);
- //embed_fields(index_rec.doc, embedding_fields, search_schema);
}
compute_token_offsets_facets(index_rec, search_schema, token_separators, symbols_to_index);
@@ -6344,33 +6342,6 @@ bool Index::common_results_exist(std::vector<art_leaf*>& leaves, bool must_match
return phrase_exists;
}
-Option<bool> Index::embed_fields(nlohmann::json& document,
- const tsl::htrie_map<char, field>& embedding_fields,
- const tsl::htrie_map<char, field> & search_schema) {
- for(const auto& field : embedding_fields) {
- std::string text_to_embed = TextEmbedderManager::get_instance().get_indexing_prefix(field.model_parameters);
- for(const auto& field_name : field.embed_from) {
- auto field_it = search_schema.find(field_name);
- if(field_it.value().type == field_types::STRING) {
- text_to_embed += document[field_name].get<std::string>() + " ";
- } else if(field_it.value().type == field_types::STRING_ARRAY) {
- for(const auto& val : document[field_name]) {
- text_to_embed += val.get<std::string>() + " ";
- }
- }
- }
- TextEmbedderManager& embedder_manager = TextEmbedderManager::get_instance();
- auto embedder = embedder_manager.get_text_embedder(field.model_parameters);
- auto embedding_op = embedder->Embed(text_to_embed);
-
- if(!embedding_op.ok()) {
- return Option<bool>(400, embedding_op.error());
- }
-
- document[field.name] = embedding_op.get();
- }
- return Option<bool>(true);
-}
Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,
const tsl::htrie_map<char, field>& embedding_fields,
diff --git a/src/text_embedder.cpp b/src/text_embedder.cpp
index 871a679a..c884fdf5 100644
--- a/src/text_embedder.cpp
+++ b/src/text_embedder.cpp
@@ -6,18 +6,24 @@
#include <sstream>
#include <filesystem>
-TextEmbedder::TextEmbedder(const std::string& model_name, TokenizerType tokenizer_type) {
+TextEmbedder::TextEmbedder(const std::string& model_name) {
// create environment
Ort::SessionOptions session_options;
std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_name);
LOG(INFO) << "Loading model from: " << abs_path;
session_ = std::make_unique<Ort::Session>(env_, abs_path.c_str(), session_options);
+ std::ifstream config_file(TextEmbedderManager::get_absolute_config_path(model_name));
+ nlohmann::json config;
+ config_file >> config;
+ TokenizerType tokenizer_type = TextEmbedderManager::get_tokenizer_type(config);
+ auto vocab_path = TextEmbedderManager::get_absolute_vocab_path(model_name, config["vocab_file_name"].get<std::string>());
if(tokenizer_type == TokenizerType::bert) {
- auto vocab_path = TextEmbedderManager::get_absolute_vocab_path(model_name);
tokenizer_ = std::make_unique<BertTokenizerWrapper>(vocab_path);
- } else {
- auto spiece_model_path = TextEmbedderManager::get_absolute_sentencepiece_model_path(model_name);
- tokenizer_ = std::make_unique<XLMRobertaTokenizer>(spiece_model_path);
+ } else if(tokenizer_type == TokenizerType::distilbert) {
+ tokenizer_ = std::make_unique<DistilbertTokenizer>(vocab_path);
+ }
+ else if(tokenizer_type == TokenizerType::xlm_roberta) {
+ tokenizer_ = std::make_unique<XLMRobertaTokenizer>(vocab_path);
}
auto output_tensor_count = session_->GetOutputCount();
for (size_t i = 0; i < output_tensor_count; i++) {
@@ -67,6 +73,7 @@ Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {
}
return Option<std::vector<float>>(nlohmann::json::parse(res)["data"][0]["embedding"].get<std::vector<float>>());
} else {
+ LOG(INFO) << "Embedding text: " << text;
auto encoded_input = tokenizer_->Encode(text);
// create input tensor object from data values
Ort::AllocatorWithDefaultOptions allocator;
@@ -159,13 +166,32 @@ bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& n
Ort::SessionOptions session_options;
Ort::Env env;
- std::string abs_path = TextEmbedderManager::get_absolute_model_path(model_name);
+ std::string abs_path = TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::get_model_name_without_namespace(model_name));
if(!std::filesystem::exists(abs_path)) {
LOG(ERROR) << "Model file not found: " << abs_path;
return false;
}
+ if(!TextEmbedderManager::get_instance().is_public_model(model_name)) {
+ if(!std::filesystem::exists(TextEmbedderManager::get_absolute_config_path(model_name))) {
+ LOG(ERROR) << "Config file not found: " << TextEmbedderManager::get_absolute_config_path(model_name);
+ return false;
+ }
+ std::ifstream config_file(TextEmbedderManager::get_absolute_config_path(model_name));
+ nlohmann::json config;
+ config_file >> config;
+ if(config["model_type"].is_null() || config["vocab_file_name"].is_null()) {
+ LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);
+ return false;
+ }
+
+ if(!std::filesystem::exists(TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>())) {
+ LOG(ERROR) << "Vocab file not found: " << TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>();
+ return false;
+ }
+ }
+
Ort::Session session(env, abs_path.c_str(), session_options);
if(session.GetInputCount() != 3 && session.GetInputCount() != 2) {
LOG(ERROR) << "Invalid model: input count is not 3 or 2";
diff --git a/src/text_embedder_manager.cpp b/src/text_embedder_manager.cpp
new file mode 100644
index 00000000..01cadc09
--- /dev/null
+++ b/src/text_embedder_manager.cpp
@@ -0,0 +1,251 @@
+#include "text_embedder_manager.h"
+
+
+TextEmbedderManager& TextEmbedderManager::get_instance() {
+ static TextEmbedderManager instance;
+ return instance;
+}
+
+TextEmbedder* TextEmbedderManager::get_text_embedder(const nlohmann::json& model_parameters) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
+ const std::string& model_name = model_parameters.at("model_name");
+ if(text_embedders[model_name] == nullptr) {
+ if(model_parameters.count("api_key") == 0) {
+ if(is_public_model(model_name)) {
+ // download the model if it doesn't exist
+ download_public_model(model_name);
+ }
+ text_embedders[model_name] = std::make_shared<TextEmbedder>(get_model_name_without_namespace(model_name));
+ } else {
+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_parameters.at("api_key").get<std::string>());
+ }
+ }
+ return text_embedders[model_name].get();
+}
+
+void TextEmbedderManager::delete_text_embedder(const std::string& model_path) {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
+ if (text_embedders.find(model_path) != text_embedders.end()) {
+ text_embedders.erase(model_path);
+ }
+}
+
+void TextEmbedderManager::delete_all_text_embedders() {
+ std::unique_lock<std::mutex> lock(text_embedders_mutex);
+ text_embedders.clear();
+}
+
+const TokenizerType TextEmbedderManager::get_tokenizer_type(const nlohmann::json& model_parameters) {
+ if(model_parameters.find("model_type") == model_parameters.end()) {
+ return TokenizerType::bert;
+ } else {
+ std::string tokenizer_type = model_parameters.at("model_type").get<std::string>();
+ if(tokenizer_type == "distilbert") {
+ return TokenizerType::distilbert;
+ } else if(tokenizer_type == "xlm_roberta") {
+ return TokenizerType::xlm_roberta;
+ } else {
+ return TokenizerType::bert;
+ }
+ }
+}
+
+const std::string TextEmbedderManager::get_indexing_prefix(const nlohmann::json& model_parameters) {
+ std::string val;
+ if(is_public_model(model_parameters["model_name"].get<std::string>())) {
+ val = public_models[model_parameters["model_name"].get<std::string>()].indexing_prefix;
+ } else {
+ val = model_parameters.count("indexing_prefix") == 0 ? "" : model_parameters["indexing_prefix"].get<std::string>();
+ }
+ if(!val.empty()) {
+ val += " ";
+ }
+
+ return val;
+}
+
+const std::string TextEmbedderManager::get_query_prefix(const nlohmann::json& model_parameters) {
+ std::string val;
+ if(is_public_model(model_parameters["model_name"].get<std::string>())) {
+ val = public_models[model_parameters["model_name"].get<std::string>()].query_prefix;
+ } else {
+ val = model_parameters.count("query_prefix") == 0 ? "" : model_parameters["query_prefix"].get<std::string>();
+ }
+ if(!val.empty()) {
+ val += " ";
+ }
+
+ return val;
+}
+
+void TextEmbedderManager::set_model_dir(const std::string& dir) {
+ // create the directory if it doesn't exist
+ if(!std::filesystem::exists(dir)) {
+ std::filesystem::create_directories(dir);
+ }
+ model_dir = dir;
+}
+
+const std::string& TextEmbedderManager::get_model_dir() {
+ return model_dir;
+}
+
+TextEmbedderManager::~TextEmbedderManager() {
+ delete_all_text_embedders();
+}
+
+const std::string TextEmbedderManager::get_absolute_model_path(const std::string& model_name) {
+ return get_model_subdir(model_name) + "/model.onnx";
+}
+const std::string TextEmbedderManager::get_absolute_vocab_path(const std::string& model_name, const std::string& vocab_file_name) {
+ return get_model_subdir(model_name) + "/" + vocab_file_name;
+}
+
+const std::string TextEmbedderManager::get_absolute_config_path(const std::string& model_name) {
+ return get_model_subdir(model_name) + "/config.json";
+}
+
+const bool TextEmbedderManager::check_md5(const std::string& file_path, const std::string& target_md5) {
+ std::ifstream stream(file_path);
+ if (stream.fail()) {
+ return false;
+ }
+ unsigned char md5[MD5_DIGEST_LENGTH];
+ std::stringstream ss,res;
+ ss << stream.rdbuf();
+ MD5((unsigned char*)ss.str().c_str(), ss.str().length(), md5);
+ for (int i = 0; i < MD5_DIGEST_LENGTH; i++) {
+ res << std::hex << (int)md5[i];
+ }
+ return res.str() == target_md5;
+}
+void TextEmbedderManager::download_public_model(const std::string& model_name) {
+ HttpClient& httpClient = HttpClient::get_instance();
+ auto model = public_models[model_name];
+ auto actual_model_name = get_model_name_without_namespace(model_name);
+ if(!check_md5(get_absolute_model_path(actual_model_name), model.model_md5)) {
+ long res = httpClient.download_file(get_model_url(model), get_absolute_model_path(actual_model_name));
+ if(res != 200) {
+ LOG(INFO) << "Failed to download public model " << model_name << ": " << res;
+ }
+ }
+
+ if(!check_md5(get_absolute_vocab_path(actual_model_name, model.vocab_file_name), model.vocab_md5)) {
+ long res = httpClient.download_file(get_vocab_url(model), get_absolute_vocab_path(actual_model_name, model.vocab_file_name));
+ if(res != 200) {
+ LOG(INFO) << "Failed to download default vocab " << model_name << ": " << res;
+ }
+ }
+
+}
+
+const bool TextEmbedderManager::is_public_model(const std::string& model_name) {
+ if(public_models.find(model_name) != public_models.end()) {
+ return true;
+ }
+
+ auto model_namespace = get_namespace(model_name);
+ if(!model_namespace.ok() || model_namespace.get() != "ts") {
+ return false;
+ }
+
+ auto actual_model_name = get_model_name_without_namespace(model_name);
+ auto model_config_op = get_public_model_config(actual_model_name);
+
+ if(model_config_op.ok()) {
+ auto config = model_config_op.get();
+ config["model_name"] = actual_model_name;
+ public_models[model_name] = text_embedding_model(config);
+ return true;
+ }
+
+ return false;
+}
+
+const std::string TextEmbedderManager::get_model_subdir(const std::string& model_name) {
+ if(model_dir.back() != '/') {
+ // create subdir <model_name> if it doesn't exist
+ if(!std::filesystem::exists(model_dir + "/" + model_name)) {
+ std::filesystem::create_directories(model_dir + "/" + model_name);
+ }
+ return model_dir + "/" + model_name;
+ } else {
+ // create subdir <model_name> if it doesn't exist
+ if(!std::filesystem::exists(model_dir + model_name)) {
+ std::filesystem::create_directories(model_dir + model_name);
+ }
+ return model_dir + model_name;
+ }
+}
+
+Option<std::string> TextEmbedderManager::get_namespace(const std::string& model_name) {
+ // <namespace>/<model_name> if / is present in model_name
+ if(model_name.find("/") != std::string::npos) {
+ return Option<std::string>(model_name.substr(0, model_name.find("/")));
+ } else {
+ return Option<std::string>(404, "Namespace not found");
+ }
+}
+
+const std::string TextEmbedderManager::get_model_name_without_namespace(const std::string& model_name) {
+ // <namespace>/<model_name> if / is present in model_name
+ if(model_name.find("/") != std::string::npos) {
+ return model_name.substr(model_name.find("/") + 1);
+ } else {
+ return model_name;
+ }
+}
+
+text_embedding_model::text_embedding_model(const nlohmann::json& json) {
+ model_name = json.at("model_name").get<std::string>();
+ model_md5 = json.at("model_md5").get<std::string>();
+ vocab_file_name = json.at("vocab_file_name").get<std::string>();
+ vocab_md5 = json.at("vocab_md5").get<std::string>();
+ tokenizer_type = TextEmbedderManager::get_tokenizer_type(json);
+ if(json.count("indexing_prefix") != 0) {
+ indexing_prefix = json.at("indexing_prefix").get<std::string>();
+ }
+ if(json.count("query_prefix") != 0) {
+ query_prefix = json.at("query_prefix").get<std::string>();
+ }
+}
+
+
+Option<nlohmann::json> TextEmbedderManager::get_public_model_config(const std::string& model_name) {
+ // check cache first
+ if(std::filesystem::exists(get_absolute_config_path(model_name))) {
+ std::ifstream config_file(get_absolute_config_path(model_name));
+ nlohmann::json config;
+ config_file >> config;
+ config_file.close();
+ return Option<nlohmann::json>(config);
+ }
+
+ auto actual_model_name = get_model_name_without_namespace(model_name);
+ HttpClient& httpClient = HttpClient::get_instance();
+ std::unordered_map<std::string, std::string> headers;
+ headers["Accept"] = "application/json";
+ std::map<std::string, std::string> response_headers;
+ std::string response_body;
+ long res = httpClient.get_response(MODELS_REPO_URL + actual_model_name + "/" + MODEL_CONFIG_FILE, response_body, response_headers, headers);
+
+ if(res == 200 || res == 302) {
+ // cache the config file
+ auto config_file_path = get_absolute_config_path(actual_model_name);
+ std::ofstream config_file(config_file_path);
+ config_file << response_body;
+ config_file.close();
+
+ return Option<nlohmann::json>(nlohmann::json::parse(response_body));
+ }
+
+ return Option<nlohmann::json>(404, "Model not found");
+}
+
+const std::string TextEmbedderManager::get_model_url(const text_embedding_model& model) {
+ return MODELS_REPO_URL + model.model_name + "/model.onnx";
+}
+
+const std::string TextEmbedderManager::get_vocab_url(const text_embedding_model& model) {
+ return MODELS_REPO_URL + model.model_name + "/" + model.vocab_file_name;
+}
\ No newline at end of file
diff --git a/src/text_embedder_tokenizer.cpp b/src/text_embedder_tokenizer.cpp
index f3dee97c..100f8e0a 100644
--- a/src/text_embedder_tokenizer.cpp
+++ b/src/text_embedder_tokenizer.cpp
@@ -28,10 +28,10 @@ encoded_input_t BertTokenizerWrapper::Encode(const std::string& text) {
-DistilBertTokenizer::DistilBertTokenizer(const std::string& vocab_path) : BertTokenizerWrapper(vocab_path) {}
+DistilbertTokenizer::DistilbertTokenizer(const std::string& vocab_path) : BertTokenizerWrapper(vocab_path) {}
-encoded_input_t DistilBertTokenizer::Encode(const std::string& text) {
+encoded_input_t DistilbertTokenizer::Encode(const std::string& text) {
auto encoded = bert_tokenizer_->Encode(bert_tokenizer_->Tokenize(ustring(text)));
auto input_ids = bert_tokenizer_->AddSpecialToken(encoded);
auto attention_mask = std::vector<int64_t>(input_ids.size(), 1);
diff --git a/test/collection_all_fields_test.cpp b/test/collection_all_fields_test.cpp
index ba1768ba..488dce4c 100644
--- a/test/collection_all_fields_test.cpp
+++ b/test/collection_all_fields_test.cpp
@@ -1593,11 +1593,13 @@ TEST_F(CollectionAllFieldsTest, FieldNameMatchingRegexpShouldNotBeIndexedInNonAu
}
TEST_F(CollectionAllFieldsTest, EmbedFromFieldJSONInvalidField) {
- TextEmbedderManager::model_dir = "/tmp/models";
+ TextEmbedderManager::set_model_dir("/tmp/typensense_test/models");
nlohmann::json field_json;
field_json["name"] = "embedding";
field_json["type"] = "float[]";
field_json["embed_from"] = {"name"};
+ field_json["model_parameters"] = nlohmann::json::object();
+ field_json["model_parameters"]["model_name"] = "ts/e5-small";
std::vector<field> fields;
std::string fallback_field_type;
@@ -1629,11 +1631,13 @@ TEST_F(CollectionAllFieldsTest, EmbedFromFieldJSONInvalidField) {
// }
TEST_F(CollectionAllFieldsTest, EmbedFromNotArray) {
- TextEmbedderManager::model_dir = "/tmp/models";
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
nlohmann::json field_json;
field_json["name"] = "embedding";
field_json["type"] = "float[]";
field_json["embed_from"] = "name";
+ field_json["model_parameters"] = nlohmann::json::object();
+ field_json["model_parameters"]["model_name"] = "ts/e5-small";
std::vector<field> fields;
std::string fallback_field_type;
@@ -1647,11 +1651,12 @@ TEST_F(CollectionAllFieldsTest, EmbedFromNotArray) {
}
TEST_F(CollectionAllFieldsTest, ModelParametersWithoutEmbedFrom) {
- TextEmbedderManager::model_dir = "/tmp/models";
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
nlohmann::json field_json;
field_json["name"] = "embedding";
field_json["type"] = "float[]";
- field_json["model_parameters"] = {{"model_name", "bert-base-uncased"}};
+ field_json["model_parameters"] = nlohmann::json::object();
+ field_json["model_parameters"]["model_name"] = "ts/e5-small";
std::vector<field> fields;
std::string fallback_field_type;
@@ -1666,10 +1671,11 @@ TEST_F(CollectionAllFieldsTest, ModelParametersWithoutEmbedFrom) {
TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {
- TextEmbedderManager::model_dir = "/tmp/typesense_test/models";
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
field embedding = field("embedding", field_types::FLOAT_ARRAY, false);
embedding.embed_from.push_back("name");
+ embedding.model_parameters["model_name"] = "ts/e5-small";
std::vector<field> fields = {field("name", field_types::STRING, false),
embedding};
auto obj_coll_op = collectionManager.create_collection("obj_coll", 1, fields, "", 0, field_types::AUTO);
@@ -1690,11 +1696,13 @@ TEST_F(CollectionAllFieldsTest, EmbedFromBasicValid) {
}
TEST_F(CollectionAllFieldsTest, WrongDataTypeForEmbedFrom) {
- TextEmbedderManager::model_dir = "/tmp/models";
+ TextEmbedderManager::set_model_dir("/tmp/typesense_test/models");
nlohmann::json field_json;
field_json["name"] = "embedding";
field_json["type"] = "float[]";
field_json["embed_from"] = {"age"};
+ field_json["model_parameters"] = nlohmann::json::object();
+ field_json["model_parameters"]["model_name"] = "ts/e5-small";
std::vector<field> fields;
std::string fallback_field_type;
diff --git a/test/collection_schema_change_test.cpp b/test/collection_schema_change_test.cpp
index 42bab54a..ebccb569 100644
--- a/test/collection_schema_change_test.cpp
+++ b/test/collection_schema_change_test.cpp
@@ -1464,7 +1464,7 @@ TEST_F(CollectionSchemaChangeTest, UpdateSchemaWithNewEmbeddingField) {
nlohmann::json update_schema = R"({
"fields": [
- {"name": "embedding", "type":"float[]", "embed_from": ["names"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["names"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -1489,7 +1489,7 @@ TEST_F(CollectionSchemaChangeTest, DropFieldUsedForEmbedding) {
"fields": [
{"name": "names", "type": "string[]"},
{"name": "category", "type":"string"},
- {"name": "embedding", "type":"float[]", "embed_from": ["names","category"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["names","category"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -1537,7 +1537,7 @@ TEST_F(CollectionSchemaChangeTest, EmbeddingFieldsMapTest) {
"name": "objects",
"fields": [
{"name": "name", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 1346fbc4..c7d7d25b 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -4617,7 +4617,7 @@ TEST_F(CollectionTest, SemanticSearchTest) {
"name": "objects",
"fields": [
{"name": "name", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -4651,7 +4651,7 @@ TEST_F(CollectionTest, InvalidSemanticSearch) {
"name": "objects",
"fields": [
{"name": "name", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -4680,7 +4680,7 @@ TEST_F(CollectionTest, HybridSearch) {
"name": "objects",
"fields": [
{"name": "name", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -4741,7 +4741,7 @@ TEST_F(CollectionTest, HybridSearchRankFusionTest) {
"name": "objects",
"fields": [
{"name": "name", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -4814,7 +4814,7 @@ TEST_F(CollectionTest, WildcardSearchWithEmbeddingField) {
"name": "objects",
"fields": [
{"name": "name", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -4847,7 +4847,7 @@ TEST_F(CollectionTest, EmbedStringArrayField) {
"name": "objects",
"fields": [
{"name": "names", "type": "string[]"},
- {"name": "embedding", "type":"float[]", "embed_from": ["names"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["names"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -4872,7 +4872,7 @@ TEST_F(CollectionTest, MissingFieldForEmbedding) {
"fields": [
{"name": "names", "type": "string[]"},
{"name": "category", "type": "string", "optional": true},
- {"name": "embedding", "type":"float[]", "embed_from": ["names", "category"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["names", "category"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -4898,7 +4898,7 @@ TEST_F(CollectionTest, WrongTypeForEmbedding) {
"name": "objects",
"fields": [
{"name": "category", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed_from": ["category"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["category"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -4921,7 +4921,7 @@ TEST_F(CollectionTest, WrongTypeOfElementForEmbeddingInStringArray) {
"name": "objects",
"fields": [
{"name": "category", "type": "string[]"},
- {"name": "embedding", "type":"float[]", "embed_from": ["category"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["category"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -4944,7 +4944,7 @@ TEST_F(CollectionTest, UpdateEmbeddingsForUpdatedDocument) {
"name": "objects",
"fields": [
{"name": "name", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
diff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cpp
index 3df3bde5..2334fd0a 100644
--- a/test/collection_vector_search_test.cpp
+++ b/test/collection_vector_search_test.cpp
@@ -683,7 +683,7 @@ TEST_F(CollectionVectorTest, HybridSearchWithExplicitVector) {
"name": "coll1",
"fields": [
{"name": "name", "type": "string"},
- {"name": "vec", "type": "float[]", "embed_from": ["name"]}
+ {"name": "vec", "type": "float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;
@@ -781,7 +781,7 @@ TEST_F(CollectionVectorTest, EmbeddingFieldVectorIndexTest) {
"name": "objects",
"fields": [
{"name": "name", "type": "string"},
- {"name": "embedding", "type":"float[]", "embed_from": ["name"]}
+ {"name": "embedding", "type":"float[]", "embed_from": ["name"], "model_parameters": {"model_name": "ts/e5-small"}}
]
})"_json;