Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add support for `openai/text-embedding-3-*` (#1558)
Ozan Armağan5722a9ddddcc
diff --git a/include/text_embedder.h b/include/text_embedder.hindex 1cf79fa9..f95c4101 100644--- a/include/text_embedder.h+++ b/include/text_embedder.h@@ -14,7 +14,7 @@ class TextEmbedder {// Constructor for local or public modelsTextEmbedder(const std::string& model_path);// Constructor for remote models- TextEmbedder(const nlohmann::json& model_config, size_t num_dims);+ TextEmbedder(const nlohmann::json& model_config, size_t num_dims, const bool has_custom_dims = false);~TextEmbedder();embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_tries = 2);std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200,diff --git a/include/text_embedder_remote.h b/include/text_embedder_remote.hindex 4aa75d82..5ba12690 100644--- a/include/text_embedder_remote.h+++ b/include/text_embedder_remote.h@@ -49,8 +49,10 @@ class OpenAIEmbedder : public RemoteEmbedder {std::string openai_model_path;static constexpr char* OPENAI_LIST_MODELS = "https://api.openai.com/v1/models";static constexpr char* OPENAI_CREATE_EMBEDDING = "https://api.openai.com/v1/embeddings";+ bool has_custom_dims;+ size_t num_dims;public:- OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key);+ OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key, const size_t num_dims, const bool has_custom_dims);static Option<bool> is_model_valid(const nlohmann::json& model_config, size_t& num_dims);embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_tries = 2) override;std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200,diff --git a/src/embedder_manager.cpp b/src/embedder_manager.cppindex f61661ba..06e80aea 100644--- a/src/embedder_manager.cpp+++ b/src/embedder_manager.cpp@@ -29,9 +29,12 @@ Option<bool> EmbedderManager::validate_and_init_remote_model(const nlohmann::jsosize_t& num_dims) {const std::string& model_name = model_config["model_name"].get<std::string>();auto model_namespace = EmbedderManager::get_model_namespace(model_name);-+ bool has_custom_dims = false;if(model_namespace == "openai") {+ auto num_dims_before = num_dims;auto op = OpenAIEmbedder::is_model_valid(model_config, num_dims);+ // if the dimensions did not change, it means the model has custom dimensions+ has_custom_dims = num_dims_before == num_dims;if(!op.ok()) {return op;}@@ -53,7 +56,7 @@ Option<bool> EmbedderManager::validate_and_init_remote_model(const nlohmann::jsostd::string model_key = is_remote_model(model_name) ? RemoteEmbedder::get_model_key(model_config) : model_name;auto text_embedder_it = text_embedders.find(model_key);if(text_embedder_it == text_embedders.end()) {- text_embedders.emplace(model_key, std::make_shared<TextEmbedder>(model_config, num_dims));+ text_embedders.emplace(model_key, std::make_shared<TextEmbedder>(model_config, num_dims, has_custom_dims));}return Option<bool>(true);diff --git a/src/field.cpp b/src/field.cppindex 1aa28eef..491190f6 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -762,7 +762,7 @@ Option<bool> field::validate_and_init_embed_field(const tsl::htrie_map<char, fie}const auto& model_config = field_json[fields::embed][fields::model_config];- size_t num_dim = 0;+ size_t num_dim = field_json[fields::num_dim].get<size_t>();auto res = EmbedderManager::get_instance().validate_and_init_model(model_config, num_dim);if(!res.ok()) {return Option<bool>(res.code(), res.error());diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex 1955409d..56dfc335 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -63,7 +63,7 @@ TextEmbedder::TextEmbedder(const std::string& model_name) {}}-TextEmbedder::TextEmbedder(const nlohmann::json& model_config, size_t num_dims) {+TextEmbedder::TextEmbedder(const nlohmann::json& model_config, size_t num_dims, const bool has_custom_dims) {const std::string& model_name = model_config["model_name"].get<std::string>();LOG(INFO) << "Initializing remote embedding model: " << model_name;auto model_namespace = EmbedderManager::get_model_namespace(model_name);@@ -71,7 +71,7 @@ TextEmbedder::TextEmbedder(const nlohmann::json& model_config, size_t num_dims)if(model_namespace == "openai") {auto api_key = model_config["api_key"].get<std::string>();- remote_embedder_ = std::make_unique<OpenAIEmbedder>(model_name, api_key);+ remote_embedder_ = std::make_unique<OpenAIEmbedder>(model_name, api_key, num_dims, has_custom_dims);} else if(model_namespace == "google") {auto api_key = model_config["api_key"].get<std::string>();diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cppindex 1fe5e3bb..46f7193a 100644--- a/src/text_embedder_remote.cpp+++ b/src/text_embedder_remote.cpp@@ -68,7 +68,7 @@ const std::string RemoteEmbedder::get_model_key(const nlohmann::json& model_conf}}-OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key) : api_key(api_key), openai_model_path(openai_model_path) {+OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key, const size_t num_dims, const bool has_custom_dims) : api_key(api_key), openai_model_path(openai_model_path), num_dims(num_dims), has_custom_dims(has_custom_dims) {}@@ -133,6 +133,10 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,// remove "openai/" prefixreq_body["model"] = model_name_without_namespace;+ if(num_dims > 0) {+ req_body["dimensions"] = num_dims;+ }+std::string embedding_res;headers["Content-Type"] = "application/json";res_code = call_remote_api("POST", OPENAI_CREATE_EMBEDDING, req_body.dump(), embedding_res, res_headers, headers);@@ -174,6 +178,9 @@ embedding_res_t OpenAIEmbedder::Embed(const std::string& text, const size_t remostd::string res;nlohmann::json req_body;req_body["input"] = std::vector<std::string>{text};+ if(has_custom_dims) {+ req_body["dimensions"] = num_dims;+ }// remove "openai/" prefixreq_body["model"] = EmbedderManager::get_model_name_without_namespace(openai_model_path);auto res_code = call_remote_api("POST", OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);@@ -202,6 +209,9 @@ std::vector<embedding_res_t> OpenAIEmbedder::batch_embed(const std::vector<std::}nlohmann::json req_body;req_body["input"] = inputs;+ if(has_custom_dims) {+ req_body["dimensions"] = num_dims;+ }// remove "openai/" prefixreq_body["model"] = openai_model_path.substr(7);std::unordered_map<std::string, std::string> headers;diff --git a/test/collection_test.cpp b/test/collection_test.cppindex 664dff23..d6edc02b 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -5243,7 +5243,7 @@ TEST_F(CollectionTest, CatchPartialResponseFromRemoteEmbedding) {]})"_json;- OpenAIEmbedder embedder("", "");+ OpenAIEmbedder embedder("", "", 0, false);auto res = embedder.get_error_json(req_body, 200, partial_json);