Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add support for `openai/text-embedding-3-*` (#1558)

Ozan Armağan5722a9ddddcc
diff --git a/include/text_embedder.h b/include/text_embedder.h
index 1cf79fa9..f95c4101 100644
--- a/include/text_embedder.h
+++ b/include/text_embedder.h
@@ -14,7 +14,7 @@ class TextEmbedder {
// Constructor for local or public models
TextEmbedder(const std::string& model_path);
// Constructor for remote models
- TextEmbedder(const nlohmann::json& model_config, size_t num_dims);
+ TextEmbedder(const nlohmann::json& model_config, size_t num_dims, const bool has_custom_dims = false);
~TextEmbedder();
embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_tries = 2);
std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200,
diff --git a/include/text_embedder_remote.h b/include/text_embedder_remote.h
index 4aa75d82..5ba12690 100644
--- a/include/text_embedder_remote.h
+++ b/include/text_embedder_remote.h
@@ -49,8 +49,10 @@ class OpenAIEmbedder : public RemoteEmbedder {
std::string openai_model_path;
static constexpr char* OPENAI_LIST_MODELS = "https://api.openai.com/v1/models";
static constexpr char* OPENAI_CREATE_EMBEDDING = "https://api.openai.com/v1/embeddings";
+ bool has_custom_dims;
+ size_t num_dims;
public:
- OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key);
+ OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key, const size_t num_dims, const bool has_custom_dims);
static Option<bool> is_model_valid(const nlohmann::json& model_config, size_t& num_dims);
embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_tries = 2) override;
std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200,
diff --git a/src/embedder_manager.cpp b/src/embedder_manager.cpp
index f61661ba..06e80aea 100644
--- a/src/embedder_manager.cpp
+++ b/src/embedder_manager.cpp
@@ -29,9 +29,12 @@ Option<bool> EmbedderManager::validate_and_init_remote_model(const nlohmann::jso
size_t& num_dims) {
const std::string& model_name = model_config["model_name"].get<std::string>();
auto model_namespace = EmbedderManager::get_model_namespace(model_name);
-
+ bool has_custom_dims = false;
if(model_namespace == "openai") {
+ auto num_dims_before = num_dims;
auto op = OpenAIEmbedder::is_model_valid(model_config, num_dims);
+ // if the dimensions did not change, it means the model has custom dimensions
+ has_custom_dims = num_dims_before == num_dims;
if(!op.ok()) {
return op;
}
@@ -53,7 +56,7 @@ Option<bool> EmbedderManager::validate_and_init_remote_model(const nlohmann::jso
std::string model_key = is_remote_model(model_name) ? RemoteEmbedder::get_model_key(model_config) : model_name;
auto text_embedder_it = text_embedders.find(model_key);
if(text_embedder_it == text_embedders.end()) {
- text_embedders.emplace(model_key, std::make_shared<TextEmbedder>(model_config, num_dims));
+ text_embedders.emplace(model_key, std::make_shared<TextEmbedder>(model_config, num_dims, has_custom_dims));
}
return Option<bool>(true);
diff --git a/src/field.cpp b/src/field.cpp
index 1aa28eef..491190f6 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -762,7 +762,7 @@ Option<bool> field::validate_and_init_embed_field(const tsl::htrie_map<char, fie
}
const auto& model_config = field_json[fields::embed][fields::model_config];
- size_t num_dim = 0;
+ size_t num_dim = field_json[fields::num_dim].get<size_t>();
auto res = EmbedderManager::get_instance().validate_and_init_model(model_config, num_dim);
if(!res.ok()) {
return Option<bool>(res.code(), res.error());
diff --git a/src/text_embedder.cpp b/src/text_embedder.cpp
index 1955409d..56dfc335 100644
--- a/src/text_embedder.cpp
+++ b/src/text_embedder.cpp
@@ -63,7 +63,7 @@ TextEmbedder::TextEmbedder(const std::string& model_name) {
}
}
-TextEmbedder::TextEmbedder(const nlohmann::json& model_config, size_t num_dims) {
+TextEmbedder::TextEmbedder(const nlohmann::json& model_config, size_t num_dims, const bool has_custom_dims) {
const std::string& model_name = model_config["model_name"].get<std::string>();
LOG(INFO) << "Initializing remote embedding model: " << model_name;
auto model_namespace = EmbedderManager::get_model_namespace(model_name);
@@ -71,7 +71,7 @@ TextEmbedder::TextEmbedder(const nlohmann::json& model_config, size_t num_dims)
if(model_namespace == "openai") {
auto api_key = model_config["api_key"].get<std::string>();
- remote_embedder_ = std::make_unique<OpenAIEmbedder>(model_name, api_key);
+ remote_embedder_ = std::make_unique<OpenAIEmbedder>(model_name, api_key, num_dims, has_custom_dims);
} else if(model_namespace == "google") {
auto api_key = model_config["api_key"].get<std::string>();
diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cpp
index 1fe5e3bb..46f7193a 100644
--- a/src/text_embedder_remote.cpp
+++ b/src/text_embedder_remote.cpp
@@ -68,7 +68,7 @@ const std::string RemoteEmbedder::get_model_key(const nlohmann::json& model_conf
}
}
-OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key) : api_key(api_key), openai_model_path(openai_model_path) {
+OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key, const size_t num_dims, const bool has_custom_dims) : api_key(api_key), openai_model_path(openai_model_path), num_dims(num_dims), has_custom_dims(has_custom_dims) {
}
@@ -133,6 +133,10 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,
// remove "openai/" prefix
req_body["model"] = model_name_without_namespace;
+ if(num_dims > 0) {
+ req_body["dimensions"] = num_dims;
+ }
+
std::string embedding_res;
headers["Content-Type"] = "application/json";
res_code = call_remote_api("POST", OPENAI_CREATE_EMBEDDING, req_body.dump(), embedding_res, res_headers, headers);
@@ -174,6 +178,9 @@ embedding_res_t OpenAIEmbedder::Embed(const std::string& text, const size_t remo
std::string res;
nlohmann::json req_body;
req_body["input"] = std::vector<std::string>{text};
+ if(has_custom_dims) {
+ req_body["dimensions"] = num_dims;
+ }
// remove "openai/" prefix
req_body["model"] = EmbedderManager::get_model_name_without_namespace(openai_model_path);
auto res_code = call_remote_api("POST", OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);
@@ -202,6 +209,9 @@ std::vector<embedding_res_t> OpenAIEmbedder::batch_embed(const std::vector<std::
}
nlohmann::json req_body;
req_body["input"] = inputs;
+ if(has_custom_dims) {
+ req_body["dimensions"] = num_dims;
+ }
// remove "openai/" prefix
req_body["model"] = openai_model_path.substr(7);
std::unordered_map<std::string, std::string> headers;
diff --git a/test/collection_test.cpp b/test/collection_test.cpp
index 664dff23..d6edc02b 100644
--- a/test/collection_test.cpp
+++ b/test/collection_test.cpp
@@ -5243,7 +5243,7 @@ TEST_F(CollectionTest, CatchPartialResponseFromRemoteEmbedding) {
]
})"_json;
- OpenAIEmbedder embedder("", "");
+ OpenAIEmbedder embedder("", "", 0, false);
auto res = embedder.get_error_json(req_body, 200, partial_json);