Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Review Changes V

ozanarmagana9cb44388d1b
diff --git a/include/collection.h b/include/collection.h
index caaf897f..482d4be7 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -275,6 +275,8 @@ private:
Option<std::string> get_reference_field(const std::string & collection_name) const;
+ static void hide_credential(nlohmann::json& json, const std::string& credential_name);
+
public:
enum {MAX_ARRAY_MATCHES = 5};
diff --git a/include/text_embedder.h b/include/text_embedder.h
index 4e1084da..38666a2e 100644
--- a/include/text_embedder.h
+++ b/include/text_embedder.h
@@ -11,8 +11,10 @@
class TextEmbedder {
public:
+ // Constructor for local or public models
TextEmbedder(const std::string& model_path);
- TextEmbedder(const std::string& model_name, const std::string& api_key);
+ // Constructor for remote models
+ TextEmbedder(const nlohmann::json& model_config);
~TextEmbedder();
Option<std::vector<float>> Embed(const std::string& text);
Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs);
@@ -20,8 +22,7 @@ class TextEmbedder {
bool is_remote() {
return remote_embedder_ != nullptr;
}
- static bool is_model_valid(const std::string& model_path, unsigned int& num_dims);
- static Option<bool> is_model_valid(const std::string model_name, const std::string api_key, unsigned int& num_dims);
+ static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);
private:
std::unique_ptr<Ort::Session> session_;
Ort::Env env_;
@@ -31,4 +32,7 @@ class TextEmbedder {
std::string vocab_file_name;
static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);
std::string output_tensor_name;
+ static Option<bool> validate_remote_model(const nlohmann::json& model_config, unsigned int& num_dims);
+ static Option<bool> validate_local_or_public_model(const nlohmann::json& model_config, unsigned int& num_dims);
+ std::mutex mutex_;
};
diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.h
index d67a2c55..18a58824 100644
--- a/include/text_embedder_manager.h
+++ b/include/text_embedder_manager.h
@@ -64,6 +64,7 @@ public:
Option<bool> download_public_model(const std::string& model_name);
const bool is_public_model(const std::string& model_name);
+ static const bool is_remote_model(const std::string& model_name);
private:
TextEmbedderManager() = default;
@@ -73,7 +74,7 @@ private:
std::mutex text_embedders_mutex;
- Option<std::string> get_namespace(const std::string& model_name);
+ static Option<std::string> get_namespace(const std::string& model_name);
};
diff --git a/include/text_embedder_remote.h b/include/text_embedder_remote.h
index 660c441d..49a1ab95 100644
--- a/include/text_embedder_remote.h
+++ b/include/text_embedder_remote.h
@@ -2,6 +2,7 @@
#include <vector>
#include <string>
+#include <mutex>
#include "http_client.h"
#include "option.h"
@@ -9,6 +10,8 @@
class RemoteEmbedder {
+ protected:
+ static Option<bool> validate_string_properties(const nlohmann::json& model_config, const std::vector<std::string>& properties);
public:
virtual Option<std::vector<float>> Embed(const std::string& text) = 0;
virtual Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs) = 0;
@@ -23,7 +26,7 @@ class OpenAIEmbedder : public RemoteEmbedder {
static constexpr char* OPENAI_CREATE_EMBEDDING = "https://api.openai.com/v1/embeddings";
public:
OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key);
- static Option<bool> is_model_valid(const std::string& model_name, const std::string& api_key, unsigned int& num_dims);
+ static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);
Option<std::vector<float>> Embed(const std::string& text) override;
Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs) override;
};
@@ -38,7 +41,33 @@ class GoogleEmbedder : public RemoteEmbedder {
std::string google_api_key;
public:
GoogleEmbedder(const std::string& google_api_key);
- static Option<bool> is_model_valid(const std::string& model_name, const std::string& api_key, unsigned int& num_dims);
+ static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);
+ Option<std::vector<float>> Embed(const std::string& text) override;
+ Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs) override;
+};
+
+
+
+class GCPEmbedder : public RemoteEmbedder {
+ private:
+ std::string project_id;
+ std::string access_token;
+ std::string refresh_token;
+ std::string client_id;
+ std::string client_secret;
+ std::string model_name;
+ inline static const std::string GCP_EMBEDDING_URL_PART1 = "https://us-central1-aiplatform.googleapis.com/v1/projects/";
+ inline static const std::string GCP_EMBEDDING_URL_PART2 = "/locations/us-central1/publishers/google/models/";
+ inline static const std::string GCP_EMBEDDING_URL_PART3 = ":predict";
+ inline static const std::string GCP_AUTH_TOKEN_URL = "https://oauth2.googleapis.com/token";
+ static Option<std::string> generate_access_token(const std::string& refresh_token, const std::string& client_id, const std::string& client_secret);
+ static std::string get_gcp_embedding_url(const std::string& project_id, const std::string& model_name) {
+ return GCP_EMBEDDING_URL_PART1 + project_id + GCP_EMBEDDING_URL_PART2 + model_name + GCP_EMBEDDING_URL_PART3;
+ }
+ public:
+ GCPEmbedder(const std::string& project_id, const std::string& model_name, const std::string& access_token,
+ const std::string& refresh_token, const std::string& client_id, const std::string& client_secret);
+ static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);
Option<std::vector<float>> Embed(const std::string& text) override;
Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs) override;
};
diff --git a/src/collection.cpp b/src/collection.cpp
index 46ab45b5..e0325708 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -242,14 +242,13 @@ nlohmann::json Collection::get_summary_json() const {
if(coll_field.embed.count(fields::from) != 0) {
field_json[fields::embed] = coll_field.embed;
- if(field_json[fields::embed].count(fields::model_config) != 0 && field_json[fields::embed][fields::model_config].count(fields::api_key) != 0) {
- // hide api key with * except first 3 chars
- std::string api_key = field_json[fields::embed][fields::model_config][fields::api_key];
- if(api_key.size() > 3) {
- field_json[fields::embed][fields::model_config][fields::api_key] = api_key.replace(3, api_key.size() - 3, api_key.size() - 3, '*');
- } else {
- field_json[fields::embed][fields::model_config][fields::api_key] = api_key.replace(0, api_key.size(), api_key.size(), '*');
- }
+ if(field_json[fields::embed].count(fields::model_config) != 0) {
+ hide_credential(field_json[fields::embed][fields::model_config], "api_key");
+ hide_credential(field_json[fields::embed][fields::model_config], "access_token");
+ hide_credential(field_json[fields::embed][fields::model_config], "refresh_token");
+ hide_credential(field_json[fields::embed][fields::model_config], "client_id");
+ hide_credential(field_json[fields::embed][fields::model_config], "client_secret");
+ hide_credential(field_json[fields::embed][fields::model_config], "project_id");
}
}
@@ -4786,4 +4785,16 @@ void Collection::process_remove_field_for_embedding_fields(const field& the_fiel
}
+}
+
+void Collection::hide_credential(nlohmann::json& json, const std::string& credential_name) {
+ if(json.count(credential_name) != 0) {
+ // hide api key with * except first 3 chars
+ std::string credential_name_str = json[credential_name];
+ if(credential_name_str.size() > 3) {
+ json[credential_name] = credential_name_str.replace(3, credential_name_str.size() - 3, credential_name_str.size() - 3, '*');
+ } else {
+ json[credential_name] = credential_name_str.replace(0, credential_name_str.size(), credential_name_str.size(), '*');
+ }
+ }
}
\ No newline at end of file
diff --git a/src/field.cpp b/src/field.cpp
index 12183c9c..d2e834f0 100644
--- a/src/field.cpp
+++ b/src/field.cpp
@@ -712,15 +712,9 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso
}
}
- if(model_config.count("api_key") != 0) {
- auto res = TextEmbedder::is_model_valid(model_config[fields::model_name].get<std::string>(), model_config[fields::api_key].get<std::string>(), num_dim);
- if(!res.ok()) {
- return Option<bool>(res.code(), res.error());
- }
- } else {
- if(!TextEmbedder::is_model_valid(model_config[fields::model_name].get<std::string>(), num_dim)) {
- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` is invalid.");
- }
+ auto res = TextEmbedder::is_model_valid(model_config, num_dim);
+ if(!res.ok()) {
+ return Option<bool>(res.code(), res.error());
}
field_json[fields::num_dim] = num_dim;
} else {
diff --git a/src/index.cpp b/src/index.cpp
index c8971785..76d80fcb 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -454,7 +454,12 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& ite
get_doc_changes(index_rec.operation, index_rec.doc, index_rec.old_doc, index_rec.new_doc,
index_rec.del_doc);
scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);
- docs_to_embed.push_back(&index_rec.new_doc);
+ for(auto& field: index_rec.doc.items()) {
+ if(embedding_fields.find(field.key()) != embedding_fields.end()) {
+ docs_to_embed.push_back(&index_rec.doc);
+ break;
+ }
+ }
} else {
docs_to_embed.push_back(&index_rec.doc);
}
@@ -6370,7 +6375,7 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,
if(!embedder_op.ok()) {
return Option<bool>(400, embedder_op.error());
}
-
+
auto embedding_op = embedder_op.get()->batch_embed(text_to_embed);
if(!embedding_op.ok()) {
@@ -6378,7 +6383,7 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,
}
auto embeddings = embedding_op.get();
- for(size_t i = 0; i < documents.size(); i++) {
+ for(size_t i = 0; i < embeddings.size(); i++) {
(*documents[i])[field.name] = embeddings[i];
}
}
diff --git a/src/text_embedder.cpp b/src/text_embedder.cpp
index c285f06e..f33a55cf 100644
--- a/src/text_embedder.cpp
+++ b/src/text_embedder.cpp
@@ -36,15 +36,29 @@ TextEmbedder::TextEmbedder(const std::string& model_name) {
}
}
-TextEmbedder::TextEmbedder(const std::string& model_name, const std::string& api_key) {
+TextEmbedder::TextEmbedder(const nlohmann::json& model_config) {
+ auto model_name = model_config["model_name"].get<std::string>();
LOG(INFO) << "Loading model from remote: " << model_name;
auto model_namespace = TextEmbedderManager::get_model_namespace(model_name);
- if (model_namespace == "openai") {
+ if(model_namespace == "openai") {
+ auto api_key = model_config["api_key"].get<std::string>();
+
remote_embedder_ = std::make_unique<OpenAIEmbedder>(model_name, api_key);
- } else if (model_namespace == "google") {
+ } else if(model_namespace == "google") {
+ auto api_key = model_config["api_key"].get<std::string>();
+
remote_embedder_ = std::make_unique<GoogleEmbedder>(api_key);
- }
+ } else if(model_namespace == "gcp") {
+ auto project_id = model_config["project_id"].get<std::string>();
+ auto model_name = model_config["model_name"].get<std::string>();
+ auto access_token = model_config["access_token"].get<std::string>();
+ auto refresh_token = model_config["refresh_token"].get<std::string>();
+ auto client_id = model_config["client_id"].get<std::string>();
+ auto client_secret = model_config["client_secret"].get<std::string>();
+
+ remote_embedder_ = std::make_unique<GCPEmbedder>(project_id, model_name, access_token, refresh_token, client_id, client_secret);
+ }
}
@@ -65,6 +79,8 @@ Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {
if(is_remote()) {
return remote_embedder_->Embed(text);
} else {
+ // Cannot run same model in parallel, so lock the mutex
+ std::unique_lock<std::mutex> lock(mutex_);
auto encoded_input = tokenizer_->Encode(text);
// create input tensor object from data values
Ort::AllocatorWithDefaultOptions allocator;
@@ -117,7 +133,11 @@ Option<std::vector<std::vector<float>>> TextEmbedder::batch_embed(const std::vec
outputs.push_back(Embed(input).get());
}
} else {
- outputs = std::move(remote_embedder_->batch_embed(inputs).get());
+ auto embed_op = remote_embedder_->batch_embed(inputs);
+ if(!embed_op.ok()) {
+ return Option<std::vector<std::vector<float>>>(embed_op.code(), embed_op.error());
+ }
+ outputs = embed_op.get();
}
return Option<std::vector<std::vector<float>>>(outputs);
}
@@ -125,73 +145,97 @@ Option<std::vector<std::vector<float>>> TextEmbedder::batch_embed(const std::vec
TextEmbedder::~TextEmbedder() {
}
+Option<bool> TextEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {
+ auto model_name = model_config["model_name"].get<std::string>();
+
+ if(TextEmbedderManager::is_remote_model(model_name)) {
+ return validate_remote_model(model_config, num_dims);
+ } else {
+ return validate_local_or_public_model(model_config, num_dims);
+ }
+}
+
+Option<bool> TextEmbedder::validate_remote_model(const nlohmann::json& model_config, unsigned int& num_dims) {
+ auto model_name = model_config["model_name"].get<std::string>();
+ auto model_namespace = TextEmbedderManager::get_model_namespace(model_name);
+
+ if(model_namespace == "openai") {
+ return OpenAIEmbedder::is_model_valid(model_config, num_dims);
+ } else if(model_namespace == "google") {
+ return GoogleEmbedder::is_model_valid(model_config, num_dims);
+ } else if(model_namespace == "gcp") {
+ return GCPEmbedder::is_model_valid(model_config, num_dims);
+ }
+
+ return Option<bool>(400, "Invalid model namespace");
+}
-bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& num_dims) {
+Option<bool> TextEmbedder::validate_local_or_public_model(const nlohmann::json& model_config, unsigned int& num_dims) {
+ auto model_name = model_config["model_name"].get<std::string>();
LOG(INFO) << "Validating model: " << model_name;
if(TextEmbedderManager::get_instance().is_public_model(model_name)) {
auto res = TextEmbedderManager::get_instance().download_public_model(model_name);
if(!res.ok()) {
LOG(ERROR) << res.error();
- return false;
+ return Option<bool>(400, res.error());
}
}
-
Ort::SessionOptions session_options;
Ort::Env env;
std::string abs_path = TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::get_model_name_without_namespace(model_name));
if(!std::filesystem::exists(abs_path)) {
LOG(ERROR) << "Model file not found: " << abs_path;
- return false;
+ return Option<bool>(400, "Model file not found");
}
if(!TextEmbedderManager::get_instance().is_public_model(model_name)) {
if(!std::filesystem::exists(TextEmbedderManager::get_absolute_config_path(model_name))) {
LOG(ERROR) << "Config file not found: " << TextEmbedderManager::get_absolute_config_path(model_name);
- return false;
+ return Option<bool>(400, "Config file not found");
}
std::ifstream config_file(TextEmbedderManager::get_absolute_config_path(model_name));
nlohmann::json config;
config_file >> config;
if(config["model_type"].is_null() || config["vocab_file_name"].is_null()) {
LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);
- return false;
+ return Option<bool>(400, "Invalid config file");
}
if(!config["model_type"].is_string() || !config["vocab_file_name"].is_string()) {
LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);
- return false;
+ return Option<bool>(400, "Invalid config file");
}
if(!std::filesystem::exists(TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>())) {
LOG(ERROR) << "Vocab file not found: " << TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>();
- return false;
+ return Option<bool>(400, "Vocab file not found");
}
if(config["model_type"].get<std::string>() != "bert" && config["model_type"].get<std::string>() != "xlm_roberta" && config["model_type"].get<std::string>() != "distilbert") {
LOG(ERROR) << "Invalid model type: " << config["model_type"].get<std::string>();
- return false;
+ return Option<bool>(400, "Invalid model type");
}
}
Ort::Session session(env, abs_path.c_str(), session_options);
if(session.GetInputCount() != 3 && session.GetInputCount() != 2) {
LOG(ERROR) << "Invalid model: input count is not 3 or 2";
- return false;
+ return Option<bool>(400, "Invalid model: input count is not 3 or 2");
}
Ort::AllocatorWithDefaultOptions allocator;
auto input_ids_name = session.GetInputNameAllocated(0, allocator);
if (std::strcmp(input_ids_name.get(), "input_ids") != 0) {
LOG(ERROR) << "Invalid model: input_ids tensor not found";
- return false;
+ return Option<bool>(400, "Invalid model: input_ids tensor not found");
}
auto attention_mask_name = session.GetInputNameAllocated(1, allocator);
if (std::strcmp(attention_mask_name.get(), "attention_mask") != 0) {
LOG(ERROR) << "Invalid model: attention_mask tensor not found";
- return false;
+ return Option<bool>(400, "Invalid model: attention_mask tensor not found");
}
@@ -199,7 +243,7 @@ bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& n
auto token_type_ids_name = session.GetInputNameAllocated(2, allocator);
if (std::strcmp(token_type_ids_name.get(), "token_type_ids") != 0) {
LOG(ERROR) << "Invalid model: token_type_ids tensor not found";
- return false;
+ return Option<bool>(400, "Invalid model: token_type_ids tensor not found");
}
}
@@ -216,21 +260,8 @@ bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& n
if (!found_output_tensor) {
LOG(ERROR) << "Invalid model: Output tensor not found";
- return false;
+ return Option<bool>(400, "Invalid model: Output tensor not found");
}
- return true;
-}
-
-
-Option<bool> TextEmbedder::is_model_valid(const std::string model_name, const std::string api_key, unsigned int& num_dims) {
- auto model_namespace = TextEmbedderManager::get_model_namespace(model_name);
-
- if(model_namespace == "openai") {
- return OpenAIEmbedder::is_model_valid(model_name, api_key, num_dims);
- } else if(model_namespace == "google") {
- return GoogleEmbedder::is_model_valid(model_name, api_key, num_dims);
- } else {
- return Option<bool>(400, "Invalid model namespace");
- }
+ return Option<bool>(true);
}
\ No newline at end of file
diff --git a/src/text_embedder_manager.cpp b/src/text_embedder_manager.cpp
index 75d472a6..de68640d 100644
--- a/src/text_embedder_manager.cpp
+++ b/src/text_embedder_manager.cpp
@@ -10,7 +10,7 @@ Option<TextEmbedder*>TextEmbedderManager::get_text_embedder(const nlohmann::json
std::unique_lock<std::mutex> lock(text_embedders_mutex);
const std::string& model_name = model_config.at("model_name");
if(text_embedders[model_name] == nullptr) {
- if(model_config.count("api_key") == 0) {
+ if(!is_remote_model(model_name)) {
if(is_public_model(model_name)) {
// download the model if it doesn't exist
auto res = download_public_model(model_name);
@@ -20,7 +20,7 @@ Option<TextEmbedder*>TextEmbedderManager::get_text_embedder(const nlohmann::json
}
text_embedders[model_name] = std::make_shared<TextEmbedder>(get_model_name_without_namespace(model_name));
} else {
- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_config.at("api_key").get<std::string>());
+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_config);
}
}
return Option<TextEmbedder*>(text_embedders[model_name].get());
@@ -262,4 +262,9 @@ const std::string TextEmbedderManager::get_model_namespace(const std::string& mo
} else {
return "ts";
}
+}
+
+const bool TextEmbedderManager::is_remote_model(const std::string& model_name) {
+ auto model_namespace = get_namespace(model_name);
+ return model_namespace.ok() && (model_namespace.get() == "openai" || model_namespace.get() == "google" || model_namespace.get() == "gcp");
}
\ No newline at end of file
diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cpp
index 8c257df8..d1a05317 100644
--- a/src/text_embedder_remote.cpp
+++ b/src/text_embedder_remote.cpp
@@ -2,19 +2,34 @@
#include "text_embedder_manager.h"
+Option<bool> RemoteEmbedder::validate_string_properties(const nlohmann::json& model_config, const std::vector<std::string>& properties) {
+ for(auto& property : properties) {
+ if(model_config.count(property) == 0 || !model_config[property].is_string()) {
+ return Option<bool>(400, "Property `embed.model_config." + property + " is missing or is not a string.");
+ }
+ }
+ return Option<bool>(true);
+}
+
+
OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key) : api_key(api_key), openai_model_path(openai_model_path) {
}
-Option<bool> OpenAIEmbedder::is_model_valid(const std::string& model_name, const std::string& api_key, unsigned int& num_dims) {
- if (model_name.empty() || api_key.empty()) {
- return Option<bool>(400, "Invalid OpenAI model name or API key");
+Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {
+ auto validate_properties = validate_string_properties(model_config, {"model_name", "api_key"});
+
+ if (!validate_properties.ok()) {
+ return validate_properties;
}
+ auto model_name = model_config["model_name"].get<std::string>();
+ auto api_key = model_config["api_key"].get<std::string>();
+
if(TextEmbedderManager::get_model_namespace(model_name) != "openai") {
- return Option<bool>(400, "Invalid OpenAI model name");
+ return Option<bool>(400, "Property `embed.model_config.model_name` malformed.");
}
HttpClient& client = HttpClient::get_instance();
@@ -43,27 +58,24 @@ Option<bool> OpenAIEmbedder::is_model_valid(const std::string& model_name, const
}
if (!found) {
- return Option<bool>(400, "OpenAI model not found");
+ return Option<bool>(400, "Property `embed.model_config.model_name` is not a valid OpenAI model.");
}
+ nlohmann::json req_body;
+ req_body["input"] = "typesense";
+ // remove "openai/" prefix
+ req_body["model"] = model_name_without_namespace;
+ res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);
- // This part is hard coded for now. Because OpenAI API does not provide a way to get the output dimensions of the model.
- if(model_name.find("-ada-") != std::string::npos) {
- if(model_name.substr(model_name.length() - 3) == "002") {
- num_dims = 1536;
- } else {
- num_dims = 1024;
+ if (res_code != 200) {
+ nlohmann::json json_res = nlohmann::json::parse(res);
+ if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
+ return Option<bool>(400, "OpenAI API error: " + res);
}
- }
- else if(model_name.find("-davinci-") != std::string::npos) {
- num_dims = 12288;
- } else if(model_name.find("-curie-") != std::string::npos) {
- num_dims = 4096;
- } else if(model_name.find("-babbage-") != std::string::npos) {
- num_dims = 2048;
- } else {
- num_dims = 768;
+ return Option<bool>(400, "OpenAI API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
}
+ auto embedding = nlohmann::json::parse(res)["data"][0]["embedding"].get<std::vector<float>>();
+ num_dims = embedding.size();
return Option<bool>(true);
}
@@ -77,7 +89,7 @@ Option<std::vector<float>> OpenAIEmbedder::Embed(const std::string& text) {
nlohmann::json req_body;
req_body["input"] = text;
// remove "openai/" prefix
- req_body["model"] = openai_model_path.substr(7);
+ req_body["model"] = TextEmbedderManager::get_model_name_without_namespace(openai_model_path);
auto res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);
if (res_code != 200) {
nlohmann::json json_res = nlohmann::json::parse(res);
@@ -125,17 +137,22 @@ GoogleEmbedder::GoogleEmbedder(const std::string& google_api_key) : google_api_k
}
-Option<bool> GoogleEmbedder::is_model_valid(const std::string& model_name, const std::string& api_key, unsigned int& num_dims) {
- if(model_name.empty() || api_key.empty()) {
- return Option<bool>(400, "Invalid Google model name or API key");
+Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {
+ auto validate_properties = validate_string_properties(model_config, {"model_name", "api_key"});
+
+ if (!validate_properties.ok()) {
+ return validate_properties;
}
+ auto model_name = model_config["model_name"].get<std::string>();
+ auto api_key = model_config["api_key"].get<std::string>();
+
if(TextEmbedderManager::get_model_namespace(model_name) != "google") {
- return Option<bool>(400, "Invalid Google model name");
+ return Option<bool>(400, "Property `embed.model_config.model_name` malformed.");
}
if(TextEmbedderManager::get_model_name_without_namespace(model_name) != std::string(SUPPORTED_MODEL)) {
- return Option<bool>(400, "Invalid Google model name");
+ return Option<bool>(400, "Property `embed.model_config.model_name` is not a supported Google model.");
}
HttpClient& client = HttpClient::get_instance();
@@ -156,7 +173,7 @@ Option<bool> GoogleEmbedder::is_model_valid(const std::string& model_name, const
return Option<bool>(400, "Google API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
}
- num_dims = GOOGLE_EMBEDDING_DIM;
+ num_dims = nlohmann::json::parse(res)["embedding"]["value"].get<std::vector<float>>().size();
return Option<bool>(true);
}
@@ -196,3 +213,197 @@ Option<std::vector<std::vector<float>>> GoogleEmbedder::batch_embed(const std::v
return Option<std::vector<std::vector<float>>>(outputs);
}
+
+
+GCPEmbedder::GCPEmbedder(const std::string& project_id, const std::string& model_name, const std::string& access_token,
+ const std::string& refresh_token, const std::string& client_id, const std::string& client_secret) :
+ project_id(project_id), access_token(access_token), refresh_token(refresh_token), client_id(client_id), client_secret(client_secret) {
+
+ this->model_name = TextEmbedderManager::get_model_name_without_namespace(model_name);
+}
+
+Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {
+ auto validate_properties = validate_string_properties(model_config, {"model_name", "project_id", "access_token", "refresh_token", "client_id", "client_secret"});
+
+ if (!validate_properties.ok()) {
+ return validate_properties;
+ }
+
+ auto model_name = model_config["model_name"].get<std::string>();
+ auto project_id = model_config["project_id"].get<std::string>();
+ auto access_token = model_config["access_token"].get<std::string>();
+ auto refresh_token = model_config["refresh_token"].get<std::string>();
+ auto client_id = model_config["client_id"].get<std::string>();
+ auto client_secret = model_config["client_secret"].get<std::string>();
+
+ if(TextEmbedderManager::get_model_namespace(model_name) != "gcp") {
+ return Option<bool>(400, "Invalid GCP model name");
+ }
+
+ auto model_name_without_namespace = TextEmbedderManager::get_model_name_without_namespace(model_name);
+
+ HttpClient& client = HttpClient::get_instance();
+ std::unordered_map<std::string, std::string> headers;
+ std::map<std::string, std::string> res_headers;
+ headers["Content-Type"] = "application/json";
+ headers["Authorization"] = "Bearer " + access_token;
+ std::string res;
+ nlohmann::json req_body;
+ req_body["instances"] = nlohmann::json::array();
+ nlohmann::json instance;
+ instance["content"] = "typesense";
+ req_body["instances"].push_back(instance);
+
+ auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name_without_namespace), req_body.dump(), res, res_headers, headers);
+
+ if(res_code != 200) {
+ nlohmann::json json_res = nlohmann::json::parse(res);
+ if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
+ return Option<bool>(400, "GCP API error: " + res);
+ }
+ return Option<bool>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
+ }
+
+ auto res_json = nlohmann::json::parse(res);
+ if(res_json.count("predictions") == 0 || res_json["predictions"].size() == 0 || res_json["predictions"][0].count("embeddings") == 0) {
+ LOG(INFO) << "Invalid response from GCP API: " << res_json.dump();
+ return Option<bool>(400, "GCP API error: Invalid response");
+ }
+
+ auto generate_access_token_res = generate_access_token(refresh_token, client_id, client_secret);
+ if(!generate_access_token_res.ok()) {
+ return Option<bool>(400, "Invalid client_id, client_secret or refresh_token in `embed.model config'.");
+ }
+
+ num_dims = res_json["predictions"][0]["embeddings"]["values"].size();
+
+ return Option<bool>(true);
+}
+
+Option<std::vector<float>> GCPEmbedder::Embed(const std::string& text) {
+ nlohmann::json req_body;
+ req_body["instances"] = nlohmann::json::array();
+ nlohmann::json instance;
+ instance["content"] = text;
+ req_body["instances"].push_back(instance);
+ std::unordered_map<std::string, std::string> headers;
+ headers["Authorization"] = "Bearer " + access_token;
+ headers["Content-Type"] = "application/json";
+ std::map<std::string, std::string> res_headers;
+ std::string res;
+ HttpClient& client = HttpClient::get_instance();
+
+ auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
+
+ if(res_code != 200) {
+ if(res_code == 401) {
+ auto refresh_op = generate_access_token(refresh_token, client_id, client_secret);
+ if(!refresh_op.ok()) {
+ return Option<std::vector<float>>(refresh_op.code(), refresh_op.error());
+ }
+ access_token = refresh_op.get();
+ // retry
+ headers["Authorization"] = "Bearer " + access_token;
+ res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
+ }
+ }
+
+ if(res_code != 200) {
+ nlohmann::json json_res = nlohmann::json::parse(res);
+ if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
+ return Option<std::vector<float>>(400, "GCP API error: " + res);
+ }
+ return Option<std::vector<float>>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
+ }
+
+ nlohmann::json res_json = nlohmann::json::parse(res);
+ return Option<std::vector<float>>(res_json["predictions"][0]["embeddings"]["values"].get<std::vector<float>>());
+}
+
+
+Option<std::vector<std::vector<float>>> GCPEmbedder::batch_embed(const std::vector<std::string>& inputs) {
+ // GCP API has a limit of 5 instances per request
+ if(inputs.size() > 5) {
+ std::vector<std::vector<float>> res;
+ for(size_t i = 0; i < inputs.size(); i += 5) {
+ auto batch_res = batch_embed(std::vector<std::string>(inputs.begin() + i, inputs.begin() + std::min(i + 5, inputs.size())));
+ if(!batch_res.ok()) {
+ LOG(INFO) << "Batch embedding failed: " << batch_res.error();
+ return Option<std::vector<std::vector<float>>>(batch_res.code(), batch_res.error());
+ }
+ auto batch = batch_res.get();
+ res.insert(res.end(), batch.begin(), batch.end());
+ }
+ auto opt = Option<std::vector<std::vector<float>>>(res);
+ return opt;
+ }
+ nlohmann::json req_body;
+ req_body["instances"] = nlohmann::json::array();
+ for(const auto& input : inputs) {
+ nlohmann::json instance;
+ instance["content"] = input;
+ req_body["instances"].push_back(instance);
+ }
+ std::unordered_map<std::string, std::string> headers;
+ headers["Authorization"] = "Bearer " + access_token;
+ headers["Content-Type"] = "application/json";
+ std::map<std::string, std::string> res_headers;
+ std::string res;
+ HttpClient& client = HttpClient::get_instance();
+ auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
+ if(res_code != 200) {
+ if(res_code == 401) {
+ auto refresh_op = generate_access_token(refresh_token, client_id, client_secret);
+ if(!refresh_op.ok()) {
+ return Option<std::vector<std::vector<float>>>(refresh_op.code(), refresh_op.error());
+ }
+ access_token = refresh_op.get();
+ // retry
+ headers["Authorization"] = "Bearer " + access_token;
+ res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
+ }
+ }
+
+ if(res_code != 200) {
+ nlohmann::json json_res = nlohmann::json::parse(res);
+ if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
+ return Option<std::vector<std::vector<float>>>(400, "GCP API error: " + res);
+ }
+ return Option<std::vector<std::vector<float>>>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
+ }
+
+ nlohmann::json res_json = nlohmann::json::parse(res);
+ std::vector<std::vector<float>> outputs;
+ for(const auto& prediction : res_json["predictions"]) {
+ outputs.push_back(prediction["embeddings"]["values"].get<std::vector<float>>());
+ }
+
+ return Option<std::vector<std::vector<float>>>(outputs);
+}
+
+Option<std::string> GCPEmbedder::generate_access_token(const std::string& refresh_token, const std::string& client_id, const std::string& client_secret) {
+ HttpClient& client = HttpClient::get_instance();
+ std::unordered_map<std::string, std::string> headers;
+ headers["Content-Type"] = "application/x-www-form-urlencoded";
+ std::map<std::string, std::string> res_headers;
+ std::string res;
+ std::string req_body;
+ req_body = "grant_type=refresh_token&client_id=" + client_id + "&client_secret=" + client_secret + "&refresh_token=" + refresh_token;
+
+ auto res_code = client.post_response(GCP_AUTH_TOKEN_URL, req_body, res, res_headers, headers);
+
+ if(res_code != 200) {
+ nlohmann::json json_res = nlohmann::json::parse(res);
+ if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
+ return Option<std::string>(400, "GCP API error: " + res);
+ }
+ return Option<std::string>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());
+ }
+
+ nlohmann::json res_json = nlohmann::json::parse(res);
+ std::string access_token = res_json["access_token"].get<std::string>();
+
+ return Option<std::string>(access_token);
+}
+
+