Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Review Changes V
ozanarmagana9cb44388d1b
diff --git a/include/collection.h b/include/collection.hindex caaf897f..482d4be7 100644--- a/include/collection.h+++ b/include/collection.h@@ -275,6 +275,8 @@ private:Option<std::string> get_reference_field(const std::string & collection_name) const;+ static void hide_credential(nlohmann::json& json, const std::string& credential_name);+public:enum {MAX_ARRAY_MATCHES = 5};diff --git a/include/text_embedder.h b/include/text_embedder.hindex 4e1084da..38666a2e 100644--- a/include/text_embedder.h+++ b/include/text_embedder.h@@ -11,8 +11,10 @@class TextEmbedder {public:+ // Constructor for local or public modelsTextEmbedder(const std::string& model_path);- TextEmbedder(const std::string& model_name, const std::string& api_key);+ // Constructor for remote models+ TextEmbedder(const nlohmann::json& model_config);~TextEmbedder();Option<std::vector<float>> Embed(const std::string& text);Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs);@@ -20,8 +22,7 @@ class TextEmbedder {bool is_remote() {return remote_embedder_ != nullptr;}- static bool is_model_valid(const std::string& model_path, unsigned int& num_dims);- static Option<bool> is_model_valid(const std::string model_name, const std::string api_key, unsigned int& num_dims);+ static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);private:std::unique_ptr<Ort::Session> session_;Ort::Env env_;@@ -31,4 +32,7 @@ class TextEmbedder {std::string vocab_file_name;static std::vector<float> mean_pooling(const std::vector<std::vector<float>>& input);std::string output_tensor_name;+ static Option<bool> validate_remote_model(const nlohmann::json& model_config, unsigned int& num_dims);+ static Option<bool> validate_local_or_public_model(const nlohmann::json& model_config, unsigned int& num_dims);+ std::mutex mutex_;};diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.hindex d67a2c55..18a58824 100644--- a/include/text_embedder_manager.h+++ b/include/text_embedder_manager.h@@ -64,6 +64,7 @@ public:Option<bool> download_public_model(const std::string& model_name);const bool is_public_model(const std::string& model_name);+ static const bool is_remote_model(const std::string& model_name);private:TextEmbedderManager() = default;@@ -73,7 +74,7 @@ private:std::mutex text_embedders_mutex;- Option<std::string> get_namespace(const std::string& model_name);+ static Option<std::string> get_namespace(const std::string& model_name);};diff --git a/include/text_embedder_remote.h b/include/text_embedder_remote.hindex 660c441d..49a1ab95 100644--- a/include/text_embedder_remote.h+++ b/include/text_embedder_remote.h@@ -2,6 +2,7 @@#include <vector>#include <string>+#include <mutex>#include "http_client.h"#include "option.h"@@ -9,6 +10,8 @@class RemoteEmbedder {+ protected:+ static Option<bool> validate_string_properties(const nlohmann::json& model_config, const std::vector<std::string>& properties);public:virtual Option<std::vector<float>> Embed(const std::string& text) = 0;virtual Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs) = 0;@@ -23,7 +26,7 @@ class OpenAIEmbedder : public RemoteEmbedder {static constexpr char* OPENAI_CREATE_EMBEDDING = "https://api.openai.com/v1/embeddings";public:OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key);- static Option<bool> is_model_valid(const std::string& model_name, const std::string& api_key, unsigned int& num_dims);+ static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);Option<std::vector<float>> Embed(const std::string& text) override;Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs) override;};@@ -38,7 +41,33 @@ class GoogleEmbedder : public RemoteEmbedder {std::string google_api_key;public:GoogleEmbedder(const std::string& google_api_key);- static Option<bool> is_model_valid(const std::string& model_name, const std::string& api_key, unsigned int& num_dims);+ static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);+ Option<std::vector<float>> Embed(const std::string& text) override;+ Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs) override;+};++++class GCPEmbedder : public RemoteEmbedder {+ private:+ std::string project_id;+ std::string access_token;+ std::string refresh_token;+ std::string client_id;+ std::string client_secret;+ std::string model_name;+ inline static const std::string GCP_EMBEDDING_URL_PART1 = "https://us-central1-aiplatform.googleapis.com/v1/projects/";+ inline static const std::string GCP_EMBEDDING_URL_PART2 = "/locations/us-central1/publishers/google/models/";+ inline static const std::string GCP_EMBEDDING_URL_PART3 = ":predict";+ inline static const std::string GCP_AUTH_TOKEN_URL = "https://oauth2.googleapis.com/token";+ static Option<std::string> generate_access_token(const std::string& refresh_token, const std::string& client_id, const std::string& client_secret);+ static std::string get_gcp_embedding_url(const std::string& project_id, const std::string& model_name) {+ return GCP_EMBEDDING_URL_PART1 + project_id + GCP_EMBEDDING_URL_PART2 + model_name + GCP_EMBEDDING_URL_PART3;+ }+ public:+ GCPEmbedder(const std::string& project_id, const std::string& model_name, const std::string& access_token,+ const std::string& refresh_token, const std::string& client_id, const std::string& client_secret);+ static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);Option<std::vector<float>> Embed(const std::string& text) override;Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs) override;};diff --git a/src/collection.cpp b/src/collection.cppindex 46ab45b5..e0325708 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -242,14 +242,13 @@ nlohmann::json Collection::get_summary_json() const {if(coll_field.embed.count(fields::from) != 0) {field_json[fields::embed] = coll_field.embed;- if(field_json[fields::embed].count(fields::model_config) != 0 && field_json[fields::embed][fields::model_config].count(fields::api_key) != 0) {- // hide api key with * except first 3 chars- std::string api_key = field_json[fields::embed][fields::model_config][fields::api_key];- if(api_key.size() > 3) {- field_json[fields::embed][fields::model_config][fields::api_key] = api_key.replace(3, api_key.size() - 3, api_key.size() - 3, '*');- } else {- field_json[fields::embed][fields::model_config][fields::api_key] = api_key.replace(0, api_key.size(), api_key.size(), '*');- }+ if(field_json[fields::embed].count(fields::model_config) != 0) {+ hide_credential(field_json[fields::embed][fields::model_config], "api_key");+ hide_credential(field_json[fields::embed][fields::model_config], "access_token");+ hide_credential(field_json[fields::embed][fields::model_config], "refresh_token");+ hide_credential(field_json[fields::embed][fields::model_config], "client_id");+ hide_credential(field_json[fields::embed][fields::model_config], "client_secret");+ hide_credential(field_json[fields::embed][fields::model_config], "project_id");}}@@ -4786,4 +4785,16 @@ void Collection::process_remove_field_for_embedding_fields(const field& the_fiel}+}++void Collection::hide_credential(nlohmann::json& json, const std::string& credential_name) {+ if(json.count(credential_name) != 0) {+ // hide api key with * except first 3 chars+ std::string credential_name_str = json[credential_name];+ if(credential_name_str.size() > 3) {+ json[credential_name] = credential_name_str.replace(3, credential_name_str.size() - 3, credential_name_str.size() - 3, '*');+ } else {+ json[credential_name] = credential_name_str.replace(0, credential_name_str.size(), credential_name_str.size(), '*');+ }+ }}\ No newline at end of filediff --git a/src/field.cpp b/src/field.cppindex 12183c9c..d2e834f0 100644--- a/src/field.cpp+++ b/src/field.cpp@@ -712,15 +712,9 @@ Option<bool> field::json_field_to_field(bool enable_nested_fields, nlohmann::jso}}- if(model_config.count("api_key") != 0) {- auto res = TextEmbedder::is_model_valid(model_config[fields::model_name].get<std::string>(), model_config[fields::api_key].get<std::string>(), num_dim);- if(!res.ok()) {- return Option<bool>(res.code(), res.error());- }- } else {- if(!TextEmbedder::is_model_valid(model_config[fields::model_name].get<std::string>(), num_dim)) {- return Option<bool>(400, "Property `" + fields::embed + "." + fields::model_config + "." + fields::model_name + "` is invalid.");- }+ auto res = TextEmbedder::is_model_valid(model_config, num_dim);+ if(!res.ok()) {+ return Option<bool>(res.code(), res.error());}field_json[fields::num_dim] = num_dim;} else {diff --git a/src/index.cpp b/src/index.cppindex c8971785..76d80fcb 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -454,7 +454,12 @@ void Index::validate_and_preprocess(Index *index, std::vector<index_record>& iteget_doc_changes(index_rec.operation, index_rec.doc, index_rec.old_doc, index_rec.new_doc,index_rec.del_doc);scrub_reindex_doc(search_schema, index_rec.doc, index_rec.del_doc, index_rec.old_doc);- docs_to_embed.push_back(&index_rec.new_doc);+ for(auto& field: index_rec.doc.items()) {+ if(embedding_fields.find(field.key()) != embedding_fields.end()) {+ docs_to_embed.push_back(&index_rec.doc);+ break;+ }+ }} else {docs_to_embed.push_back(&index_rec.doc);}@@ -6370,7 +6375,7 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,if(!embedder_op.ok()) {return Option<bool>(400, embedder_op.error());}-+auto embedding_op = embedder_op.get()->batch_embed(text_to_embed);if(!embedding_op.ok()) {@@ -6378,7 +6383,7 @@ Option<bool> Index::batch_embed_fields(std::vector<nlohmann::json*>& documents,}auto embeddings = embedding_op.get();- for(size_t i = 0; i < documents.size(); i++) {+ for(size_t i = 0; i < embeddings.size(); i++) {(*documents[i])[field.name] = embeddings[i];}}diff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex c285f06e..f33a55cf 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -36,15 +36,29 @@ TextEmbedder::TextEmbedder(const std::string& model_name) {}}-TextEmbedder::TextEmbedder(const std::string& model_name, const std::string& api_key) {+TextEmbedder::TextEmbedder(const nlohmann::json& model_config) {+ auto model_name = model_config["model_name"].get<std::string>();LOG(INFO) << "Loading model from remote: " << model_name;auto model_namespace = TextEmbedderManager::get_model_namespace(model_name);- if (model_namespace == "openai") {+ if(model_namespace == "openai") {+ auto api_key = model_config["api_key"].get<std::string>();+remote_embedder_ = std::make_unique<OpenAIEmbedder>(model_name, api_key);- } else if (model_namespace == "google") {+ } else if(model_namespace == "google") {+ auto api_key = model_config["api_key"].get<std::string>();+remote_embedder_ = std::make_unique<GoogleEmbedder>(api_key);- }+ } else if(model_namespace == "gcp") {+ auto project_id = model_config["project_id"].get<std::string>();+ auto model_name = model_config["model_name"].get<std::string>();+ auto access_token = model_config["access_token"].get<std::string>();+ auto refresh_token = model_config["refresh_token"].get<std::string>();+ auto client_id = model_config["client_id"].get<std::string>();+ auto client_secret = model_config["client_secret"].get<std::string>();++ remote_embedder_ = std::make_unique<GCPEmbedder>(project_id, model_name, access_token, refresh_token, client_id, client_secret);+ }}@@ -65,6 +79,8 @@ Option<std::vector<float>> TextEmbedder::Embed(const std::string& text) {if(is_remote()) {return remote_embedder_->Embed(text);} else {+ // Cannot run same model in parallel, so lock the mutex+ std::unique_lock<std::mutex> lock(mutex_);auto encoded_input = tokenizer_->Encode(text);// create input tensor object from data valuesOrt::AllocatorWithDefaultOptions allocator;@@ -117,7 +133,11 @@ Option<std::vector<std::vector<float>>> TextEmbedder::batch_embed(const std::vecoutputs.push_back(Embed(input).get());}} else {- outputs = std::move(remote_embedder_->batch_embed(inputs).get());+ auto embed_op = remote_embedder_->batch_embed(inputs);+ if(!embed_op.ok()) {+ return Option<std::vector<std::vector<float>>>(embed_op.code(), embed_op.error());+ }+ outputs = embed_op.get();}return Option<std::vector<std::vector<float>>>(outputs);}@@ -125,73 +145,97 @@ Option<std::vector<std::vector<float>>> TextEmbedder::batch_embed(const std::vecTextEmbedder::~TextEmbedder() {}+Option<bool> TextEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {+ auto model_name = model_config["model_name"].get<std::string>();++ if(TextEmbedderManager::is_remote_model(model_name)) {+ return validate_remote_model(model_config, num_dims);+ } else {+ return validate_local_or_public_model(model_config, num_dims);+ }+}++Option<bool> TextEmbedder::validate_remote_model(const nlohmann::json& model_config, unsigned int& num_dims) {+ auto model_name = model_config["model_name"].get<std::string>();+ auto model_namespace = TextEmbedderManager::get_model_namespace(model_name);++ if(model_namespace == "openai") {+ return OpenAIEmbedder::is_model_valid(model_config, num_dims);+ } else if(model_namespace == "google") {+ return GoogleEmbedder::is_model_valid(model_config, num_dims);+ } else if(model_namespace == "gcp") {+ return GCPEmbedder::is_model_valid(model_config, num_dims);+ }++ return Option<bool>(400, "Invalid model namespace");+}-bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& num_dims) {+Option<bool> TextEmbedder::validate_local_or_public_model(const nlohmann::json& model_config, unsigned int& num_dims) {+ auto model_name = model_config["model_name"].get<std::string>();LOG(INFO) << "Validating model: " << model_name;if(TextEmbedderManager::get_instance().is_public_model(model_name)) {auto res = TextEmbedderManager::get_instance().download_public_model(model_name);if(!res.ok()) {LOG(ERROR) << res.error();- return false;+ return Option<bool>(400, res.error());}}-Ort::SessionOptions session_options;Ort::Env env;std::string abs_path = TextEmbedderManager::get_absolute_model_path(TextEmbedderManager::get_model_name_without_namespace(model_name));if(!std::filesystem::exists(abs_path)) {LOG(ERROR) << "Model file not found: " << abs_path;- return false;+ return Option<bool>(400, "Model file not found");}if(!TextEmbedderManager::get_instance().is_public_model(model_name)) {if(!std::filesystem::exists(TextEmbedderManager::get_absolute_config_path(model_name))) {LOG(ERROR) << "Config file not found: " << TextEmbedderManager::get_absolute_config_path(model_name);- return false;+ return Option<bool>(400, "Config file not found");}std::ifstream config_file(TextEmbedderManager::get_absolute_config_path(model_name));nlohmann::json config;config_file >> config;if(config["model_type"].is_null() || config["vocab_file_name"].is_null()) {LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);- return false;+ return Option<bool>(400, "Invalid config file");}if(!config["model_type"].is_string() || !config["vocab_file_name"].is_string()) {LOG(ERROR) << "Invalid config file: " << TextEmbedderManager::get_absolute_config_path(model_name);- return false;+ return Option<bool>(400, "Invalid config file");}if(!std::filesystem::exists(TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>())) {LOG(ERROR) << "Vocab file not found: " << TextEmbedderManager::get_model_subdir(model_name) + "/" + config["vocab_file_name"].get<std::string>();- return false;+ return Option<bool>(400, "Vocab file not found");}if(config["model_type"].get<std::string>() != "bert" && config["model_type"].get<std::string>() != "xlm_roberta" && config["model_type"].get<std::string>() != "distilbert") {LOG(ERROR) << "Invalid model type: " << config["model_type"].get<std::string>();- return false;+ return Option<bool>(400, "Invalid model type");}}Ort::Session session(env, abs_path.c_str(), session_options);if(session.GetInputCount() != 3 && session.GetInputCount() != 2) {LOG(ERROR) << "Invalid model: input count is not 3 or 2";- return false;+ return Option<bool>(400, "Invalid model: input count is not 3 or 2");}Ort::AllocatorWithDefaultOptions allocator;auto input_ids_name = session.GetInputNameAllocated(0, allocator);if (std::strcmp(input_ids_name.get(), "input_ids") != 0) {LOG(ERROR) << "Invalid model: input_ids tensor not found";- return false;+ return Option<bool>(400, "Invalid model: input_ids tensor not found");}auto attention_mask_name = session.GetInputNameAllocated(1, allocator);if (std::strcmp(attention_mask_name.get(), "attention_mask") != 0) {LOG(ERROR) << "Invalid model: attention_mask tensor not found";- return false;+ return Option<bool>(400, "Invalid model: attention_mask tensor not found");}@@ -199,7 +243,7 @@ bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& nauto token_type_ids_name = session.GetInputNameAllocated(2, allocator);if (std::strcmp(token_type_ids_name.get(), "token_type_ids") != 0) {LOG(ERROR) << "Invalid model: token_type_ids tensor not found";- return false;+ return Option<bool>(400, "Invalid model: token_type_ids tensor not found");}}@@ -216,21 +260,8 @@ bool TextEmbedder::is_model_valid(const std::string& model_name, unsigned int& nif (!found_output_tensor) {LOG(ERROR) << "Invalid model: Output tensor not found";- return false;+ return Option<bool>(400, "Invalid model: Output tensor not found");}- return true;-}---Option<bool> TextEmbedder::is_model_valid(const std::string model_name, const std::string api_key, unsigned int& num_dims) {- auto model_namespace = TextEmbedderManager::get_model_namespace(model_name);-- if(model_namespace == "openai") {- return OpenAIEmbedder::is_model_valid(model_name, api_key, num_dims);- } else if(model_namespace == "google") {- return GoogleEmbedder::is_model_valid(model_name, api_key, num_dims);- } else {- return Option<bool>(400, "Invalid model namespace");- }+ return Option<bool>(true);}\ No newline at end of filediff --git a/src/text_embedder_manager.cpp b/src/text_embedder_manager.cppindex 75d472a6..de68640d 100644--- a/src/text_embedder_manager.cpp+++ b/src/text_embedder_manager.cpp@@ -10,7 +10,7 @@ Option<TextEmbedder*>TextEmbedderManager::get_text_embedder(const nlohmann::jsonstd::unique_lock<std::mutex> lock(text_embedders_mutex);const std::string& model_name = model_config.at("model_name");if(text_embedders[model_name] == nullptr) {- if(model_config.count("api_key") == 0) {+ if(!is_remote_model(model_name)) {if(is_public_model(model_name)) {// download the model if it doesn't existauto res = download_public_model(model_name);@@ -20,7 +20,7 @@ Option<TextEmbedder*>TextEmbedderManager::get_text_embedder(const nlohmann::json}text_embedders[model_name] = std::make_shared<TextEmbedder>(get_model_name_without_namespace(model_name));} else {- text_embedders[model_name] = std::make_shared<TextEmbedder>(model_name, model_config.at("api_key").get<std::string>());+ text_embedders[model_name] = std::make_shared<TextEmbedder>(model_config);}}return Option<TextEmbedder*>(text_embedders[model_name].get());@@ -262,4 +262,9 @@ const std::string TextEmbedderManager::get_model_namespace(const std::string& mo} else {return "ts";}+}++const bool TextEmbedderManager::is_remote_model(const std::string& model_name) {+ auto model_namespace = get_namespace(model_name);+ return model_namespace.ok() && (model_namespace.get() == "openai" || model_namespace.get() == "google" || model_namespace.get() == "gcp");}\ No newline at end of filediff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cppindex 8c257df8..d1a05317 100644--- a/src/text_embedder_remote.cpp+++ b/src/text_embedder_remote.cpp@@ -2,19 +2,34 @@#include "text_embedder_manager.h"+Option<bool> RemoteEmbedder::validate_string_properties(const nlohmann::json& model_config, const std::vector<std::string>& properties) {+ for(auto& property : properties) {+ if(model_config.count(property) == 0 || !model_config[property].is_string()) {+ return Option<bool>(400, "Property `embed.model_config." + property + " is missing or is not a string.");+ }+ }+ return Option<bool>(true);+}++OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key) : api_key(api_key), openai_model_path(openai_model_path) {}-Option<bool> OpenAIEmbedder::is_model_valid(const std::string& model_name, const std::string& api_key, unsigned int& num_dims) {- if (model_name.empty() || api_key.empty()) {- return Option<bool>(400, "Invalid OpenAI model name or API key");+Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {+ auto validate_properties = validate_string_properties(model_config, {"model_name", "api_key"});++ if (!validate_properties.ok()) {+ return validate_properties;}+ auto model_name = model_config["model_name"].get<std::string>();+ auto api_key = model_config["api_key"].get<std::string>();+if(TextEmbedderManager::get_model_namespace(model_name) != "openai") {- return Option<bool>(400, "Invalid OpenAI model name");+ return Option<bool>(400, "Property `embed.model_config.model_name` malformed.");}HttpClient& client = HttpClient::get_instance();@@ -43,27 +58,24 @@ Option<bool> OpenAIEmbedder::is_model_valid(const std::string& model_name, const}if (!found) {- return Option<bool>(400, "OpenAI model not found");+ return Option<bool>(400, "Property `embed.model_config.model_name` is not a valid OpenAI model.");}+ nlohmann::json req_body;+ req_body["input"] = "typesense";+ // remove "openai/" prefix+ req_body["model"] = model_name_without_namespace;+ res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);- // This part is hard coded for now. Because OpenAI API does not provide a way to get the output dimensions of the model.- if(model_name.find("-ada-") != std::string::npos) {- if(model_name.substr(model_name.length() - 3) == "002") {- num_dims = 1536;- } else {- num_dims = 1024;+ if (res_code != 200) {+ nlohmann::json json_res = nlohmann::json::parse(res);+ if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {+ return Option<bool>(400, "OpenAI API error: " + res);}- }- else if(model_name.find("-davinci-") != std::string::npos) {- num_dims = 12288;- } else if(model_name.find("-curie-") != std::string::npos) {- num_dims = 4096;- } else if(model_name.find("-babbage-") != std::string::npos) {- num_dims = 2048;- } else {- num_dims = 768;+ return Option<bool>(400, "OpenAI API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());}+ auto embedding = nlohmann::json::parse(res)["data"][0]["embedding"].get<std::vector<float>>();+ num_dims = embedding.size();return Option<bool>(true);}@@ -77,7 +89,7 @@ Option<std::vector<float>> OpenAIEmbedder::Embed(const std::string& text) {nlohmann::json req_body;req_body["input"] = text;// remove "openai/" prefix- req_body["model"] = openai_model_path.substr(7);+ req_body["model"] = TextEmbedderManager::get_model_name_without_namespace(openai_model_path);auto res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);if (res_code != 200) {nlohmann::json json_res = nlohmann::json::parse(res);@@ -125,17 +137,22 @@ GoogleEmbedder::GoogleEmbedder(const std::string& google_api_key) : google_api_k}-Option<bool> GoogleEmbedder::is_model_valid(const std::string& model_name, const std::string& api_key, unsigned int& num_dims) {- if(model_name.empty() || api_key.empty()) {- return Option<bool>(400, "Invalid Google model name or API key");+Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {+ auto validate_properties = validate_string_properties(model_config, {"model_name", "api_key"});++ if (!validate_properties.ok()) {+ return validate_properties;}+ auto model_name = model_config["model_name"].get<std::string>();+ auto api_key = model_config["api_key"].get<std::string>();+if(TextEmbedderManager::get_model_namespace(model_name) != "google") {- return Option<bool>(400, "Invalid Google model name");+ return Option<bool>(400, "Property `embed.model_config.model_name` malformed.");}if(TextEmbedderManager::get_model_name_without_namespace(model_name) != std::string(SUPPORTED_MODEL)) {- return Option<bool>(400, "Invalid Google model name");+ return Option<bool>(400, "Property `embed.model_config.model_name` is not a supported Google model.");}HttpClient& client = HttpClient::get_instance();@@ -156,7 +173,7 @@ Option<bool> GoogleEmbedder::is_model_valid(const std::string& model_name, constreturn Option<bool>(400, "Google API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());}- num_dims = GOOGLE_EMBEDDING_DIM;+ num_dims = nlohmann::json::parse(res)["embedding"]["value"].get<std::vector<float>>().size();return Option<bool>(true);}@@ -196,3 +213,197 @@ Option<std::vector<std::vector<float>>> GoogleEmbedder::batch_embed(const std::vreturn Option<std::vector<std::vector<float>>>(outputs);}+++GCPEmbedder::GCPEmbedder(const std::string& project_id, const std::string& model_name, const std::string& access_token,+ const std::string& refresh_token, const std::string& client_id, const std::string& client_secret) :+ project_id(project_id), access_token(access_token), refresh_token(refresh_token), client_id(client_id), client_secret(client_secret) {++ this->model_name = TextEmbedderManager::get_model_name_without_namespace(model_name);+}++Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims) {+ auto validate_properties = validate_string_properties(model_config, {"model_name", "project_id", "access_token", "refresh_token", "client_id", "client_secret"});++ if (!validate_properties.ok()) {+ return validate_properties;+ }++ auto model_name = model_config["model_name"].get<std::string>();+ auto project_id = model_config["project_id"].get<std::string>();+ auto access_token = model_config["access_token"].get<std::string>();+ auto refresh_token = model_config["refresh_token"].get<std::string>();+ auto client_id = model_config["client_id"].get<std::string>();+ auto client_secret = model_config["client_secret"].get<std::string>();++ if(TextEmbedderManager::get_model_namespace(model_name) != "gcp") {+ return Option<bool>(400, "Invalid GCP model name");+ }++ auto model_name_without_namespace = TextEmbedderManager::get_model_name_without_namespace(model_name);++ HttpClient& client = HttpClient::get_instance();+ std::unordered_map<std::string, std::string> headers;+ std::map<std::string, std::string> res_headers;+ headers["Content-Type"] = "application/json";+ headers["Authorization"] = "Bearer " + access_token;+ std::string res;+ nlohmann::json req_body;+ req_body["instances"] = nlohmann::json::array();+ nlohmann::json instance;+ instance["content"] = "typesense";+ req_body["instances"].push_back(instance);++ auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name_without_namespace), req_body.dump(), res, res_headers, headers);++ if(res_code != 200) {+ nlohmann::json json_res = nlohmann::json::parse(res);+ if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {+ return Option<bool>(400, "GCP API error: " + res);+ }+ return Option<bool>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());+ }++ auto res_json = nlohmann::json::parse(res);+ if(res_json.count("predictions") == 0 || res_json["predictions"].size() == 0 || res_json["predictions"][0].count("embeddings") == 0) {+ LOG(INFO) << "Invalid response from GCP API: " << res_json.dump();+ return Option<bool>(400, "GCP API error: Invalid response");+ }++ auto generate_access_token_res = generate_access_token(refresh_token, client_id, client_secret);+ if(!generate_access_token_res.ok()) {+ return Option<bool>(400, "Invalid client_id, client_secret or refresh_token in `embed.model config'.");+ }++ num_dims = res_json["predictions"][0]["embeddings"]["values"].size();++ return Option<bool>(true);+}++Option<std::vector<float>> GCPEmbedder::Embed(const std::string& text) {+ nlohmann::json req_body;+ req_body["instances"] = nlohmann::json::array();+ nlohmann::json instance;+ instance["content"] = text;+ req_body["instances"].push_back(instance);+ std::unordered_map<std::string, std::string> headers;+ headers["Authorization"] = "Bearer " + access_token;+ headers["Content-Type"] = "application/json";+ std::map<std::string, std::string> res_headers;+ std::string res;+ HttpClient& client = HttpClient::get_instance();++ auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);++ if(res_code != 200) {+ if(res_code == 401) {+ auto refresh_op = generate_access_token(refresh_token, client_id, client_secret);+ if(!refresh_op.ok()) {+ return Option<std::vector<float>>(refresh_op.code(), refresh_op.error());+ }+ access_token = refresh_op.get();+ // retry+ headers["Authorization"] = "Bearer " + access_token;+ res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);+ }+ }++ if(res_code != 200) {+ nlohmann::json json_res = nlohmann::json::parse(res);+ if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {+ return Option<std::vector<float>>(400, "GCP API error: " + res);+ }+ return Option<std::vector<float>>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());+ }++ nlohmann::json res_json = nlohmann::json::parse(res);+ return Option<std::vector<float>>(res_json["predictions"][0]["embeddings"]["values"].get<std::vector<float>>());+}+++Option<std::vector<std::vector<float>>> GCPEmbedder::batch_embed(const std::vector<std::string>& inputs) {+ // GCP API has a limit of 5 instances per request+ if(inputs.size() > 5) {+ std::vector<std::vector<float>> res;+ for(size_t i = 0; i < inputs.size(); i += 5) {+ auto batch_res = batch_embed(std::vector<std::string>(inputs.begin() + i, inputs.begin() + std::min(i + 5, inputs.size())));+ if(!batch_res.ok()) {+ LOG(INFO) << "Batch embedding failed: " << batch_res.error();+ return Option<std::vector<std::vector<float>>>(batch_res.code(), batch_res.error());+ }+ auto batch = batch_res.get();+ res.insert(res.end(), batch.begin(), batch.end());+ }+ auto opt = Option<std::vector<std::vector<float>>>(res);+ return opt;+ }+ nlohmann::json req_body;+ req_body["instances"] = nlohmann::json::array();+ for(const auto& input : inputs) {+ nlohmann::json instance;+ instance["content"] = input;+ req_body["instances"].push_back(instance);+ }+ std::unordered_map<std::string, std::string> headers;+ headers["Authorization"] = "Bearer " + access_token;+ headers["Content-Type"] = "application/json";+ std::map<std::string, std::string> res_headers;+ std::string res;+ HttpClient& client = HttpClient::get_instance();+ auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);+ if(res_code != 200) {+ if(res_code == 401) {+ auto refresh_op = generate_access_token(refresh_token, client_id, client_secret);+ if(!refresh_op.ok()) {+ return Option<std::vector<std::vector<float>>>(refresh_op.code(), refresh_op.error());+ }+ access_token = refresh_op.get();+ // retry+ headers["Authorization"] = "Bearer " + access_token;+ res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);+ }+ }++ if(res_code != 200) {+ nlohmann::json json_res = nlohmann::json::parse(res);+ if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {+ return Option<std::vector<std::vector<float>>>(400, "GCP API error: " + res);+ }+ return Option<std::vector<std::vector<float>>>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());+ }++ nlohmann::json res_json = nlohmann::json::parse(res);+ std::vector<std::vector<float>> outputs;+ for(const auto& prediction : res_json["predictions"]) {+ outputs.push_back(prediction["embeddings"]["values"].get<std::vector<float>>());+ }++ return Option<std::vector<std::vector<float>>>(outputs);+}++Option<std::string> GCPEmbedder::generate_access_token(const std::string& refresh_token, const std::string& client_id, const std::string& client_secret) {+ HttpClient& client = HttpClient::get_instance();+ std::unordered_map<std::string, std::string> headers;+ headers["Content-Type"] = "application/x-www-form-urlencoded";+ std::map<std::string, std::string> res_headers;+ std::string res;+ std::string req_body;+ req_body = "grant_type=refresh_token&client_id=" + client_id + "&client_secret=" + client_secret + "&refresh_token=" + refresh_token;++ auto res_code = client.post_response(GCP_AUTH_TOKEN_URL, req_body, res, res_headers, headers);++ if(res_code != 200) {+ nlohmann::json json_res = nlohmann::json::parse(res);+ if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {+ return Option<std::string>(400, "GCP API error: " + res);+ }+ return Option<std::string>(400, "GCP API error: " + nlohmann::json::parse(res)["error"]["message"].get<std::string>());+ }++ nlohmann::json res_json = nlohmann::json::parse(res);+ std::string access_token = res_json["access_token"].get<std::string>();++ return Option<std::string>(access_token);+}++