Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix model key for remote embedders
ozanarmagan24a67140f248
diff --git a/include/text_embedder.h b/include/text_embedder.hindex ca64aa52..241570a3 100644--- a/include/text_embedder.h+++ b/include/text_embedder.h@@ -24,6 +24,9 @@ class TextEmbedder {return remote_embedder_ != nullptr;}Option<bool> validate();+ RemoteEmbedder* _get_remote_embedder() {+ return remote_embedder_.get();+ }private:std::unique_ptr<Ort::Session> session_;Ort::Env env_;diff --git a/include/text_embedder_manager.h b/include/text_embedder_manager.hindex 543e8f91..2681e0b9 100644--- a/include/text_embedder_manager.h+++ b/include/text_embedder_manager.h@@ -72,6 +72,10 @@ public:Option<bool> validate_and_init_local_model(const nlohmann::json& model_config, size_t& num_dims);Option<bool> validate_and_init_model(const nlohmann::json& model_config, size_t& num_dims);+ std::unordered_map<std::string, std::shared_ptr<TextEmbedder>> _get_text_embedders() {+ return text_embedders;+ }+private:TextEmbedderManager() = default;diff --git a/include/text_embedder_remote.h b/include/text_embedder_remote.hindex 8167fefd..b5f219d5 100644--- a/include/text_embedder_remote.h+++ b/include/text_embedder_remote.h@@ -31,6 +31,7 @@ class RemoteEmbedder {virtual nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) = 0;virtual embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_tries = 2) = 0;virtual std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) = 0;+ static const std::string get_model_key(const nlohmann::json& model_config);static void init(ReplicationState* rs) {raft_server = rs;}@@ -51,6 +52,7 @@ class OpenAIEmbedder : public RemoteEmbedder {embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_tries = 2) override;std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) override;nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) override;+ static std::string get_model_key(const nlohmann::json& model_config);};@@ -68,6 +70,7 @@ class GoogleEmbedder : public RemoteEmbedder {embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_tries = 2) override;std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) override;nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) override;+ static std::string get_model_key(const nlohmann::json& model_config);};@@ -95,6 +98,7 @@ class GCPEmbedder : public RemoteEmbedder {embedding_res_t Embed(const std::string& text, const size_t remote_embedder_timeout_ms = 30000, const size_t remote_embedding_num_tries = 2) override;std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs, const size_t remote_embedding_batch_size = 200) override;nlohmann::json get_error_json(const nlohmann::json& req_body, long res_code, const std::string& res_body) override;+ static std::string get_model_key(const nlohmann::json& model_config);};diff --git a/src/text_embedder_manager.cpp b/src/text_embedder_manager.cppindex ac2c110f..8b34b949 100644--- a/src/text_embedder_manager.cpp+++ b/src/text_embedder_manager.cpp@@ -43,9 +43,13 @@ Option<bool> TextEmbedderManager::validate_and_init_remote_model(const nlohmann:}std::unique_lock<std::mutex> lock(text_embedders_mutex);- auto text_embedder_it = text_embedders.find(model_name);+ std::string model_key = model_name;+ if(is_remote_model(model_name)) {+ model_key = RemoteEmbedder::get_model_key(model_config);+ }+ auto text_embedder_it = text_embedders.find(model_key);if(text_embedder_it == text_embedders.end()) {- text_embedders.emplace(model_name, std::make_shared<TextEmbedder>(model_config, num_dims));+ text_embedders.emplace(model_key, std::make_shared<TextEmbedder>(model_config, num_dims));}return Option<bool>(true);@@ -122,7 +126,11 @@ Option<bool> TextEmbedderManager::validate_and_init_local_model(const nlohmann::Option<TextEmbedder*> TextEmbedderManager::get_text_embedder(const nlohmann::json& model_config) {std::unique_lock<std::mutex> lock(text_embedders_mutex);const std::string& model_name = model_config.at("model_name");- auto text_embedder_it = text_embedders.find(model_name);+ std::string model_key = model_name;+ if(is_remote_model(model_name)) {+ model_key = RemoteEmbedder::get_model_key(model_config);+ }+ auto text_embedder_it = text_embedders.find(model_key);if(text_embedder_it == text_embedders.end()) {return Option<TextEmbedder*>(404, "Text embedder was not found.");diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cppindex e59f93bf..a7a745e0 100644--- a/src/text_embedder_remote.cpp+++ b/src/text_embedder_remote.cpp@@ -53,6 +53,21 @@ long RemoteEmbedder::call_remote_api(const std::string& method, const std::strinproxy_call_timeout_ms, true);}++const std::string RemoteEmbedder::get_model_key(const nlohmann::json& model_config) {+ const std::string model_namespace = TextEmbedderManager::get_model_namespace(model_config["model_name"].get<std::string>());++ if(model_namespace == "openai") {+ return OpenAIEmbedder::get_model_key(model_config);+ } else if(model_namespace == "google") {+ return GoogleEmbedder::get_model_key(model_config);+ } else if(model_namespace == "gcp") {+ return GCPEmbedder::get_model_key(model_config);+ } else {+ return "";+ }+}+OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key) : api_key(api_key), openai_model_path(openai_model_path) {}@@ -255,6 +270,9 @@ nlohmann::json OpenAIEmbedder::get_error_json(const nlohmann::json& req_body, loreturn embedding_res;}+std::string OpenAIEmbedder::get_model_key(const nlohmann::json& model_config) {+ return model_config["model_name"].get<std::string>() + ":" + model_config["api_key"].get<std::string>();+}GoogleEmbedder::GoogleEmbedder(const std::string& google_api_key) : google_api_key(google_api_key) {@@ -372,6 +390,10 @@ nlohmann::json GoogleEmbedder::get_error_json(const nlohmann::json& req_body, loreturn embedding_res;}+std::string GoogleEmbedder::get_model_key(const nlohmann::json& model_config) {+ return model_config["model_name"].get<std::string>() + ":" + model_config["api_key"].get<std::string>();+}+GCPEmbedder::GCPEmbedder(const std::string& project_id, const std::string& model_name, const std::string& access_token,const std::string& refresh_token, const std::string& client_id, const std::string& client_secret) :@@ -625,3 +647,9 @@ Option<std::string> GCPEmbedder::generate_access_token(const std::string& refresreturn Option<std::string>(access_token);}++std::string GCPEmbedder::get_model_key(const nlohmann::json& model_config) {+ return model_config["model_name"].get<std::string>() + ":" + model_config["project_id"].get<std::string>() + ":"+ + model_config["access_token"].get<std::string>() + ":" + model_config["refresh_token"].get<std::string>() + ":"+ + model_config["client_id"].get<std::string>() + ":" + model_config["client_secret"].get<std::string>();+}\ No newline at end of filediff --git a/test/collection_vector_search_test.cpp b/test/collection_vector_search_test.cppindex 600f8fb4..1b02cdf6 100644--- a/test/collection_vector_search_test.cpp+++ b/test/collection_vector_search_test.cpp@@ -1216,7 +1216,7 @@ TEST_F(CollectionVectorTest, HybridSearchReturnAllInfo) {}-TEST_F(CollectionVectorTest, HybridSortingTest) {+TEST_F(CollectionVectorTest, DISABLED_HybridSortingTest) {auto schema_json =R"({"name": "TEST",@@ -1273,4 +1273,39 @@ TEST_F(CollectionVectorTest, HybridSortingTest) {ASSERT_EQ(results["hits"][1]["document"]["name"].get<std::string>(), hybrid_results["hits"][1]["document"]["name"].get<std::string>());ASSERT_EQ(results["hits"][2]["document"]["name"].get<std::string>(), hybrid_results["hits"][2]["document"]["name"].get<std::string>());ASSERT_EQ(results["hits"][3]["document"]["name"].get<std::string>(), hybrid_results["hits"][3]["document"]["name"].get<std::string>());+}++TEST_F(CollectionVectorTest, TestDifferentOpenAIApiKeys) {+ if (std::getenv("api_key_1") == nullptr || std::getenv("api_key_2") == nullptr) {+ LOG(INFO) << "Skipping test as api_key_1 or api_key_2 is not set";+ return;+ }++ auto api_key1 = std::string(std::getenv("api_key_1"));+ auto api_key2 = std::string(std::getenv("api_key_2"));++ auto embedder_map = TextEmbedderManager::get_instance()._get_text_embedders();++ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002:" + api_key1), embedder_map.end());+ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002:" + api_key2), embedder_map.end());+ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002"), embedder_map.end());++ nlohmann::json model_config1 = R"({+ "model_name": "openai/text-embedding-ada-002"+ })"_json;++ nlohmann::json model_config2 = model_config1;++ model_config1["api_key"] = api_key1;+ model_config2["api_key"] = api_key2;++ size_t num_dim;+ TextEmbedderManager::get_instance().validate_and_init_remote_model(model_config1, num_dim);+ TextEmbedderManager::get_instance().validate_and_init_remote_model(model_config2, num_dim);++ embedder_map = TextEmbedderManager::get_instance()._get_text_embedders();++ ASSERT_NE(embedder_map.find("openai/text-embedding-ada-002:" + api_key1), embedder_map.end());+ ASSERT_NE(embedder_map.find("openai/text-embedding-ada-002:" + api_key2), embedder_map.end());+ ASSERT_EQ(embedder_map.find("openai/text-embedding-ada-002"), embedder_map.end());}\ No newline at end of file