Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Proxy remote embedding API calls from leader node
ozanarmagana29c76ff54a5
diff --git a/include/core_api.h b/include/core_api.hindex 9d31d65a..cd8cdff8 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -167,3 +167,6 @@ bool is_doc_del_route(uint64_t route_hash);Option<std::pair<std::string,std::string>> get_api_key_and_ip(const std::string& metadata);void init_api(uint32_t cache_num_entries);+++bool proxy_embedding(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);diff --git a/include/http_proxy.h b/include/http_proxy.hnew file mode 100644index 00000000..e116f0af--- /dev/null+++ b/include/http_proxy.h@@ -0,0 +1,43 @@+#pragma once++#include <unordered_map>+#include <string>+#include "http_client.h"+#include "lru/lru.hpp"+++struct http_proxy_res_t {+ std::string body;+ std::map<std::string, std::string> headers;+ long status_code;++ bool operator==(const http_proxy_res_t& other) const {+ return body == other.body && headers == other.headers && status_code == other.status_code;+ }++ bool operator!=(const http_proxy_res_t& other) const {+ return !(*this == other);+ }+};+++class HttpProxy {+ // singleton class for http proxy+ public:+ static HttpProxy& get_instance() {+ static HttpProxy instance;+ return instance;+ }+ HttpProxy(const HttpProxy&) = delete;+ void operator=(const HttpProxy&) = delete;+ HttpProxy(HttpProxy&&) = delete;+ void operator=(HttpProxy&&) = delete;+ http_proxy_res_t send(const std::string& url, const std::string& method, const std::string& body, const std::unordered_map<std::string, std::string>& headers);+ private:+ HttpProxy();+ ~HttpProxy() = default;+++ // lru cache for http requests+ LRU::TimedCache<std::string, http_proxy_res_t> cache;+};\ No newline at end of filediff --git a/include/text_embedder_remote.h b/include/text_embedder_remote.hindex 095272c2..131973d6 100644--- a/include/text_embedder_remote.h+++ b/include/text_embedder_remote.h@@ -4,6 +4,7 @@#include <string>#include <mutex>#include "http_client.h"+#include "raft_server.h"#include "option.h"@@ -12,9 +13,15 @@class RemoteEmbedder {protected:static Option<bool> validate_string_properties(const nlohmann::json& model_config, const std::vector<std::string>& properties);+ static long call_remote_api(const std::string& method, const std::string& url, const std::string& body, std::string& res_body, std::map<std::string, std::string>& headers, const std::unordered_map<std::string, std::string>& req_headers);+ static inline ReplicationState* raft_server = nullptr;public:virtual Option<std::vector<float>> Embed(const std::string& text) = 0;virtual Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs) = 0;+ static void set_raft_server(ReplicationState* rs) {+ raft_server = rs;+ }+};diff --git a/src/core_api.cpp b/src/core_api.cppindex 6b9534e9..b97a2c62 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -15,6 +15,7 @@#include "lru/lru.hpp"#include "ratelimit_manager.h"#include "event_manager.h"+#include "http_proxy.h"using namespace std::chrono_literals;@@ -2136,3 +2137,44 @@ bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::sharedres->set_200(R"({"ok": true)");return true;}+++bool proxy_embedding(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ HttpProxy& proxy = HttpProxy::get_instance();++ nlohmann::json req_json;++ try {+ req_json = nlohmann::json::parse(req->body);+ } catch(const nlohmann::json::parse_error& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ res->set_400("Bad JSON.");+ return false;+ }++ std::string body, url, method;+ std::unordered_map<std::string, std::string> headers;++ try {+ body = req_json["body"].get<std::string>();+ url = req_json["url"].get<std::string>();+ method = req_json["method"].get<std::string>();+ for(auto& header: req_json["headers"].items()) {+ headers[header.key()] = header.value().get<std::string>();+ }+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ res->set_400("Bad JSON.");+ return false;+ }++ auto response = proxy.send(url, method, body, headers);++ if(response.status_code != 200) {+ res->set(response.status_code, response.body);+ return false;+ }++ res->set_200(response.body);+ return true;+}\ No newline at end of filediff --git a/src/http_proxy.cpp b/src/http_proxy.cppnew file mode 100644index 00000000..19e8b61c--- /dev/null+++ b/src/http_proxy.cpp@@ -0,0 +1,45 @@+#include "http_proxy.h"+#include "logger.h"+#include <chrono>++using namespace std::chrono_literals;++HttpProxy::HttpProxy() : cache(30s){+}+++http_proxy_res_t HttpProxy::send(const std::string& url, const std::string& method, const std::string& body, const std::unordered_map<std::string, std::string>& headers) {+ // check if url is in cache+ std::string key;+ key += url;+ key += method;+ key += body;+ for (auto& header : headers) {+ key += header.first;+ key += header.second;+ }+ if (cache.contains(key)) {+ return cache[key];+ }+ // if not, make http request+ HttpClient& client = HttpClient::get_instance();+ http_proxy_res_t res;++ if(method == "GET") {+ res.status_code = client.get_response(url, res.body, res.headers, headers);+ } else if(method == "POST") {+ res.status_code = client.post_response(url, body, res.body, res.headers, headers);+ } else if(method == "PUT") {+ res.status_code = client.put_response(url, body, res.body, res.headers);+ } else if(method == "DELETE") {+ res.status_code = client.delete_response(url, res.body, res.headers);+ } else {+ res.status_code = 400;+ res.body = "Invalid method: " + method;+ }++ // add to cache+ cache.insert(key, res);++ return res;+}\ No newline at end of filediff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 30b7281d..f57f15c4 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -98,6 +98,9 @@ void master_server_routes() {server->del("/limits/active/:id", del_throttle);server->del("/limits/exceeds/:id", del_exceed);server->post("/config", post_config, false, false);++ // for proxying remote embedders+ server->post("/proxy", proxy_embedding);}void (*backward::SignalHandling::_callback)(int sig, backward::StackTrace&) = nullptr;diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 920b6b34..d5912fd1 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -219,6 +219,15 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const stdreturn write_to_leader(request, response);}+ route_path* rpath;+ bool found = server->get_route(request->route_hash, &rpath);+ found = found && rpath->handler == proxy_embedding;+ if(found && !node->is_leader()) {+ response->set_500("Proxy embedding is only supported on leader.");+ auto req_res = new async_req_res_t(request, response, true);+ return message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);+ }+// Serialize request to replicated WAL so that all the nodes in the group receive it as well.// NOTE: actual write must be done only on the `on_apply` method to maintain consistency.diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cppindex 604e1071..cb0fe240 100644--- a/src/text_embedder_remote.cpp+++ b/src/text_embedder_remote.cpp@@ -11,6 +11,21 @@ Option<bool> RemoteEmbedder::validate_string_properties(const nlohmann::json& moreturn Option<bool>(true);}+long RemoteEmbedder::call_remote_api(const std::string& method, const std::string& url, const std::string& body, std::string& res_body,+ std::map<std::string, std::string>& headers, const std::unordered_map<std::string, std::string>& req_headers) {+ auto leader_url = raft_server->get_leader_url();+ leader_url += "proxy";+ nlohmann::json req_body;+ req_body["method"] = method;+ req_body["url"] = url;+ req_body["body"] = body;+ req_body["headers"] = nlohmann::json::object();+ for(auto& header : req_headers) {+ req_body["headers"][header.first] = header.second;+ }+ return HttpClient::get_instance().post_response(leader_url, req_body.dump(), res_body, headers, {}, 10000, true);+}+OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key) : api_key(api_key), openai_model_path(openai_model_path) {@@ -32,12 +47,11 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,return Option<bool>(400, "Property `embed.model_config.model_name` malformed.");}- HttpClient& client = HttpClient::get_instance();std::unordered_map<std::string, std::string> headers;std::map<std::string, std::string> res_headers;headers["Authorization"] = "Bearer " + api_key;std::string res;- auto res_code = client.get_response(OPENAI_LIST_MODELS, res, res_headers, headers);+ auto res_code = call_remote_api("GET", OPENAI_LIST_MODELS, "", res, res_headers, headers);if (res_code != 200) {nlohmann::json json_res = nlohmann::json::parse(res);if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {@@ -67,7 +81,7 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,std::string embedding_res;headers["Content-Type"] = "application/json";- res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), embedding_res, res_headers, headers);+ res_code = call_remote_api("POST", OPENAI_CREATE_EMBEDDING, req_body.dump(), embedding_res, res_headers, headers);if (res_code != 200) {@@ -84,7 +98,6 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,}Option<std::vector<float>> OpenAIEmbedder::Embed(const std::string& text) {- HttpClient& client = HttpClient::get_instance();std::unordered_map<std::string, std::string> headers;std::map<std::string, std::string> res_headers;headers["Authorization"] = "Bearer " + api_key;@@ -94,7 +107,7 @@ Option<std::vector<float>> OpenAIEmbedder::Embed(const std::string& text) {req_body["input"] = text;// remove "openai/" prefixreq_body["model"] = TextEmbedderManager::get_model_name_without_namespace(openai_model_path);- auto res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);+ auto res_code = call_remote_api("POST", OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);if (res_code != 200) {nlohmann::json json_res = nlohmann::json::parse(res);if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {@@ -115,9 +128,7 @@ Option<std::vector<std::vector<float>>> OpenAIEmbedder::batch_embed(const std::vheaders["Content-Type"] = "application/json";std::map<std::string, std::string> res_headers;std::string res;- HttpClient& client = HttpClient::get_instance();-- auto res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);+ auto res_code = call_remote_api("POST", OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);if(res_code != 200) {nlohmann::json json_res = nlohmann::json::parse(res);@@ -159,7 +170,6 @@ Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config,return Option<bool>(400, "Property `embed.model_config.model_name` is not a supported Google model.");}- HttpClient& client = HttpClient::get_instance();std::unordered_map<std::string, std::string> headers;std::map<std::string, std::string> res_headers;headers["Content-Type"] = "application/json";@@ -167,7 +177,7 @@ Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config,nlohmann::json req_body;req_body["text"] = "test";- auto res_code = client.post_response(std::string(GOOGLE_CREATE_EMBEDDING) + api_key, req_body.dump(), res, res_headers, headers);+ auto res_code = call_remote_api("POST", std::string(GOOGLE_CREATE_EMBEDDING) + api_key, req_body.dump(), res, res_headers, headers);if(res_code != 200) {nlohmann::json json_res = nlohmann::json::parse(res);@@ -183,7 +193,6 @@ Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config,}Option<std::vector<float>> GoogleEmbedder::Embed(const std::string& text) {- HttpClient& client = HttpClient::get_instance();std::unordered_map<std::string, std::string> headers;std::map<std::string, std::string> res_headers;headers["Content-Type"] = "application/json";@@ -191,7 +200,7 @@ Option<std::vector<float>> GoogleEmbedder::Embed(const std::string& text) {nlohmann::json req_body;req_body["text"] = text;- auto res_code = client.post_response(std::string(GOOGLE_CREATE_EMBEDDING) + google_api_key, req_body.dump(), res, res_headers, headers);+ auto res_code = call_remote_api("POST", std::string(GOOGLE_CREATE_EMBEDDING) + google_api_key, req_body.dump(), res, res_headers, headers);if(res_code != 200) {nlohmann::json json_res = nlohmann::json::parse(res);@@ -246,7 +255,6 @@ Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, unsauto model_name_without_namespace = TextEmbedderManager::get_model_name_without_namespace(model_name);- HttpClient& client = HttpClient::get_instance();std::unordered_map<std::string, std::string> headers;std::map<std::string, std::string> res_headers;headers["Content-Type"] = "application/json";@@ -258,7 +266,7 @@ Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, unsinstance["content"] = "typesense";req_body["instances"].push_back(instance);- auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name_without_namespace), req_body.dump(), res, res_headers, headers);+ auto res_code = call_remote_api("POST", get_gcp_embedding_url(project_id, model_name_without_namespace), req_body.dump(), res, res_headers, headers);if(res_code != 200) {nlohmann::json json_res = nlohmann::json::parse(res);@@ -295,9 +303,8 @@ Option<std::vector<float>> GCPEmbedder::Embed(const std::string& text) {headers["Content-Type"] = "application/json";std::map<std::string, std::string> res_headers;std::string res;- HttpClient& client = HttpClient::get_instance();- auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);+ auto res_code = call_remote_api("POST", get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);if(res_code != 200) {if(res_code == 401) {@@ -308,7 +315,7 @@ Option<std::vector<float>> GCPEmbedder::Embed(const std::string& text) {access_token = refresh_op.get();// retryheaders["Authorization"] = "Bearer " + access_token;- res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);+ res_code = call_remote_api("POST", get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);}}@@ -353,8 +360,7 @@ Option<std::vector<std::vector<float>>> GCPEmbedder::batch_embed(const std::vectheaders["Content-Type"] = "application/json";std::map<std::string, std::string> res_headers;std::string res;- HttpClient& client = HttpClient::get_instance();- auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);+ auto res_code = call_remote_api("POST", get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);if(res_code != 200) {if(res_code == 401) {auto refresh_op = generate_access_token(refresh_token, client_id, client_secret);@@ -364,7 +370,7 @@ Option<std::vector<std::vector<float>>> GCPEmbedder::batch_embed(const std::vectaccess_token = refresh_op.get();// retryheaders["Authorization"] = "Bearer " + access_token;- res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);+ res_code = call_remote_api("POST", get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);}}@@ -386,7 +392,6 @@ Option<std::vector<std::vector<float>>> GCPEmbedder::batch_embed(const std::vect}Option<std::string> GCPEmbedder::generate_access_token(const std::string& refresh_token, const std::string& client_id, const std::string& client_secret) {- HttpClient& client = HttpClient::get_instance();std::unordered_map<std::string, std::string> headers;headers["Content-Type"] = "application/x-www-form-urlencoded";std::map<std::string, std::string> res_headers;@@ -394,7 +399,7 @@ Option<std::string> GCPEmbedder::generate_access_token(const std::string& refresstd::string req_body;req_body = "grant_type=refresh_token&client_id=" + client_id + "&client_secret=" + client_secret + "&refresh_token=" + refresh_token;- auto res_code = client.post_response(GCP_AUTH_TOKEN_URL, req_body, res, res_headers, headers);+ auto res_code = call_remote_api("POST", GCP_AUTH_TOKEN_URL, req_body, res, res_headers, headers);if(res_code != 200) {nlohmann::json json_res = nlohmann::json::parse(res);diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 46f66ab0..8d7acad7 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -453,6 +453,8 @@ int run_server(const Config & config, const std::string & version, void (*masterAnalyticsManager::get_instance().run(&replication_state);});+ RemoteEmbedder::set_raft_server(&replication_state);+std::string path_to_nodes = config.get_nodes();start_raft_server(replication_state, state_dir, path_to_nodes,config.get_peering_address(),