Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Proxy remote embedding API calls from leader node

ozanarmagana29c76ff54a5
diff --git a/include/core_api.h b/include/core_api.h
index 9d31d65a..cd8cdff8 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -167,3 +167,6 @@ bool is_doc_del_route(uint64_t route_hash);
Option<std::pair<std::string,std::string>> get_api_key_and_ip(const std::string& metadata);
void init_api(uint32_t cache_num_entries);
+
+
+bool proxy_embedding(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
diff --git a/include/http_proxy.h b/include/http_proxy.h
new file mode 100644
index 00000000..e116f0af
--- /dev/null
+++ b/include/http_proxy.h
@@ -0,0 +1,43 @@
+#pragma once
+
+#include <unordered_map>
+#include <string>
+#include "http_client.h"
+#include "lru/lru.hpp"
+
+
+struct http_proxy_res_t {
+ std::string body;
+ std::map<std::string, std::string> headers;
+ long status_code;
+
+ bool operator==(const http_proxy_res_t& other) const {
+ return body == other.body && headers == other.headers && status_code == other.status_code;
+ }
+
+ bool operator!=(const http_proxy_res_t& other) const {
+ return !(*this == other);
+ }
+};
+
+
+class HttpProxy {
+ // singleton class for http proxy
+ public:
+ static HttpProxy& get_instance() {
+ static HttpProxy instance;
+ return instance;
+ }
+ HttpProxy(const HttpProxy&) = delete;
+ void operator=(const HttpProxy&) = delete;
+ HttpProxy(HttpProxy&&) = delete;
+ void operator=(HttpProxy&&) = delete;
+ http_proxy_res_t send(const std::string& url, const std::string& method, const std::string& body, const std::unordered_map<std::string, std::string>& headers);
+ private:
+ HttpProxy();
+ ~HttpProxy() = default;
+
+
+ // lru cache for http requests
+ LRU::TimedCache<std::string, http_proxy_res_t> cache;
+};
\ No newline at end of file
diff --git a/include/text_embedder_remote.h b/include/text_embedder_remote.h
index 095272c2..131973d6 100644
--- a/include/text_embedder_remote.h
+++ b/include/text_embedder_remote.h
@@ -4,6 +4,7 @@
#include <string>
#include <mutex>
#include "http_client.h"
+#include "raft_server.h"
#include "option.h"
@@ -12,9 +13,15 @@
class RemoteEmbedder {
protected:
static Option<bool> validate_string_properties(const nlohmann::json& model_config, const std::vector<std::string>& properties);
+ static long call_remote_api(const std::string& method, const std::string& url, const std::string& body, std::string& res_body, std::map<std::string, std::string>& headers, const std::unordered_map<std::string, std::string>& req_headers);
+ static inline ReplicationState* raft_server = nullptr;
public:
virtual Option<std::vector<float>> Embed(const std::string& text) = 0;
virtual Option<std::vector<std::vector<float>>> batch_embed(const std::vector<std::string>& inputs) = 0;
+ static void set_raft_server(ReplicationState* rs) {
+ raft_server = rs;
+ }
+
};
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 6b9534e9..b97a2c62 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -15,6 +15,7 @@
#include "lru/lru.hpp"
#include "ratelimit_manager.h"
#include "event_manager.h"
+#include "http_proxy.h"
using namespace std::chrono_literals;
@@ -2136,3 +2137,44 @@ bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared
res->set_200(R"({"ok": true)");
return true;
}
+
+
+bool proxy_embedding(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ HttpProxy& proxy = HttpProxy::get_instance();
+
+ nlohmann::json req_json;
+
+ try {
+ req_json = nlohmann::json::parse(req->body);
+ } catch(const nlohmann::json::parse_error& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ std::string body, url, method;
+ std::unordered_map<std::string, std::string> headers;
+
+ try {
+ body = req_json["body"].get<std::string>();
+ url = req_json["url"].get<std::string>();
+ method = req_json["method"].get<std::string>();
+ for(auto& header: req_json["headers"].items()) {
+ headers[header.key()] = header.value().get<std::string>();
+ }
+ } catch(const std::exception& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ auto response = proxy.send(url, method, body, headers);
+
+ if(response.status_code != 200) {
+ res->set(response.status_code, response.body);
+ return false;
+ }
+
+ res->set_200(response.body);
+ return true;
+}
\ No newline at end of file
diff --git a/src/http_proxy.cpp b/src/http_proxy.cpp
new file mode 100644
index 00000000..19e8b61c
--- /dev/null
+++ b/src/http_proxy.cpp
@@ -0,0 +1,45 @@
+#include "http_proxy.h"
+#include "logger.h"
+#include <chrono>
+
+using namespace std::chrono_literals;
+
+HttpProxy::HttpProxy() : cache(30s){
+}
+
+
+http_proxy_res_t HttpProxy::send(const std::string& url, const std::string& method, const std::string& body, const std::unordered_map<std::string, std::string>& headers) {
+ // check if url is in cache
+ std::string key;
+ key += url;
+ key += method;
+ key += body;
+ for (auto& header : headers) {
+ key += header.first;
+ key += header.second;
+ }
+ if (cache.contains(key)) {
+ return cache[key];
+ }
+ // if not, make http request
+ HttpClient& client = HttpClient::get_instance();
+ http_proxy_res_t res;
+
+ if(method == "GET") {
+ res.status_code = client.get_response(url, res.body, res.headers, headers);
+ } else if(method == "POST") {
+ res.status_code = client.post_response(url, body, res.body, res.headers, headers);
+ } else if(method == "PUT") {
+ res.status_code = client.put_response(url, body, res.body, res.headers);
+ } else if(method == "DELETE") {
+ res.status_code = client.delete_response(url, res.body, res.headers);
+ } else {
+ res.status_code = 400;
+ res.body = "Invalid method: " + method;
+ }
+
+ // add to cache
+ cache.insert(key, res);
+
+ return res;
+}
\ No newline at end of file
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 30b7281d..f57f15c4 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -98,6 +98,9 @@ void master_server_routes() {
server->del("/limits/active/:id", del_throttle);
server->del("/limits/exceeds/:id", del_exceed);
server->post("/config", post_config, false, false);
+
+ // for proxying remote embedders
+ server->post("/proxy", proxy_embedding);
}
void (*backward::SignalHandling::_callback)(int sig, backward::StackTrace&) = nullptr;
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 920b6b34..d5912fd1 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -219,6 +219,15 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const std
return write_to_leader(request, response);
}
+ route_path* rpath;
+ bool found = server->get_route(request->route_hash, &rpath);
+ found = found && rpath->handler == proxy_embedding;
+ if(found && !node->is_leader()) {
+ response->set_500("Proxy embedding is only supported on leader.");
+ auto req_res = new async_req_res_t(request, response, true);
+ return message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);
+ }
+
// Serialize request to replicated WAL so that all the nodes in the group receive it as well.
// NOTE: actual write must be done only on the `on_apply` method to maintain consistency.
diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cpp
index 604e1071..cb0fe240 100644
--- a/src/text_embedder_remote.cpp
+++ b/src/text_embedder_remote.cpp
@@ -11,6 +11,21 @@ Option<bool> RemoteEmbedder::validate_string_properties(const nlohmann::json& mo
return Option<bool>(true);
}
+long RemoteEmbedder::call_remote_api(const std::string& method, const std::string& url, const std::string& body, std::string& res_body,
+ std::map<std::string, std::string>& headers, const std::unordered_map<std::string, std::string>& req_headers) {
+ auto leader_url = raft_server->get_leader_url();
+ leader_url += "proxy";
+ nlohmann::json req_body;
+ req_body["method"] = method;
+ req_body["url"] = url;
+ req_body["body"] = body;
+ req_body["headers"] = nlohmann::json::object();
+ for(auto& header : req_headers) {
+ req_body["headers"][header.first] = header.second;
+ }
+ return HttpClient::get_instance().post_response(leader_url, req_body.dump(), res_body, headers, {}, 10000, true);
+}
+
OpenAIEmbedder::OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key) : api_key(api_key), openai_model_path(openai_model_path) {
@@ -32,12 +47,11 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,
return Option<bool>(400, "Property `embed.model_config.model_name` malformed.");
}
- HttpClient& client = HttpClient::get_instance();
std::unordered_map<std::string, std::string> headers;
std::map<std::string, std::string> res_headers;
headers["Authorization"] = "Bearer " + api_key;
std::string res;
- auto res_code = client.get_response(OPENAI_LIST_MODELS, res, res_headers, headers);
+ auto res_code = call_remote_api("GET", OPENAI_LIST_MODELS, "", res, res_headers, headers);
if (res_code != 200) {
nlohmann::json json_res = nlohmann::json::parse(res);
if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
@@ -67,7 +81,7 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,
std::string embedding_res;
headers["Content-Type"] = "application/json";
- res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), embedding_res, res_headers, headers);
+ res_code = call_remote_api("POST", OPENAI_CREATE_EMBEDDING, req_body.dump(), embedding_res, res_headers, headers);
if (res_code != 200) {
@@ -84,7 +98,6 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,
}
Option<std::vector<float>> OpenAIEmbedder::Embed(const std::string& text) {
- HttpClient& client = HttpClient::get_instance();
std::unordered_map<std::string, std::string> headers;
std::map<std::string, std::string> res_headers;
headers["Authorization"] = "Bearer " + api_key;
@@ -94,7 +107,7 @@ Option<std::vector<float>> OpenAIEmbedder::Embed(const std::string& text) {
req_body["input"] = text;
// remove "openai/" prefix
req_body["model"] = TextEmbedderManager::get_model_name_without_namespace(openai_model_path);
- auto res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);
+ auto res_code = call_remote_api("POST", OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);
if (res_code != 200) {
nlohmann::json json_res = nlohmann::json::parse(res);
if(json_res.count("error") == 0 || json_res["error"].count("message") == 0) {
@@ -115,9 +128,7 @@ Option<std::vector<std::vector<float>>> OpenAIEmbedder::batch_embed(const std::v
headers["Content-Type"] = "application/json";
std::map<std::string, std::string> res_headers;
std::string res;
- HttpClient& client = HttpClient::get_instance();
-
- auto res_code = client.post_response(OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);
+ auto res_code = call_remote_api("POST", OPENAI_CREATE_EMBEDDING, req_body.dump(), res, res_headers, headers);
if(res_code != 200) {
nlohmann::json json_res = nlohmann::json::parse(res);
@@ -159,7 +170,6 @@ Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config,
return Option<bool>(400, "Property `embed.model_config.model_name` is not a supported Google model.");
}
- HttpClient& client = HttpClient::get_instance();
std::unordered_map<std::string, std::string> headers;
std::map<std::string, std::string> res_headers;
headers["Content-Type"] = "application/json";
@@ -167,7 +177,7 @@ Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config,
nlohmann::json req_body;
req_body["text"] = "test";
- auto res_code = client.post_response(std::string(GOOGLE_CREATE_EMBEDDING) + api_key, req_body.dump(), res, res_headers, headers);
+ auto res_code = call_remote_api("POST", std::string(GOOGLE_CREATE_EMBEDDING) + api_key, req_body.dump(), res, res_headers, headers);
if(res_code != 200) {
nlohmann::json json_res = nlohmann::json::parse(res);
@@ -183,7 +193,6 @@ Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config,
}
Option<std::vector<float>> GoogleEmbedder::Embed(const std::string& text) {
- HttpClient& client = HttpClient::get_instance();
std::unordered_map<std::string, std::string> headers;
std::map<std::string, std::string> res_headers;
headers["Content-Type"] = "application/json";
@@ -191,7 +200,7 @@ Option<std::vector<float>> GoogleEmbedder::Embed(const std::string& text) {
nlohmann::json req_body;
req_body["text"] = text;
- auto res_code = client.post_response(std::string(GOOGLE_CREATE_EMBEDDING) + google_api_key, req_body.dump(), res, res_headers, headers);
+ auto res_code = call_remote_api("POST", std::string(GOOGLE_CREATE_EMBEDDING) + google_api_key, req_body.dump(), res, res_headers, headers);
if(res_code != 200) {
nlohmann::json json_res = nlohmann::json::parse(res);
@@ -246,7 +255,6 @@ Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, uns
auto model_name_without_namespace = TextEmbedderManager::get_model_name_without_namespace(model_name);
- HttpClient& client = HttpClient::get_instance();
std::unordered_map<std::string, std::string> headers;
std::map<std::string, std::string> res_headers;
headers["Content-Type"] = "application/json";
@@ -258,7 +266,7 @@ Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, uns
instance["content"] = "typesense";
req_body["instances"].push_back(instance);
- auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name_without_namespace), req_body.dump(), res, res_headers, headers);
+ auto res_code = call_remote_api("POST", get_gcp_embedding_url(project_id, model_name_without_namespace), req_body.dump(), res, res_headers, headers);
if(res_code != 200) {
nlohmann::json json_res = nlohmann::json::parse(res);
@@ -295,9 +303,8 @@ Option<std::vector<float>> GCPEmbedder::Embed(const std::string& text) {
headers["Content-Type"] = "application/json";
std::map<std::string, std::string> res_headers;
std::string res;
- HttpClient& client = HttpClient::get_instance();
- auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
+ auto res_code = call_remote_api("POST", get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
if(res_code != 200) {
if(res_code == 401) {
@@ -308,7 +315,7 @@ Option<std::vector<float>> GCPEmbedder::Embed(const std::string& text) {
access_token = refresh_op.get();
// retry
headers["Authorization"] = "Bearer " + access_token;
- res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
+ res_code = call_remote_api("POST", get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
}
}
@@ -353,8 +360,7 @@ Option<std::vector<std::vector<float>>> GCPEmbedder::batch_embed(const std::vect
headers["Content-Type"] = "application/json";
std::map<std::string, std::string> res_headers;
std::string res;
- HttpClient& client = HttpClient::get_instance();
- auto res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
+ auto res_code = call_remote_api("POST", get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
if(res_code != 200) {
if(res_code == 401) {
auto refresh_op = generate_access_token(refresh_token, client_id, client_secret);
@@ -364,7 +370,7 @@ Option<std::vector<std::vector<float>>> GCPEmbedder::batch_embed(const std::vect
access_token = refresh_op.get();
// retry
headers["Authorization"] = "Bearer " + access_token;
- res_code = client.post_response(get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
+ res_code = call_remote_api("POST", get_gcp_embedding_url(project_id, model_name), req_body.dump(), res, res_headers, headers);
}
}
@@ -386,7 +392,6 @@ Option<std::vector<std::vector<float>>> GCPEmbedder::batch_embed(const std::vect
}
Option<std::string> GCPEmbedder::generate_access_token(const std::string& refresh_token, const std::string& client_id, const std::string& client_secret) {
- HttpClient& client = HttpClient::get_instance();
std::unordered_map<std::string, std::string> headers;
headers["Content-Type"] = "application/x-www-form-urlencoded";
std::map<std::string, std::string> res_headers;
@@ -394,7 +399,7 @@ Option<std::string> GCPEmbedder::generate_access_token(const std::string& refres
std::string req_body;
req_body = "grant_type=refresh_token&client_id=" + client_id + "&client_secret=" + client_secret + "&refresh_token=" + refresh_token;
- auto res_code = client.post_response(GCP_AUTH_TOKEN_URL, req_body, res, res_headers, headers);
+ auto res_code = call_remote_api("POST", GCP_AUTH_TOKEN_URL, req_body, res, res_headers, headers);
if(res_code != 200) {
nlohmann::json json_res = nlohmann::json::parse(res);
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 46f66ab0..8d7acad7 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -453,6 +453,8 @@ int run_server(const Config & config, const std::string & version, void (*master
AnalyticsManager::get_instance().run(&replication_state);
});
+ RemoteEmbedder::set_raft_server(&replication_state);
+
std::string path_to_nodes = config.get_nodes();
start_raft_server(replication_state, state_dir, path_to_nodes,
config.get_peering_address(),