Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add timeout and num retries as search params
ozanarmagan65dbeb6e75ea
diff --git a/include/collection.h b/include/collection.hindex 2f79bc68..f4c930f4 100644--- a/include/collection.h+++ b/include/collection.h@@ -464,7 +464,9 @@ public:const size_t facet_sample_percent = 100,const size_t facet_sample_threshold = 0,const size_t page_offset = UINT32_MAX,- const size_t vector_query_hits = 250) const;+ const size_t vector_query_hits = 250,+ const size_t remote_embedding_timeout_ms = 30000,+ const size_t remote_embedding_num_retries = 2) const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --git a/include/http_proxy.h b/include/http_proxy.hindex 851daa91..232576f6 100644--- a/include/http_proxy.h+++ b/include/http_proxy.h@@ -32,11 +32,11 @@ class HttpProxy {void operator=(const HttpProxy&) = delete;HttpProxy(HttpProxy&&) = delete;void operator=(HttpProxy&&) = delete;- http_proxy_res_t send(const std::string& url, const std::string& method, const std::string& body, const std::unordered_map<std::string, std::string>& headers);+ http_proxy_res_t send(const std::string& url, const std::string& method, const std::string& body, std::unordered_map<std::string, std::string>& headers);private:HttpProxy();~HttpProxy() = default;- http_proxy_res_t call(const std::string& url, const std::string& method, const std::string& body = "", const std::unordered_map<std::string, std::string>& headers = {});+ http_proxy_res_t call(const std::string& url, const std::string& method, const std::string& body = "", const std::unordered_map<std::string, std::string>& headers = {}, const size_t timeout_ms = 30000);// lru cache for http requestsdiff --git a/include/text_embedder.h b/include/text_embedder.hindex cff7c7c7..4d74292e 100644--- a/include/text_embedder.h+++ b/include/text_embedder.h@@ -16,7 +16,7 @@ class TextEmbedder {// Constructor for remote modelsTextEmbedder(const nlohmann::json& model_config);~TextEmbedder();- embedding_res_t Embed(const std::string& text);+ embedding_res_t Embed(const std::string& text, const size_t remote_embedding_timeout_ms = 30000, const int remote_embedding_num_tries = 2);std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs);const std::string& get_vocab_file_name() const;bool is_remote() {diff --git a/include/text_embedder_remote.h b/include/text_embedder_remote.hindex 652c0b61..3f9937c3 100644--- a/include/text_embedder_remote.h+++ b/include/text_embedder_remote.h@@ -28,7 +28,7 @@ class RemoteEmbedder {static long call_remote_api(const std::string& method, const std::string& url, const std::string& body, std::string& res_body, std::map<std::string, std::string>& headers, const std::unordered_map<std::string, std::string>& req_headers);static inline ReplicationState* raft_server = nullptr;public:- virtual embedding_res_t Embed(const std::string& text) = 0;+ virtual embedding_res_t Embed(const std::string& text, const size_t remote_embedding_timeout_ms = 30000, const int remote_embedding_num_tries = 2) = 0;virtual std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs) = 0;static void init(ReplicationState* rs) {raft_server = rs;@@ -47,7 +47,7 @@ class OpenAIEmbedder : public RemoteEmbedder {public:OpenAIEmbedder(const std::string& openai_model_path, const std::string& api_key);static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);- embedding_res_t Embed(const std::string& text) override;+ embedding_res_t Embed(const std::string& text, const size_t remote_embedding_timeout_ms = 30000, const int remote_embedding_num_tries = 2) override;std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs) override;};@@ -62,7 +62,7 @@ class GoogleEmbedder : public RemoteEmbedder {public:GoogleEmbedder(const std::string& google_api_key);static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);- embedding_res_t Embed(const std::string& text) override;+ embedding_res_t Embed(const std::string& text, const size_t remote_embedding_timeout_ms = 30000, const int remote_embedding_num_tries = 2) override;std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs) override;};@@ -88,7 +88,7 @@ class GCPEmbedder : public RemoteEmbedder {GCPEmbedder(const std::string& project_id, const std::string& model_name, const std::string& access_token,const std::string& refresh_token, const std::string& client_id, const std::string& client_secret);static Option<bool> is_model_valid(const nlohmann::json& model_config, unsigned int& num_dims);- embedding_res_t Embed(const std::string& text) override;+ embedding_res_t Embed(const std::string& text, const size_t remote_embedding_timeout_ms = 30000, const int remote_embedding_num_tries = 2) override;std::vector<embedding_res_t> batch_embed(const std::vector<std::string>& inputs) override;};diff --git a/src/collection.cpp b/src/collection.cppindex c55bee7d..8c193fc0 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1107,7 +1107,9 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const size_t facet_sample_percent,const size_t facet_sample_threshold,const size_t page_offset,- const size_t vector_query_hits) const {+ const size_t vector_query_hits,+ const size_t remote_embedding_timeout_ms,+ const size_t remote_embedding_num_retries) const {std::shared_lock lock(mutex);@@ -1236,7 +1238,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}std::string embed_query = embedder_manager.get_query_prefix(search_field.embed[fields::model_config]) + raw_query;- auto embedding_op = embedder->Embed(embed_query);+ auto embedding_op = embedder->Embed(embed_query, remote_embedding_timeout_ms, remote_embedding_num_retries);if(!embedding_op.success) {if(!embedding_op.error["error"].get<std::string>().empty()) {return Option<nlohmann::json>(400, embedding_op.error["error"].get<std::string>());diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 5323e1b0..d86ec2a1 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -671,6 +671,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *VECTOR_QUERY = "vector_query";const char *VECTOR_QUERY_HITS = "vector_query_hits";+ const char *REMOTE_EMBEDDING_TIMEOUT_MS = "remote_embedding_timeout_ms";+ const char *REMOTE_EMBEDDING_NUM_RETRIES = "remote_embedding_num_retries";+const char *GROUP_BY = "group_by";const char *GROUP_LIMIT = "group_limit";@@ -824,6 +827,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& retext_match_type_t match_type = max_score;size_t vector_query_hits = 250;+ size_t remote_embedding_timeout_ms = 30000;+ size_t remote_embedding_num_retries = 2;+size_t facet_sample_percent = 100;size_t facet_sample_threshold = 0;@@ -850,6 +856,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re{FACET_SAMPLE_PERCENT, &facet_sample_percent},{FACET_SAMPLE_THRESHOLD, &facet_sample_threshold},{VECTOR_QUERY_HITS, &vector_query_hits},+ {REMOTE_EMBEDDING_TIMEOUT_MS, &remote_embedding_timeout_ms},+ {REMOTE_EMBEDDING_NUM_RETRIES, &remote_embedding_num_retries}};std::unordered_map<std::string, std::string*> str_values = {@@ -1070,7 +1078,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rematch_type,facet_sample_percent,facet_sample_threshold,- offset+ offset,+ vector_query_hits,+ remote_embedding_timeout_ms,+ remote_embedding_num_retries);uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(diff --git a/src/http_proxy.cpp b/src/http_proxy.cppindex eb562849..d44ee06a 100644--- a/src/http_proxy.cpp+++ b/src/http_proxy.cpp@@ -8,17 +8,19 @@ HttpProxy::HttpProxy() : cache(30s){}-http_proxy_res_t HttpProxy::call(const std::string& url, const std::string& method, const std::string& body, const std::unordered_map<std::string, std::string>& headers) {+http_proxy_res_t HttpProxy::call(const std::string& url, const std::string& method, const std::string& body,+ const std::unordered_map<std::string, std::string>& headers, const size_t timeout_ms) {HttpClient& client = HttpClient::get_instance();http_proxy_res_t res;+if(method == "GET") {- res.status_code = client.get_response(url, res.body, res.headers, headers, 20 * 1000);+ res.status_code = client.get_response(url, res.body, res.headers, headers, timeout_ms);} else if(method == "POST") {- res.status_code = client.post_response(url, body, res.body, res.headers, headers, 20 * 1000);+ res.status_code = client.post_response(url, body, res.body, res.headers, headers, timeout_ms);} else if(method == "PUT") {- res.status_code = client.put_response(url, body, res.body, res.headers, 20 * 1000);+ res.status_code = client.put_response(url, body, res.body, res.headers, timeout_ms);} else if(method == "DELETE") {- res.status_code = client.delete_response(url, res.body, res.headers, 20 * 1000);+ res.status_code = client.delete_response(url, res.body, res.headers, timeout_ms);} else {res.status_code = 400;nlohmann::json j;@@ -29,7 +31,7 @@ http_proxy_res_t HttpProxy::call(const std::string& url, const std::string& meth}-http_proxy_res_t HttpProxy::send(const std::string& url, const std::string& method, const std::string& body, const std::unordered_map<std::string, std::string>& headers) {+http_proxy_res_t HttpProxy::send(const std::string& url, const std::string& method, const std::string& body, std::unordered_map<std::string, std::string>& headers) {// check if url is in cacheuint64_t key = StringUtils::hash_wy(url.c_str(), url.size());key = StringUtils::hash_combine(key, StringUtils::hash_wy(method.c_str(), method.size()));@@ -42,13 +44,34 @@ http_proxy_res_t HttpProxy::send(const std::string& url, const std::string& methreturn cache[key];}- auto res = call(url, method, body, headers);+ size_t timeout_ms = 30000;+ size_t num_retries = 2;- if(res.status_code == 500){- // retry- res = call(url, method, body, headers);+ if(headers.find("remote_embedding_timeout_ms") != headers.end()) {+ std::stringstream ss(headers["remote_embedding_timeout_ms"]);+ ss >> timeout_ms;++ headers.erase("remote_embedding_timeout_ms");}++ if(headers.find("remote_embedding_num_retries") != headers.end()) {+ std::stringstream ss(headers["remote_embedding_num_retries"]);+ ss >> num_retries;++ headers.erase("remote_embedding_num_retries");+ }++ http_proxy_res_t res;+ for(size_t i = 0;i < num_retries;i++) {+ res = call(url, method, body, headers, timeout_ms);++ if(res.status_code != 500) {+ break;+ }+ }++if(res.status_code == 500){nlohmann::json j;j["message"] = "Server error on remote server. Please try again later.";@@ -61,5 +84,6 @@ http_proxy_res_t HttpProxy::send(const std::string& url, const std::string& methcache.insert(key, res);}+return res;}\ No newline at end of filediff --git a/src/text_embedder.cpp b/src/text_embedder.cppindex b2a67607..c235ecf8 100644--- a/src/text_embedder.cpp+++ b/src/text_embedder.cpp@@ -83,9 +83,9 @@ std::vector<float> TextEmbedder::mean_pooling(const std::vector<std::vector<floareturn pooled_output;}-embedding_res_t TextEmbedder::Embed(const std::string& text) {+embedding_res_t TextEmbedder::Embed(const std::string& text, const size_t remote_embedding_timeout_ms, const int remote_embedding_num_tries) {if(is_remote()) {- return remote_embedder_->Embed(text);+ return remote_embedder_->Embed(text, remote_embedding_timeout_ms, remote_embedding_num_tries);} else {// Cannot run same model in parallel, so lock the mutexstd::lock_guard<std::mutex> lock(mutex_);diff --git a/src/text_embedder_remote.cpp b/src/text_embedder_remote.cppindex bb984e86..336c7a74 100644--- a/src/text_embedder_remote.cpp+++ b/src/text_embedder_remote.cpp@@ -104,11 +104,13 @@ Option<bool> OpenAIEmbedder::is_model_valid(const nlohmann::json& model_config,return Option<bool>(true);}-embedding_res_t OpenAIEmbedder::Embed(const std::string& text) {+embedding_res_t OpenAIEmbedder::Embed(const std::string& text, const size_t remote_embedding_timeout_ms, const int remote_embedding_num_tries) {std::unordered_map<std::string, std::string> headers;std::map<std::string, std::string> res_headers;headers["Authorization"] = "Bearer " + api_key;headers["Content-Type"] = "application/json";+ headers["remote_embedding_timeout_ms"] = std::to_string(remote_embedding_timeout_ms);+ headers["remote_embedding_num_tries"] = std::to_string(remote_embedding_num_tries);std::string res;nlohmann::json req_body;req_body["input"] = text;@@ -170,10 +172,12 @@ std::vector<embedding_res_t> OpenAIEmbedder::batch_embed(const std::vector<std::nlohmann::json res_json = nlohmann::json::parse(res);std::vector<embedding_res_t> outputs;+for(auto& data : res_json["data"]) {outputs.push_back(embedding_res_t(data["embedding"].get<std::vector<float>>()));}+return outputs;}@@ -222,10 +226,12 @@ Option<bool> GoogleEmbedder::is_model_valid(const nlohmann::json& model_config,return Option<bool>(true);}-embedding_res_t GoogleEmbedder::Embed(const std::string& text) {+embedding_res_t GoogleEmbedder::Embed(const std::string& text, const size_t remote_embedding_timeout_ms, const int remote_embedding_num_tries) {std::unordered_map<std::string, std::string> headers;std::map<std::string, std::string> res_headers;headers["Content-Type"] = "application/json";+ headers["remote_embedding_timeout_ms"] = std::to_string(remote_embedding_timeout_ms);+ headers["remote_embedding_num_tries"] = std::to_string(remote_embedding_num_tries);std::string res;nlohmann::json req_body;req_body["text"] = text;@@ -325,7 +331,7 @@ Option<bool> GCPEmbedder::is_model_valid(const nlohmann::json& model_config, unsreturn Option<bool>(true);}-embedding_res_t GCPEmbedder::Embed(const std::string& text) {+embedding_res_t GCPEmbedder::Embed(const std::string& text, const size_t remote_embedding_timeout_ms, const int remote_embedding_num_tries) {nlohmann::json req_body;req_body["instances"] = nlohmann::json::array();nlohmann::json instance;@@ -334,6 +340,8 @@ embedding_res_t GCPEmbedder::Embed(const std::string& text) {std::unordered_map<std::string, std::string> headers;headers["Authorization"] = "Bearer " + access_token;headers["Content-Type"] = "application/json";+ headers["remote_embedding_timeout_ms"] = std::to_string(remote_embedding_timeout_ms);+ headers["remote_embedding_num_tries"] = std::to_string(remote_embedding_num_tries);std::map<std::string, std::string> res_headers;std::string res;