Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Throttle imports.
kishorenc80a3e0fb8778
diff --git a/include/http_data.h b/include/http_data.hindex f50099d9..101d0e26 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -14,6 +14,17 @@ extern "C" {#include "h2o.h"}+struct h2o_custom_timer_t {+ h2o_timer_t timer{};+ void *data;++ h2o_custom_timer_t(): data(nullptr) {}++ explicit h2o_custom_timer_t(void *data): data(data) {++ }+};+struct http_res {uint32_t status_code;std::string content_type_header;@@ -120,21 +131,26 @@ struct http_req {std::map<std::string, std::string> params;std::string stream_state;-size_t chunk_length;+std::string body;+ size_t body_index;std::string metadata;void* data;- http_req(): _req(nullptr), route_hash(1), chunk_length(0), data(nullptr) {+ // for deffered processing of async handlers+ h2o_custom_timer_t defer_timer;++ http_req(): _req(nullptr), route_hash(1), chunk_length(0), body_index(0), data(nullptr) {}http_req(h2o_req_t* _req, const std::string & http_method, uint64_t route_hash,const std::map<std::string, std::string> & params, const std::string& body):_req(_req), http_method(http_method), route_hash(route_hash), params(params),- stream_state("NON_STREAMING"), chunk_length(0), body(body), data(nullptr) {+ stream_state("NON_STREAMING"), chunk_length(0), body(body), body_index(0),+ data(nullptr) {}diff --git a/include/http_server.h b/include/http_server.hindex aedbbf34..574297df 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -18,17 +18,6 @@ extern "C" {class ReplicationState;class HttpServer;-struct h2o_custom_timer_t {- h2o_timer_t timer{};- HttpServer *server;-- h2o_custom_timer_t(): server(nullptr) {}-- explicit h2o_custom_timer_t(HttpServer *server): server(server) {-- }-};-struct h2o_custom_req_handler_t {h2o_handler_t super;HttpServer* http_server;@@ -42,6 +31,12 @@ struct h2o_custom_generator_t {http_res* response;};+struct deferred_req_res_t {+ http_req* req;+ http_res* res;+ HttpServer* server;+};+class HttpServer {private:h2o_globalconf_t config;@@ -154,4 +149,8 @@ public:static int process_request(http_req* request, http_res* response, route_path *rpath,const h2o_custom_req_handler_t *req_handler);++ static void on_deferred_process_request(h2o_timer_t *entry);++ void defer_processing(http_req& req, http_res& res, size_t timeout_ms);};\ No newline at end of filediff --git a/include/string_utils.h b/include/string_utils.hindex 1b12da8b..7f131904 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -26,25 +26,39 @@ struct StringUtils {}// Adapted from: http://stackoverflow.com/a/236180/131050- static void split(const std::string& s, std::vector<std::string> & result, const std::string& delim, const bool keep_empty = false) {+ static size_t split(const std::string& s, std::vector<std::string> & result, const std::string& delim,+ const bool keep_empty = false, const size_t start_index = 0,+ const size_t max_values = (std::numeric_limits<size_t>::max()-1)) {if (delim.empty()) {result.push_back(s);- return ;+ return s.size();}- std::string::const_iterator substart = s.begin(), subend;++ std::string::const_iterator substart = s.begin()+start_index, subend;+ size_t end_index = start_index;+while (true) {subend = std::search(substart, s.end(), delim.begin(), delim.end());std::string temp(substart, subend);++ end_index += temp.size() + delim.size();temp = trim(temp);if (keep_empty || !temp.empty()) {result.push_back(temp);}++ if(result.size() == max_values) {+ break;+ }+if (subend == s.end()) {break;}substart = subend + delim.size();}++ return std::min(end_index, s.size());}static std::string join(std::vector<std::string> vec, const std::string& delimiter, size_t start_index = 0) {diff --git a/src/collection.cpp b/src/collection.cppindex efedf3a3..a1219c93 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -186,7 +186,7 @@ Option<nlohmann::json> Collection::add(const std::string & json_str) {}nlohmann::json Collection::add_many(std::vector<std::string>& json_lines) {- LOG(INFO) << "Memory ratio. Max = " << max_memory_ratio << ", Used = " << SystemMetrics::used_memory_ratio();+ //LOG(INFO) << "Memory ratio. Max = " << max_memory_ratio << ", Used = " << SystemMetrics::used_memory_ratio();std::vector<std::vector<index_record>> iter_batch;diff --git a/src/core_api.cpp b/src/core_api.cppindex 60db7149..aead9d48 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -558,6 +558,28 @@ bool get_export_documents(http_req & req, http_res & res) {}bool post_import_documents(http_req& req, http_res& res) {+ const char *BATCH_SIZE = "batch_size";++ if(req.stream_state == "DISPOSE") {+ // we don't cache any state across iterations, so just return+ return true;+ }++ if(req.params.count(BATCH_SIZE) == 0) {+ req.params[BATCH_SIZE] = "40";+ }++ if(!StringUtils::is_uint32_t(req.params[BATCH_SIZE])) {+ req.stream_state = "NON_STREAMING";+ res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be an unsigned integer.");+ HttpServer::stream_response(req, res);+ return false;+ }++ const size_t IMPORT_BATCH_SIZE = std::stoi(req.params[BATCH_SIZE]);++ LOG(INFO) << "Import batch size: " << IMPORT_BATCH_SIZE;+CollectionManager & collectionManager = CollectionManager::get_instance();Collection* collection = collectionManager.get_collection(req.params["collection"]);@@ -568,55 +590,67 @@ bool post_import_documents(http_req& req, http_res& res) {return false;}- LOG(INFO) << "Import, req.body.size: " << req.body.size() << ", state: " << req.stream_state;-- if(req.stream_state == "DISPOSE") {- // we don't cache any state across iterations, so just return- return true;- }+ //LOG(INFO) << "Import, req.body.size: " << req.body.size() << ", state: " << req.stream_state;std::vector<std::string> json_lines;- StringUtils::split(req.body, json_lines, "\n");+ req.body_index = StringUtils::split(req.body, json_lines, "\n", false, req.body_index, IMPORT_BATCH_SIZE);- if(req.stream_state == "END" || req.stream_state == "NON_STREAMING") {- req.body = "";- } else {- if(!json_lines.empty()) {- // check if req.body had incomplete last record- if(!nlohmann::json::accept(json_lines.back())) {- // eject partial record- req.body = json_lines.back();- json_lines.pop_back();+ bool stream_proceed = false; // default state++ //LOG(INFO) << "req.body_index=" << req.body_index << ", req.body.size=" << req.body.size();+ //LOG(INFO) << "req body percentage: " << (float(req.body_index)/req.body.size())*100;++ if(req.body_index == req.body.size()) {+ // body has been consumed fully, see whether we can fetch more request body+ req.body_index = 0;+ stream_proceed = true;++ if(req.stream_state == "END" || req.stream_state == "NON_STREAMING") {+ req.body = "";+ } else {+ if(!json_lines.empty()) {+ // check if req.body had incomplete last record+ if(!nlohmann::json::accept(json_lines.back())) {+ // eject partial record+ req.body = json_lines.back();+ json_lines.pop_back();+ }}}}//LOG(INFO) << "json_lines.size: " << json_lines.size() << ", req.stream_state: " << req.stream_state;- if(json_lines.empty() && !req.body.empty()) {- // detects the case when only one partial record arrives as a chunk- return true;- }-- nlohmann::json json_res = collection->add_many(json_lines);-+ // When only one partial record arrives as a chunk, an empty body is pushed to response stream+ bool single_partial_record_body = (json_lines.empty() && !req.body.empty());std::stringstream ss;- const std::string& import_summary_json = json_res.dump();-- ss << import_summary_json << "\n";- for (size_t i = 0; i < json_lines.size(); i++) {- if(i == json_lines.size()-1) {- ss << json_lines[i];- } else {- ss << json_lines[i] << "\n";+ if(!single_partial_record_body) {+ nlohmann::json json_res = collection->add_many(json_lines);+ //const std::string& import_summary_json = json_res.dump();+ //ss << import_summary_json << "\n";++ for (size_t i = 0; i < json_lines.size(); i++) {+ if(i == json_lines.size()-1 && req.stream_state == "END" && req.body_index == req.body.size()) {+ // indicates last record of last batch+ ss << json_lines[i];+ } else {+ ss << json_lines[i] << "\n";+ }}}res.content_type_header = "text/plain; charset=utf8";- res.set_200(ss.str());+ res.status_code = 200;+ res.body += ss.str();++ if(stream_proceed) {+ HttpServer::stream_response(req, res);+ } else {+ // push handler back onto the event loop: we must process the next batch without blocking the event loop+ server->defer_processing(req, res, 1);+ }- HttpServer::stream_response(req, res);return true;}@@ -1029,6 +1063,12 @@ bool async_write_request(void *data) {server->send_response(index_arg->req, index_arg->res);}+ if(index_arg->req->_req == nullptr) {+ // indicates raft serialized request and response -- lifecycle must be managed here+ delete index_arg->req;+ delete index_arg->res;+ }+if(index_arg->promise != nullptr) {index_arg->promise->set_value(true); // returns control back to caller}diff --git a/src/http_server.cpp b/src/http_server.cppindex 74da4656..ef55e4f7 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -51,7 +51,7 @@ void HttpServer::on_ssl_refresh_timeout(h2o_timer_t *entry) {LOG(INFO) << "Refreshing SSL certs from disk.";- HttpServer *hs = custom_timer->server;+ HttpServer *hs = static_cast<HttpServer*>(custom_timer->data);SSL_CTX *ssl_ctx = hs->accept_ctx->ssl_ctx;if (SSL_CTX_use_certificate_chain_file(ssl_ctx, hs->ssl_cert_path.c_str()) != 1) {@@ -71,9 +71,9 @@ void HttpServer::on_ssl_refresh_timeout(h2o_timer_t *entry) {int HttpServer::setup_ssl(const char *cert_file, const char *key_file) {// Set up a timer to refresh SSL config from disk. Also, initializing upfront so that destructor worksssl_refresh_timer = h2o_custom_timer_t(this);- ssl_refresh_timer.timer.expire_at = 10000000;- h2o_timer_init(&ssl_refresh_timer.timer, on_ssl_refresh_timeout); // every 8 hours- h2o_timer_link(ctx.loop, SSL_REFRESH_INTERVAL_MS, &ssl_refresh_timer.timer);+ ssl_refresh_timer.timer.expire_at = UINT64_MAX-1;+ h2o_timer_init(&ssl_refresh_timer.timer, on_ssl_refresh_timeout);+ h2o_timer_link(ctx.loop, SSL_REFRESH_INTERVAL_MS, &ssl_refresh_timer.timer); // every 8 hoursaccept_ctx->ssl_ctx = SSL_CTX_new(SSLv23_server_method());@@ -269,7 +269,7 @@ uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, con}void HttpServer::on_res_generator_dispose(void *self) {- LOG(INFO) << "on_res_generator_dispose fires";+ //LOG(INFO) << "on_res_generator_dispose fires";h2o_custom_generator_t* res_generator = static_cast<h2o_custom_generator_t*>(self);if(res_generator->rpath->async_res) {@@ -278,6 +278,11 @@ void HttpServer::on_res_generator_dispose(void *self) {res_generator->rpath->handler(*res_generator->request, *res_generator->response);}+ if(res_generator->request->defer_timer.data != nullptr) {+ deferred_req_res_t* deferred_req_res = static_cast<deferred_req_res_t*>(res_generator->request->defer_timer.data);+ delete deferred_req_res;+ }+// res_generator itself is reference counted, so we only delete the membersdelete res_generator->request;delete res_generator->response;@@ -489,6 +494,28 @@ int HttpServer::process_request(http_req* request, http_res* response, route_patreturn 0;}+void HttpServer::on_deferred_process_request(h2o_timer_t *entry) {+ h2o_custom_timer_t* custom_timer = reinterpret_cast<h2o_custom_timer_t*>(entry);+ deferred_req_res_t* deferred_req_res = static_cast<deferred_req_res_t*>(custom_timer->data);++ route_path* found_rpath = nullptr;+ deferred_req_res->server->get_route(deferred_req_res->req->route_hash, &found_rpath);+ if(found_rpath) {+ found_rpath->handler(*deferred_req_res->req, *deferred_req_res->res);+ }+}++void HttpServer::defer_processing(http_req& req, http_res& res, size_t timeout_ms) {+ if(req.defer_timer.data == nullptr) {+ auto deferred_req_res = new deferred_req_res_t{&req, &res, this};+ req.defer_timer.data = deferred_req_res;+ req.defer_timer.timer.expire_at = UINT64_MAX-1;+ h2o_timer_init(&req.defer_timer.timer, on_deferred_process_request);+ }++ h2o_timer_link(ctx.loop, timeout_ms, &req.defer_timer.timer);+}+void HttpServer::send_response(http_req* request, http_res* response) {h2o_req_t* req = request->_req;h2o_generator_t& generator = *response->generator;@@ -543,7 +570,7 @@ void HttpServer::stream_response(http_req& request, http_res& response) {}h2o_req_t* req = request._req;- h2o_custom_generator_t *custom_generator = reinterpret_cast<h2o_custom_generator_t *>(response.generator);+ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(response.generator);if (request.stream_state == "START" || request.stream_state == "NON_STREAMING") {req->res.status = response.status_code;@@ -557,6 +584,8 @@ void HttpServer::stream_response(http_req& request, http_res& response) {custom_generator->response->final = (request.stream_state == "END" || request.stream_state == "NON_STREAMING");h2o_iovec_t body = h2o_strdup(&req->pool, response.body.c_str(), SIZE_MAX);+ response.body = "";+const h2o_send_state_t state = custom_generator->response->final ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;h2o_send(req, &body, 1, state);}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 3946a090..abff98cb 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -352,7 +352,7 @@ int run_server(const Config & config, const std::string & version, void (*master// first we start the peering service- ThreadPool thread_pool(4);+ ThreadPool thread_pool(32);ReplicationState replication_state(&store, &thread_pool, server->get_message_dispatcher(), create_init_db_snapshot);std::thread raft_thread([&replication_state, &config, &state_dir]() {diff --git a/test/string_utils_test.cpp b/test/string_utils_test.cppindex 9ca0ea5a..1ab808e6 100644--- a/test/string_utils_test.cpp+++ b/test/string_utils_test.cpp@@ -82,4 +82,16 @@ TEST(StringUtilsTest, ShouldSplitString) {std::vector<std::string> lines_empty;StringUtils::split("", lines_empty, "\n");ASSERT_TRUE(lines_empty.empty());++ // restrict list of max_values+ std::vector<std::string> lines_limited;+ size_t end_index = StringUtils::split("a b c d e f", lines_limited, " ", false, 0, 3);+ ASSERT_EQ(3, lines_limited.size());+ ASSERT_EQ(6, end_index);++ // start from an arbitrary position in string+ std::vector<std::string> lines_custom_start;+ end_index = StringUtils::split("a b c d e f", lines_custom_start, " ", false, 2, 100);+ ASSERT_EQ(5, lines_custom_start.size());+ ASSERT_EQ(11, end_index);}