Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Implemented import streaming end-to-end on follower.
kishorenc6d7c6b2ea0c6
diff --git a/include/http_client.h b/include/http_client.hindex cda44f6c..a777bfec 100644--- a/include/http_client.h+++ b/include/http_client.h@@ -22,9 +22,11 @@ private:static size_t curl_write_async(char *buffer, size_t size, size_t nmemb, void* context);+ static size_t curl_write_async_done(void* context, curl_socket_t item);+static CURL* init_curl(const std::string& url, std::string& response);- static CURL* init_curl_async(const std::string& url, deferred_req_res_t* req_res);+ static CURL* init_curl_async(const std::string& url, request_response* req_res);static size_t curl_req_send_callback(char* buffer, size_t size, size_t nitems, void *userdata);diff --git a/include/http_data.h b/include/http_data.hindex 446e94a3..cead00a8 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -37,9 +37,9 @@ struct http_res {h2o_generator_t* generator = nullptr;// for async requests, automatically progresses request body on response proceed- bool async_request_proceed = true;+ bool proceed_req_after_write = true;- http_res(): status_code(501), content_type_header("application/json; charset=utf-8"), final(true) {+ http_res(): status_code(0), content_type_header("application/json; charset=utf-8"), final(true) {}@@ -147,14 +147,14 @@ struct http_req {void* data;// used during forwarding of requests from follower to leader- std::atomic<int> proxy_status;+ std::promise<bool>* promise = nullptr;// for deffered processing of async handlersh2o_custom_timer_t defer_timer;http_req(): _req(nullptr), route_hash(1),first_chunk_aggregate(true), last_chunk_aggregate(false),- chunk_len(0), body_index(0), data(nullptr), proxy_status(0) {+ chunk_len(0), body_index(0), data(nullptr), promise(nullptr) {}@@ -162,7 +162,7 @@ struct http_req {const std::map<std::string, std::string> & params, const std::string& body):_req(_req), http_method(http_method), route_hash(route_hash), params(params),first_chunk_aggregate(true), last_chunk_aggregate(false),- chunk_len(0), body(body), body_index(0), data(nullptr), proxy_status(0) {+ chunk_len(0), body(body), body_index(0), data(nullptr), promise(nullptr) {}@@ -332,7 +332,7 @@ struct http_message_dispatcher {}void send_message(const std::string & type, void* data) {- h2o_custom_res_message_t* message = new h2o_custom_res_message_t{{{NULL, NULL}}, &message_handlers, type.c_str(), data};+ h2o_custom_res_message_t* message = new h2o_custom_res_message_t{{{nullptr, nullptr}}, &message_handlers, type, data};h2o_multithread_send_message(message_receiver, &message->super);}diff --git a/include/http_server.h b/include/http_server.hindex 7622df15..471ccb05 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -91,6 +91,8 @@ private:static void response_proceed(h2o_generator_t *generator, h2o_req_t *req);+ static void response_abort(h2o_generator_t *generator, h2o_req_t *req);+static void on_res_generator_dispose(void *self);static int send_response(h2o_req_t *req, int status_code, const std::string & message);@@ -148,11 +150,14 @@ public:static bool on_stream_response_message(void *data);+ static bool on_request_proceed_message(void *data);+std::string get_version();static constexpr const char* AUTH_HEADER = "x-typesense-api-key";static constexpr const char* STOP_SERVER_MESSAGE = "STOP_SERVER";static constexpr const char* STREAM_RESPONSE_MESSAGE = "STREAM_RESPONSE";+ static constexpr const char* REQUEST_PROCEED_MESSAGE = "REQUEST_PROCEED";static int process_request(http_req* request, http_res* response, route_path *rpath,const h2o_custom_req_handler_t *req_handler);diff --git a/include/raft_server.h b/include/raft_server.hindex 50e20349..e2a8be01 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -214,4 +214,6 @@ private:void on_stop_following(const ::braft::LeaderChangeContext& ctx) {LOG(INFO) << "Node stops following " << ctx;}++ void follower_write(http_req *request, http_res *response) const;};diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 035b82f4..d102ffbe 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -38,7 +38,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collectioncollection_meta[COLLECTION_NUM_MEMORY_SHARDS].get<size_t>() :DEFAULT_NUM_MEMORY_SHARDS;- LOG(INFO) << "Loading collection with " << num_memory_shards << " memory shards.";+ LOG(INFO) << "Found collection " << this_collection_name << " with " << num_memory_shards << " memory shards.";Collection* collection = new Collection(this_collection_name,collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),diff --git a/src/core_api.cpp b/src/core_api.cppindex ca0ba7fe..0c3d3836 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -525,6 +525,7 @@ bool get_export_documents(http_req & req, http_res & res) {if(collection == nullptr) {req.last_chunk_aggregate = true;+ res.final = true;res.set_404();HttpServer::stream_response(req, res);return false;@@ -551,6 +552,7 @@ bool get_export_documents(http_req & req, http_res & res) {req.last_chunk_aggregate = false;} else {req.last_chunk_aggregate = true;+ res.final = true;delete it;req.data = nullptr;}@@ -574,6 +576,7 @@ bool post_import_documents(http_req& req, http_res& res) {if(!StringUtils::is_uint32_t(req.params[BATCH_SIZE])) {req.last_chunk_aggregate = true;+ res.final = true;res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");HttpServer::stream_response(req, res);return false;@@ -598,6 +601,7 @@ bool post_import_documents(http_req& req, http_res& res) {if(collection == nullptr) {req.last_chunk_aggregate = true;+ res.final = true;res.set_404();HttpServer::stream_response(req, res);return false;@@ -673,6 +677,7 @@ bool post_import_documents(http_req& req, http_res& res) {res.body += response_stream.str();if(stream_proceed) {+ res.final = req.last_chunk_aggregate;HttpServer::stream_response(req, res);} else {// push handler back onto the event loop: we must process the next batch without blocking the event loop@@ -1100,6 +1105,7 @@ bool raft_write_send_response(void *data) {if(index_arg->promise != nullptr) {index_arg->promise->set_value(true); // returns control back to raft replication thread+ index_arg->promise = nullptr;}}diff --git a/src/http_client.cpp b/src/http_client.cppindex 7dd54965..6eb8cabc 100644--- a/src/http_client.cpp+++ b/src/http_client.cpp@@ -19,8 +19,8 @@ long HttpClient::post_response(const std::string &url, const std::string &body,}long HttpClient::post_response_async(const std::string &url, http_req* request, http_res* response) {- deferred_req_res_t* req_res = new deferred_req_res_t{request, response, nullptr};- std::unique_ptr<deferred_req_res_t> index_arg_guard(req_res);+ request_response* req_res = new request_response{request, response};+ std::unique_ptr<request_response> req_res_guard(req_res);CURL *curl = init_curl_async(url, req_res);if(curl == nullptr) {@@ -120,7 +120,7 @@ void HttpClient::extract_response_headers(CURL* curl, std::map<std::string, std:size_t HttpClient::curl_req_send_callback(char* buffer, size_t size, size_t nitems, void* userdata) {// callback for request body to be sent to remote host- deferred_req_res_t* req_res = static_cast<deferred_req_res_t *>(userdata);+ request_response* req_res = static_cast<request_response *>(userdata);size_t max_req_bytes = (size * nitems);const char* total_body_buf = req_res->req->body.c_str();@@ -139,34 +139,30 @@ size_t HttpClient::curl_req_send_callback(char* buffer, size_t size, size_t niteif(req_res->req->body_index == req_res->req->body.size()) {LOG(INFO) << "Current body buffer has been consumed fully.";- size_t written = req_res->req->chunk_len;- req_res->req->chunk_len = 0;req_res->req->body_index = 0;req_res->req->body = "";- req_res->res->final = req_res->req->last_chunk_aggregate;- req_res->req->proxy_status = -1;- auto stream_state = (req_res->req->last_chunk_aggregate) ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;+ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(req_res->res->generator);+ HttpServer *server = custom_generator->h2o_handler->http_server;if(req_res->req->last_chunk_aggregate) {LOG(INFO) << "Request forwarding done.";- if(req_res->req->_req->proceed_req) {- req_res->req->_req->proceed_req(req_res->req->_req, written, stream_state);- }+ server->get_message_dispatcher()->send_message(HttpServer::REQUEST_PROCEED_MESSAGE, req_res);} else {LOG(INFO) << "Pausing forwarding and requesting more input.";- //curl_easy_pause(req_res->req->data, CURL_READFUNC_PAUSE);- req_res->req->_req->proceed_req(req_res->req->_req, written, stream_state);+ std::promise<bool> promise;+ std::future<bool> future = promise.get_future();+ req_res->req->promise = &promise;- while(req_res->req->proxy_status != 0 && req_res->req->body.empty()) {- LOG(INFO) << "Sleeping for 1 second...";- std::this_thread::sleep_for(std::chrono::milliseconds(1000));- }+ server->get_message_dispatcher()->send_message(HttpServer::REQUEST_PROCEED_MESSAGE, req_res);- req_res->req->proxy_status = 1;+ LOG(INFO) << "Waiting for request body to be ready";+ future.get();+ LOG(INFO) << "Request body is ready";++ req_res->req->promise = nullptr;LOG(INFO) << "Buffer refilled, unpausing request forwarding, body_size=" << req_res->req->body.size();- //curl_easy_pause(req_res->req->data, CURLPAUSE_RECV_CONT);}}@@ -177,28 +173,51 @@ size_t HttpClient::curl_write_async(char *buffer, size_t size, size_t nmemb, voi// callback for response body to be sent back to clientLOG(INFO) << "curl_write_async";- deferred_req_res_t* req_res = static_cast<deferred_req_res_t *>(context);+ request_response* req_res = static_cast<request_response *>(context);size_t res_size = size * nmemb;// FIXME: use header from remote response// we've got response from remote host: write to client and ask for more request bodyreq_res->res->content_type_header = "text/plain; charset=utf8";req_res->res->status_code = 200;-- // FIXME: cannot mutate body since it will be used by h2o to writereq_res->res->body = std::string(buffer, res_size);+ req_res->res->final = false;++ std::promise<bool> promise;+ std::future<bool> future = promise.get_future();+ req_res->res->promise = &promise;LOG(INFO) << "curl_write_async response, res body size: " << req_res->res->body.size();- // this needs to be sent from http thread- //h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(req_res->res->generator);- //HttpServer *server = custom_generator->h2o_handler->http_server;- //server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);+ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(req_res->res->generator);+ HttpServer *server = custom_generator->h2o_handler->http_server;+ server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);++ // wait until response is sent+ LOG(INFO) << "Waiting for response to be sent";+ future.get();+ LOG(INFO) << "Response sent";++ req_res->res->promise = nullptr;return res_size;}-CURL *HttpClient::init_curl_async(const std::string& url, deferred_req_res_t* req_res) {+size_t HttpClient::curl_write_async_done(void *context, curl_socket_t item) {+ LOG(INFO) << "curl_write_async_done";++ request_response* req_res = static_cast<request_response *>(context);+ req_res->res->body = "";+ req_res->res->final = true;++ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(req_res->res->generator);+ HttpServer *server = custom_generator->h2o_handler->http_server;+ server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);++ return 0;+}++CURL *HttpClient::init_curl_async(const std::string& url, request_response* req_res) {CURL *curl = curl_easy_init();if(curl == nullptr) {@@ -234,6 +253,9 @@ CURL *HttpClient::init_curl_async(const std::string& url, deferred_req_res_t* recurl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, HttpClient::curl_write_async);curl_easy_setopt(curl, CURLOPT_WRITEDATA, req_res);+ curl_easy_setopt(curl, CURLOPT_CLOSESOCKETFUNCTION, HttpClient::curl_write_async_done);+ curl_easy_setopt(curl, CURLOPT_CLOSESOCKETDATA, req_res);+return curl;}@@ -270,3 +292,4 @@ size_t HttpClient::curl_write(char *contents, size_t size, size_t nmemb, std::sts->append(contents, size*nmemb);return size*nmemb;}+diff --git a/src/http_server.cpp b/src/http_server.cppindex 4569a0f6..91ffc2b7 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -269,7 +269,7 @@ uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, con}void HttpServer::on_res_generator_dispose(void *self) {- //LOG(INFO) << "on_res_generator_dispose fires";+ LOG(INFO) << "on_res_generator_dispose fires";h2o_custom_generator_t* res_generator = static_cast<h2o_custom_generator_t*>(self);destroy_request_response(res_generator->request, res_generator->response);}@@ -376,11 +376,11 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {http_req* request = new http_req(req, rpath->http_method, route_hash, query_map, body);http_res* response = new http_res();- // add custom generator with a dipose function for cleaning up resources+ // add custom generator with a dispose function for cleaning up resourcesh2o_custom_generator_t* custom_generator = static_cast<h2o_custom_generator_t*>(h2o_mem_alloc_shared(&req->pool, sizeof(*custom_generator), on_res_generator_dispose));- custom_generator->super = h2o_generator_t {response_proceed, nullptr};+ custom_generator->super = h2o_generator_t {response_proceed, response_abort};custom_generator->request = request;custom_generator->response = response;custom_generator->rpath = rpath;@@ -524,6 +524,12 @@ void HttpServer::defer_processing(http_req& req, http_res& res, size_t timeout_mh2o_timer_unlink(&req.defer_timer.timer);h2o_timer_link(ctx.loop, timeout_ms, &req.defer_timer.timer);++ if(exit_loop && res.promise) {+ // otherwise, replication thread could be stuck waiting on a future+ res.promise->set_value(true);+ res.promise = nullptr;+ }}void HttpServer::send_message(const std::string & type, void* data) {@@ -559,42 +565,67 @@ void HttpServer::send_response(http_req* request, http_res* response) {h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL);}+void HttpServer::response_abort(h2o_generator_t *generator, h2o_req_t *req) {+ LOG(INFO) << "response_abort called";+ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t*>(generator);++ if(custom_generator->response->promise) {+ // returns control back to caller (raft replication or follower forward)+ LOG(INFO) << "response_abort: fulfilling promise.";+ custom_generator->response->promise->set_value(true);+ custom_generator->response->promise = nullptr;+ }+}+void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {- //LOG(INFO) << "response_proceed called";+ LOG(INFO) << "response_proceed called";h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t*>(generator);- // if the request itself is async, we will proceed the request to fetch input content- // otherwise, call the handler since it will be the handler that will be producing content+ if(custom_generator->response->promise) {+ // returns control back to caller (raft replication or follower forward)+ LOG(INFO) << "response_proceed: fulfilling promise.";+ custom_generator->response->promise->set_value(true);+ custom_generator->response->promise = nullptr;+ }- if(!custom_generator->response->async_request_proceed) {+ LOG(INFO) << "proceed_req_after_write: " << custom_generator->response->proceed_req_after_write;+ if(!custom_generator->response->proceed_req_after_write) {// request progression is not tied to response generation- //LOG(INFO) << "Ignoring request proceed";+ LOG(INFO) << "Ignoring request proceed";return ;}- if (custom_generator->rpath->async_req &&- custom_generator->request->_req && custom_generator->request->_req->proceed_req) {+ // if the request itself is async, we will proceed the request to fetch input content+ if (custom_generator->rpath->async_req) {+ auto stream_state = (custom_generator->response->final) ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;+ // `written` is ignored by http1.1 implementation, so meant only for http 2+size_t written = custom_generator->request->chunk_len;custom_generator->request->chunk_len = 0;- auto stream_state = (custom_generator->response->final) ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;-- // NOTE: `written` is ignored by http1.1 implementation, so it's meant only for http 2 and http 3- custom_generator->request->_req->proceed_req(custom_generator->request->_req, written, stream_state);+ if(custom_generator->request->_req->proceed_req) {+ LOG(INFO) << "response_proceed: proceeding req";+ custom_generator->request->_req->proceed_req(custom_generator->request->_req, written, stream_state);+ }} else {+ // otherwise, call the handler since it will be the handler that will be producing content+ // (streaming response but not request)custom_generator->rpath->handler(*custom_generator->request, *custom_generator->response);}}void HttpServer::stream_response(http_req& request, http_res& response) {- //LOG(INFO) << "stream_response called";+ LOG(INFO) << "stream_response called";if(request._req == nullptr) {// raft log replay+ LOG(INFO) << "request._req == nullptr";+if(response.promise) {// returns control back to raft replication threadresponse.promise->set_value(true);+ response.promise = nullptr;}+destroy_request_response(&request, &response);return;}@@ -603,7 +634,7 @@ void HttpServer::stream_response(http_req& request, http_res& response) {h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(response.generator);if (req->res.status == 0) {- //LOG(INFO) << "h2o_start_response";+ LOG(INFO) << "h2o_start_response, content_type=" << response.content_type_header;response.status_code = (response.status_code == 0) ? 503 : response.status_code; // just to be surereq->res.status = response.status_code;req->res.reason = http_res::get_status_reason(response.status_code);@@ -613,20 +644,20 @@ void HttpServer::stream_response(http_req& request, http_res& response) {h2o_start_response(req, &custom_generator->super);}- //LOG(INFO) << "stream_response, body_size: " << response.body.size();+ LOG(INFO) << "stream_response, body_size: " << response.body.size() << ", response_final="+ << custom_generator->response->final;h2o_iovec_t body = h2o_strdup(&req->pool, response.body.c_str(), SIZE_MAX);response.body = "";- // FIXME: should this be moved outside?- custom_generator->response->final = request.last_chunk_aggregate;-const h2o_send_state_t state = custom_generator->response->final ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;h2o_send(req, &body, 1, state);- if(response.promise) {+ // for intermediate responses, promise fulfillment will be handled by `response_proceed`+ if(custom_generator->response->final && response.promise) {// returns control back to raft replication threadresponse.promise->set_value(true);+ response.promise = nullptr;}}@@ -727,8 +758,23 @@ uint64_t HttpServer::node_state() const {}bool HttpServer::on_stream_response_message(void *data) {- //LOG(INFO) << "on_stream_response_message";+ LOG(INFO) << "on_stream_response_message";deferred_req_res_t* req_res = static_cast<deferred_req_res_t *>(data);stream_response(*req_res->req, *req_res->res);return true;}++bool HttpServer::on_request_proceed_message(void *data) {+ LOG(INFO) << "on_request_proceed_message";+ deferred_req_res_t* req_res = static_cast<deferred_req_res_t *>(data);+ auto stream_state = (req_res->req->last_chunk_aggregate) ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;++ size_t written = req_res->req->chunk_len;+ req_res->req->chunk_len = 0;++ if(req_res->req->_req->proceed_req) {+ req_res->req->_req->proceed_req(req_res->req->_req, written, stream_state);+ }++ return true;+}diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cppindex 3000c26e..ce6155b2 100644--- a/src/main/benchmark.cpp+++ b/src/main/benchmark.cpp@@ -51,7 +51,7 @@ void benchmark_hn_titles(char* file_path) {Collection *collection = collectionManager.get_collection("hnstories_direct");if(collection == nullptr) {- collection = collectionManager.create_collection("hnstories_direct", fields_to_index, "points").get();+ collection = collectionManager.create_collection("hnstories_direct", 4, fields_to_index, "points").get();}std::ifstream infile(file_path);diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 7293da62..237efa16 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -92,83 +92,8 @@ std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_end}void ReplicationState::write(http_req* request, http_res* response) {- // NOTE: this is executed on a different thread and runs concurrent to http threadif (!node->is_leader()) {- if(node->leader_id().is_empty()) {- // Handle no leader scenario- // FIXME: could happen in the middle of streaming, so a h2o_start_response can bomb.- LOG(ERROR) << "Rejecting write: could not find a leader.";- response->set_500("Could not find a leader.");- auto replication_arg = new AsyncIndexArg{request, response, nullptr};- replication_arg->req->route_hash = static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED);- return message_dispatcher->send_message(REPLICATION_MSG, replication_arg);- }-- if (request->proxy_status == -1) {- // indicates async request body of in-flight request- //LOG(INFO) << "Inflight request, updating proxy status, body size: " << request->body.size();- request->proxy_status = 0;- return ;- }-- const std::string & leader_addr = node->leader_id().to_string();- LOG(INFO) << "Redirecting write to leader at: " << leader_addr;-- thread_pool->enqueue([leader_addr, request, response, this]() {- auto raw_req = request->_req;- std::string scheme = std::string(raw_req->scheme->name.base, raw_req->scheme->name.len);- std::vector<std::string> addr_parts;- StringUtils::split(leader_addr, addr_parts, ":");- std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];- const std::string & path = std::string(raw_req->path.base, raw_req->path.len);- std::string url = scheme + "://" + leader_host_port + path;-- std::map<std::string, std::string> res_headers;-- if(request->http_method == "POST") {- std::vector<std::string> path_parts;- StringUtils::split(path, path_parts, "/");-- if(path_parts.back().rfind("import", 0) == 0) {- // imports are handled asynchronously- response->async_request_proceed = false;- request->proxy_status = 1;-- long status = HttpClient::post_response_async(url, request, response);- if(status == 500) {- response->content_type_header = res_headers["content-type"];- response->set_500("");- } else {- return ;- }- } else {- std::string api_res;- long status = HttpClient::post_response(url, request->body, api_res, res_headers);- response->content_type_header = res_headers["content-type"];- response->set_body(status, api_res);- }- } else if(request->http_method == "PUT") {- std::string api_res;- long status = HttpClient::put_response(url, request->body, api_res, res_headers);- response->content_type_header = res_headers["content-type"];- response->set_body(status, api_res);- } else if(request->http_method == "DELETE") {- std::string api_res;- long status = HttpClient::delete_response(url, api_res, res_headers);- response->content_type_header = res_headers["content-type"];- response->set_body(status, api_res);- } else {- const std::string& err = "Forwarding for http method not implemented: " + request->http_method;- LOG(ERROR) << err;- response->set_500(err);- }-- auto replication_arg = new AsyncIndexArg{request, response, nullptr};- replication_arg->req->route_hash = static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED);- message_dispatcher->send_message(REPLICATION_MSG, replication_arg);- });-- return ;+ return follower_write(request, response);}// Serialize request to replicated WAL so that all the nodes in the group receive it as well.@@ -187,13 +112,111 @@ void ReplicationState::write(http_req* request, http_res* response) {// To avoid ABA problemtask.expected_term = leader_term.load(butil::memory_order_relaxed);- //LOG(INFO) << ":::" << "body size before apply: " << request->body.size();+ LOG(INFO) << ":::" << "body size before apply: " << request->body.size();// Now the task is applied to the group, waiting for the result.return node->apply(task);}+size_t follower_write_count = 0;++void ReplicationState::follower_write(http_req *request, http_res *response) const {+ follower_write_count++;++ LOG(INFO) << "follower_write_count: " << follower_write_count;++ if(follower_write_count == 1) {+ //LOG(INFO) << "follower_write, will sleep for 10 seconds...";+ //std::this_thread::sleep_for(std::chrono::seconds(10));+ }++ if(node->leader_id().is_empty()) {+ // Handle no leader scenario+ LOG(ERROR) << "Rejecting write: could not find a leader.";++ if(request->_req->proceed_req && request->promise) {+ // streaming in progress: ensure graceful termination (cannot start response again)+ LOG(ERROR) << "Terminating streaming request gracefully.";+ request->promise->set_value(true);+ request->promise = nullptr;+ return ;+ }++ response->set_500("Could not find a leader.");+ auto replication_arg = new AsyncIndexArg{request, response, nullptr};+ replication_arg->req->route_hash = static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED);+ return message_dispatcher->send_message(REPLICATION_MSG, replication_arg);+ }++ if (request->_req->proceed_req && request->promise) {+ // indicates async request body of in-flight request+ LOG(INFO) << "Inflight proxied request, returning control to caller, body_size=" << request->body.size();+ request->promise->set_value(true);+ request->promise = nullptr;+ return ;+ }++ const std::string & leader_addr = node->leader_id().to_string();+ LOG(INFO) << "Redirecting write to leader at: " << leader_addr;++ thread_pool->enqueue([leader_addr, request, response, this]() {+ auto raw_req = request->_req;+ std::string scheme = std::string(raw_req->scheme->name.base, raw_req->scheme->name.len);+ std::vector<std::string> addr_parts;+ StringUtils::split(leader_addr, addr_parts, ":");+ std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];+ const std::string & path = std::string(raw_req->path.base, raw_req->path.len);+ std::string url = scheme + "://" + leader_host_port + path;++ std::map<std::string, std::string> res_headers;++ if(request->http_method == "POST") {+ std::vector<std::string> path_parts;+ StringUtils::split(path, path_parts, "/");++ if(path_parts.back().rfind("import", 0) == 0) {+ // imports are handled asynchronously+ response->proceed_req_after_write = false;++ long status = HttpClient::post_response_async(url, request, response);+ if(status == 500) {+ response->content_type_header = res_headers["content-type"];+ response->set_500("");+ } else {+ return ;+ }+ } else {+ std::string api_res;+ long status = HttpClient::post_response(url, request->body, api_res, res_headers);+ response->content_type_header = res_headers["content-type"];+ response->set_body(status, api_res);+ }+ } else if(request->http_method == "PUT") {+ std::string api_res;+ long status = HttpClient::put_response(url, request->body, api_res, res_headers);+ response->content_type_header = res_headers["content-type"];+ response->set_body(status, api_res);+ } else if(request->http_method == "DELETE") {+ std::string api_res;+ long status = HttpClient::delete_response(url, api_res, res_headers);+ response->content_type_header = res_headers["content-type"];+ response->set_body(status, api_res);+ } else {+ const std::string& err = "Forwarding for http method not implemented: " + request->http_method;+ LOG(ERROR) << err;+ response->set_500(err);+ }++ auto replication_arg = new AsyncIndexArg{request, response, nullptr};+ replication_arg->req->route_hash = static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED);+ message_dispatcher->send_message(REPLICATION_MSG, replication_arg);+ });+}+void ReplicationState::on_apply(braft::Iterator& iter) {+ LOG(INFO) << "ReplicationState::on_apply";++ // NOTE: this is executed on a different thread and runs concurrent to http thread// A batch of tasks are committed, which must be processed through// |iter|for (; iter.valid(); iter.next()) {@@ -237,7 +260,9 @@ void ReplicationState::on_apply(braft::Iterator& iter) {std::future<bool> future = promise.get_future();auto replication_arg = new AsyncIndexArg{request, response, &promise};message_dispatcher->send_message(REPLICATION_MSG, replication_arg);+ LOG(INFO) << "Raft write waiting for future";future.get();+ LOG(INFO) << "Raft write got the future";}}@@ -325,7 +350,7 @@ int ReplicationState::init_db() {}int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {- CHECK(!node->is_leader()) << "Leader is not supposed to load snapshot";+ CHECK(!node || !node->is_leader()) << "Leader is not supposed to load snapshot";LOG(INFO) << "on_snapshot_load";diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex e09a12c2..4350d340 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -252,7 +252,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& stLOG(INFO) << "Typesense peering service is going to quit.";- // Stop counter before server+ // Stop application before serverreplication_state.shutdown();raft_server.Stop(0);@@ -349,6 +349,7 @@ int run_server(const Config & config, const std::string & version, void (*masterserver->on(SEND_RESPONSE_MSG, on_send_response);server->on(ReplicationState::REPLICATION_MSG, raft_write_send_response);server->on(HttpServer::STREAM_RESPONSE_MESSAGE, HttpServer::on_stream_response_message);+ server->on(HttpServer::REQUEST_PROCEED_MESSAGE, HttpServer::on_request_proceed_message);// first we start the peering service