Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Implemented import streaming end-to-end on follower.

kishorenc6d7c6b2ea0c6
diff --git a/include/http_client.h b/include/http_client.h
index cda44f6c..a777bfec 100644
--- a/include/http_client.h
+++ b/include/http_client.h
@@ -22,9 +22,11 @@ private:
static size_t curl_write_async(char *buffer, size_t size, size_t nmemb, void* context);
+ static size_t curl_write_async_done(void* context, curl_socket_t item);
+
static CURL* init_curl(const std::string& url, std::string& response);
- static CURL* init_curl_async(const std::string& url, deferred_req_res_t* req_res);
+ static CURL* init_curl_async(const std::string& url, request_response* req_res);
static size_t curl_req_send_callback(char* buffer, size_t size, size_t nitems, void *userdata);
diff --git a/include/http_data.h b/include/http_data.h
index 446e94a3..cead00a8 100644
--- a/include/http_data.h
+++ b/include/http_data.h
@@ -37,9 +37,9 @@ struct http_res {
h2o_generator_t* generator = nullptr;
// for async requests, automatically progresses request body on response proceed
- bool async_request_proceed = true;
+ bool proceed_req_after_write = true;
- http_res(): status_code(501), content_type_header("application/json; charset=utf-8"), final(true) {
+ http_res(): status_code(0), content_type_header("application/json; charset=utf-8"), final(true) {
}
@@ -147,14 +147,14 @@ struct http_req {
void* data;
// used during forwarding of requests from follower to leader
- std::atomic<int> proxy_status;
+ std::promise<bool>* promise = nullptr;
// for deffered processing of async handlers
h2o_custom_timer_t defer_timer;
http_req(): _req(nullptr), route_hash(1),
first_chunk_aggregate(true), last_chunk_aggregate(false),
- chunk_len(0), body_index(0), data(nullptr), proxy_status(0) {
+ chunk_len(0), body_index(0), data(nullptr), promise(nullptr) {
}
@@ -162,7 +162,7 @@ struct http_req {
const std::map<std::string, std::string> & params, const std::string& body):
_req(_req), http_method(http_method), route_hash(route_hash), params(params),
first_chunk_aggregate(true), last_chunk_aggregate(false),
- chunk_len(0), body(body), body_index(0), data(nullptr), proxy_status(0) {
+ chunk_len(0), body(body), body_index(0), data(nullptr), promise(nullptr) {
}
@@ -332,7 +332,7 @@ struct http_message_dispatcher {
}
void send_message(const std::string & type, void* data) {
- h2o_custom_res_message_t* message = new h2o_custom_res_message_t{{{NULL, NULL}}, &message_handlers, type.c_str(), data};
+ h2o_custom_res_message_t* message = new h2o_custom_res_message_t{{{nullptr, nullptr}}, &message_handlers, type, data};
h2o_multithread_send_message(message_receiver, &message->super);
}
diff --git a/include/http_server.h b/include/http_server.h
index 7622df15..471ccb05 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -91,6 +91,8 @@ private:
static void response_proceed(h2o_generator_t *generator, h2o_req_t *req);
+ static void response_abort(h2o_generator_t *generator, h2o_req_t *req);
+
static void on_res_generator_dispose(void *self);
static int send_response(h2o_req_t *req, int status_code, const std::string & message);
@@ -148,11 +150,14 @@ public:
static bool on_stream_response_message(void *data);
+ static bool on_request_proceed_message(void *data);
+
std::string get_version();
static constexpr const char* AUTH_HEADER = "x-typesense-api-key";
static constexpr const char* STOP_SERVER_MESSAGE = "STOP_SERVER";
static constexpr const char* STREAM_RESPONSE_MESSAGE = "STREAM_RESPONSE";
+ static constexpr const char* REQUEST_PROCEED_MESSAGE = "REQUEST_PROCEED";
static int process_request(http_req* request, http_res* response, route_path *rpath,
const h2o_custom_req_handler_t *req_handler);
diff --git a/include/raft_server.h b/include/raft_server.h
index 50e20349..e2a8be01 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -214,4 +214,6 @@ private:
void on_stop_following(const ::braft::LeaderChangeContext& ctx) {
LOG(INFO) << "Node stops following " << ctx;
}
+
+ void follower_write(http_req *request, http_res *response) const;
};
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 035b82f4..d102ffbe 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -38,7 +38,7 @@ Collection* CollectionManager::init_collection(const nlohmann::json & collection
collection_meta[COLLECTION_NUM_MEMORY_SHARDS].get<size_t>() :
DEFAULT_NUM_MEMORY_SHARDS;
- LOG(INFO) << "Loading collection with " << num_memory_shards << " memory shards.";
+ LOG(INFO) << "Found collection " << this_collection_name << " with " << num_memory_shards << " memory shards.";
Collection* collection = new Collection(this_collection_name,
collection_meta[COLLECTION_ID_KEY].get<uint32_t>(),
diff --git a/src/core_api.cpp b/src/core_api.cpp
index ca0ba7fe..0c3d3836 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -525,6 +525,7 @@ bool get_export_documents(http_req & req, http_res & res) {
if(collection == nullptr) {
req.last_chunk_aggregate = true;
+ res.final = true;
res.set_404();
HttpServer::stream_response(req, res);
return false;
@@ -551,6 +552,7 @@ bool get_export_documents(http_req & req, http_res & res) {
req.last_chunk_aggregate = false;
} else {
req.last_chunk_aggregate = true;
+ res.final = true;
delete it;
req.data = nullptr;
}
@@ -574,6 +576,7 @@ bool post_import_documents(http_req& req, http_res& res) {
if(!StringUtils::is_uint32_t(req.params[BATCH_SIZE])) {
req.last_chunk_aggregate = true;
+ res.final = true;
res.set_400("Parameter `" + std::string(BATCH_SIZE) + "` must be a positive integer.");
HttpServer::stream_response(req, res);
return false;
@@ -598,6 +601,7 @@ bool post_import_documents(http_req& req, http_res& res) {
if(collection == nullptr) {
req.last_chunk_aggregate = true;
+ res.final = true;
res.set_404();
HttpServer::stream_response(req, res);
return false;
@@ -673,6 +677,7 @@ bool post_import_documents(http_req& req, http_res& res) {
res.body += response_stream.str();
if(stream_proceed) {
+ res.final = req.last_chunk_aggregate;
HttpServer::stream_response(req, res);
} else {
// push handler back onto the event loop: we must process the next batch without blocking the event loop
@@ -1100,6 +1105,7 @@ bool raft_write_send_response(void *data) {
if(index_arg->promise != nullptr) {
index_arg->promise->set_value(true); // returns control back to raft replication thread
+ index_arg->promise = nullptr;
}
}
diff --git a/src/http_client.cpp b/src/http_client.cpp
index 7dd54965..6eb8cabc 100644
--- a/src/http_client.cpp
+++ b/src/http_client.cpp
@@ -19,8 +19,8 @@ long HttpClient::post_response(const std::string &url, const std::string &body,
}
long HttpClient::post_response_async(const std::string &url, http_req* request, http_res* response) {
- deferred_req_res_t* req_res = new deferred_req_res_t{request, response, nullptr};
- std::unique_ptr<deferred_req_res_t> index_arg_guard(req_res);
+ request_response* req_res = new request_response{request, response};
+ std::unique_ptr<request_response> req_res_guard(req_res);
CURL *curl = init_curl_async(url, req_res);
if(curl == nullptr) {
@@ -120,7 +120,7 @@ void HttpClient::extract_response_headers(CURL* curl, std::map<std::string, std:
size_t HttpClient::curl_req_send_callback(char* buffer, size_t size, size_t nitems, void* userdata) {
// callback for request body to be sent to remote host
- deferred_req_res_t* req_res = static_cast<deferred_req_res_t *>(userdata);
+ request_response* req_res = static_cast<request_response *>(userdata);
size_t max_req_bytes = (size * nitems);
const char* total_body_buf = req_res->req->body.c_str();
@@ -139,34 +139,30 @@ size_t HttpClient::curl_req_send_callback(char* buffer, size_t size, size_t nite
if(req_res->req->body_index == req_res->req->body.size()) {
LOG(INFO) << "Current body buffer has been consumed fully.";
- size_t written = req_res->req->chunk_len;
- req_res->req->chunk_len = 0;
req_res->req->body_index = 0;
req_res->req->body = "";
- req_res->res->final = req_res->req->last_chunk_aggregate;
- req_res->req->proxy_status = -1;
- auto stream_state = (req_res->req->last_chunk_aggregate) ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;
+ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(req_res->res->generator);
+ HttpServer *server = custom_generator->h2o_handler->http_server;
if(req_res->req->last_chunk_aggregate) {
LOG(INFO) << "Request forwarding done.";
- if(req_res->req->_req->proceed_req) {
- req_res->req->_req->proceed_req(req_res->req->_req, written, stream_state);
- }
+ server->get_message_dispatcher()->send_message(HttpServer::REQUEST_PROCEED_MESSAGE, req_res);
} else {
LOG(INFO) << "Pausing forwarding and requesting more input.";
- //curl_easy_pause(req_res->req->data, CURL_READFUNC_PAUSE);
- req_res->req->_req->proceed_req(req_res->req->_req, written, stream_state);
+ std::promise<bool> promise;
+ std::future<bool> future = promise.get_future();
+ req_res->req->promise = &promise;
- while(req_res->req->proxy_status != 0 && req_res->req->body.empty()) {
- LOG(INFO) << "Sleeping for 1 second...";
- std::this_thread::sleep_for(std::chrono::milliseconds(1000));
- }
+ server->get_message_dispatcher()->send_message(HttpServer::REQUEST_PROCEED_MESSAGE, req_res);
- req_res->req->proxy_status = 1;
+ LOG(INFO) << "Waiting for request body to be ready";
+ future.get();
+ LOG(INFO) << "Request body is ready";
+
+ req_res->req->promise = nullptr;
LOG(INFO) << "Buffer refilled, unpausing request forwarding, body_size=" << req_res->req->body.size();
- //curl_easy_pause(req_res->req->data, CURLPAUSE_RECV_CONT);
}
}
@@ -177,28 +173,51 @@ size_t HttpClient::curl_write_async(char *buffer, size_t size, size_t nmemb, voi
// callback for response body to be sent back to client
LOG(INFO) << "curl_write_async";
- deferred_req_res_t* req_res = static_cast<deferred_req_res_t *>(context);
+ request_response* req_res = static_cast<request_response *>(context);
size_t res_size = size * nmemb;
// FIXME: use header from remote response
// we've got response from remote host: write to client and ask for more request body
req_res->res->content_type_header = "text/plain; charset=utf8";
req_res->res->status_code = 200;
-
- // FIXME: cannot mutate body since it will be used by h2o to write
req_res->res->body = std::string(buffer, res_size);
+ req_res->res->final = false;
+
+ std::promise<bool> promise;
+ std::future<bool> future = promise.get_future();
+ req_res->res->promise = &promise;
LOG(INFO) << "curl_write_async response, res body size: " << req_res->res->body.size();
- // this needs to be sent from http thread
- //h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(req_res->res->generator);
- //HttpServer *server = custom_generator->h2o_handler->http_server;
- //server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);
+ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(req_res->res->generator);
+ HttpServer *server = custom_generator->h2o_handler->http_server;
+ server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);
+
+ // wait until response is sent
+ LOG(INFO) << "Waiting for response to be sent";
+ future.get();
+ LOG(INFO) << "Response sent";
+
+ req_res->res->promise = nullptr;
return res_size;
}
-CURL *HttpClient::init_curl_async(const std::string& url, deferred_req_res_t* req_res) {
+size_t HttpClient::curl_write_async_done(void *context, curl_socket_t item) {
+ LOG(INFO) << "curl_write_async_done";
+
+ request_response* req_res = static_cast<request_response *>(context);
+ req_res->res->body = "";
+ req_res->res->final = true;
+
+ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(req_res->res->generator);
+ HttpServer *server = custom_generator->h2o_handler->http_server;
+ server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, req_res);
+
+ return 0;
+}
+
+CURL *HttpClient::init_curl_async(const std::string& url, request_response* req_res) {
CURL *curl = curl_easy_init();
if(curl == nullptr) {
@@ -234,6 +253,9 @@ CURL *HttpClient::init_curl_async(const std::string& url, deferred_req_res_t* re
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, HttpClient::curl_write_async);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, req_res);
+ curl_easy_setopt(curl, CURLOPT_CLOSESOCKETFUNCTION, HttpClient::curl_write_async_done);
+ curl_easy_setopt(curl, CURLOPT_CLOSESOCKETDATA, req_res);
+
return curl;
}
@@ -270,3 +292,4 @@ size_t HttpClient::curl_write(char *contents, size_t size, size_t nmemb, std::st
s->append(contents, size*nmemb);
return size*nmemb;
}
+
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 4569a0f6..91ffc2b7 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -269,7 +269,7 @@ uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, con
}
void HttpServer::on_res_generator_dispose(void *self) {
- //LOG(INFO) << "on_res_generator_dispose fires";
+ LOG(INFO) << "on_res_generator_dispose fires";
h2o_custom_generator_t* res_generator = static_cast<h2o_custom_generator_t*>(self);
destroy_request_response(res_generator->request, res_generator->response);
}
@@ -376,11 +376,11 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {
http_req* request = new http_req(req, rpath->http_method, route_hash, query_map, body);
http_res* response = new http_res();
- // add custom generator with a dipose function for cleaning up resources
+ // add custom generator with a dispose function for cleaning up resources
h2o_custom_generator_t* custom_generator = static_cast<h2o_custom_generator_t*>(
h2o_mem_alloc_shared(&req->pool, sizeof(*custom_generator), on_res_generator_dispose)
);
- custom_generator->super = h2o_generator_t {response_proceed, nullptr};
+ custom_generator->super = h2o_generator_t {response_proceed, response_abort};
custom_generator->request = request;
custom_generator->response = response;
custom_generator->rpath = rpath;
@@ -524,6 +524,12 @@ void HttpServer::defer_processing(http_req& req, http_res& res, size_t timeout_m
h2o_timer_unlink(&req.defer_timer.timer);
h2o_timer_link(ctx.loop, timeout_ms, &req.defer_timer.timer);
+
+ if(exit_loop && res.promise) {
+ // otherwise, replication thread could be stuck waiting on a future
+ res.promise->set_value(true);
+ res.promise = nullptr;
+ }
}
void HttpServer::send_message(const std::string & type, void* data) {
@@ -559,42 +565,67 @@ void HttpServer::send_response(http_req* request, http_res* response) {
h2o_send(req, &body, 1, H2O_SEND_STATE_FINAL);
}
+void HttpServer::response_abort(h2o_generator_t *generator, h2o_req_t *req) {
+ LOG(INFO) << "response_abort called";
+ h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t*>(generator);
+
+ if(custom_generator->response->promise) {
+ // returns control back to caller (raft replication or follower forward)
+ LOG(INFO) << "response_abort: fulfilling promise.";
+ custom_generator->response->promise->set_value(true);
+ custom_generator->response->promise = nullptr;
+ }
+}
+
void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {
- //LOG(INFO) << "response_proceed called";
+ LOG(INFO) << "response_proceed called";
h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t*>(generator);
- // if the request itself is async, we will proceed the request to fetch input content
- // otherwise, call the handler since it will be the handler that will be producing content
+ if(custom_generator->response->promise) {
+ // returns control back to caller (raft replication or follower forward)
+ LOG(INFO) << "response_proceed: fulfilling promise.";
+ custom_generator->response->promise->set_value(true);
+ custom_generator->response->promise = nullptr;
+ }
- if(!custom_generator->response->async_request_proceed) {
+ LOG(INFO) << "proceed_req_after_write: " << custom_generator->response->proceed_req_after_write;
+ if(!custom_generator->response->proceed_req_after_write) {
// request progression is not tied to response generation
- //LOG(INFO) << "Ignoring request proceed";
+ LOG(INFO) << "Ignoring request proceed";
return ;
}
- if (custom_generator->rpath->async_req &&
- custom_generator->request->_req && custom_generator->request->_req->proceed_req) {
+ // if the request itself is async, we will proceed the request to fetch input content
+ if (custom_generator->rpath->async_req) {
+ auto stream_state = (custom_generator->response->final) ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;
+ // `written` is ignored by http1.1 implementation, so meant only for http 2+
size_t written = custom_generator->request->chunk_len;
custom_generator->request->chunk_len = 0;
- auto stream_state = (custom_generator->response->final) ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;
-
- // NOTE: `written` is ignored by http1.1 implementation, so it's meant only for http 2 and http 3
- custom_generator->request->_req->proceed_req(custom_generator->request->_req, written, stream_state);
+ if(custom_generator->request->_req->proceed_req) {
+ LOG(INFO) << "response_proceed: proceeding req";
+ custom_generator->request->_req->proceed_req(custom_generator->request->_req, written, stream_state);
+ }
} else {
+ // otherwise, call the handler since it will be the handler that will be producing content
+ // (streaming response but not request)
custom_generator->rpath->handler(*custom_generator->request, *custom_generator->response);
}
}
void HttpServer::stream_response(http_req& request, http_res& response) {
- //LOG(INFO) << "stream_response called";
+ LOG(INFO) << "stream_response called";
if(request._req == nullptr) {
// raft log replay
+ LOG(INFO) << "request._req == nullptr";
+
if(response.promise) {
// returns control back to raft replication thread
response.promise->set_value(true);
+ response.promise = nullptr;
}
+
destroy_request_response(&request, &response);
return;
}
@@ -603,7 +634,7 @@ void HttpServer::stream_response(http_req& request, http_res& response) {
h2o_custom_generator_t* custom_generator = reinterpret_cast<h2o_custom_generator_t *>(response.generator);
if (req->res.status == 0) {
- //LOG(INFO) << "h2o_start_response";
+ LOG(INFO) << "h2o_start_response, content_type=" << response.content_type_header;
response.status_code = (response.status_code == 0) ? 503 : response.status_code; // just to be sure
req->res.status = response.status_code;
req->res.reason = http_res::get_status_reason(response.status_code);
@@ -613,20 +644,20 @@ void HttpServer::stream_response(http_req& request, http_res& response) {
h2o_start_response(req, &custom_generator->super);
}
- //LOG(INFO) << "stream_response, body_size: " << response.body.size();
+ LOG(INFO) << "stream_response, body_size: " << response.body.size() << ", response_final="
+ << custom_generator->response->final;
h2o_iovec_t body = h2o_strdup(&req->pool, response.body.c_str(), SIZE_MAX);
response.body = "";
- // FIXME: should this be moved outside?
- custom_generator->response->final = request.last_chunk_aggregate;
-
const h2o_send_state_t state = custom_generator->response->final ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;
h2o_send(req, &body, 1, state);
- if(response.promise) {
+ // for intermediate responses, promise fulfillment will be handled by `response_proceed`
+ if(custom_generator->response->final && response.promise) {
// returns control back to raft replication thread
response.promise->set_value(true);
+ response.promise = nullptr;
}
}
@@ -727,8 +758,23 @@ uint64_t HttpServer::node_state() const {
}
bool HttpServer::on_stream_response_message(void *data) {
- //LOG(INFO) << "on_stream_response_message";
+ LOG(INFO) << "on_stream_response_message";
deferred_req_res_t* req_res = static_cast<deferred_req_res_t *>(data);
stream_response(*req_res->req, *req_res->res);
return true;
}
+
+bool HttpServer::on_request_proceed_message(void *data) {
+ LOG(INFO) << "on_request_proceed_message";
+ deferred_req_res_t* req_res = static_cast<deferred_req_res_t *>(data);
+ auto stream_state = (req_res->req->last_chunk_aggregate) ? H2O_SEND_STATE_FINAL : H2O_SEND_STATE_IN_PROGRESS;
+
+ size_t written = req_res->req->chunk_len;
+ req_res->req->chunk_len = 0;
+
+ if(req_res->req->_req->proceed_req) {
+ req_res->req->_req->proceed_req(req_res->req->_req, written, stream_state);
+ }
+
+ return true;
+}
diff --git a/src/main/benchmark.cpp b/src/main/benchmark.cpp
index 3000c26e..ce6155b2 100644
--- a/src/main/benchmark.cpp
+++ b/src/main/benchmark.cpp
@@ -51,7 +51,7 @@ void benchmark_hn_titles(char* file_path) {
Collection *collection = collectionManager.get_collection("hnstories_direct");
if(collection == nullptr) {
- collection = collectionManager.create_collection("hnstories_direct", fields_to_index, "points").get();
+ collection = collectionManager.create_collection("hnstories_direct", 4, fields_to_index, "points").get();
}
std::ifstream infile(file_path);
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 7293da62..237efa16 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -92,83 +92,8 @@ std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_end
}
void ReplicationState::write(http_req* request, http_res* response) {
- // NOTE: this is executed on a different thread and runs concurrent to http thread
if (!node->is_leader()) {
- if(node->leader_id().is_empty()) {
- // Handle no leader scenario
- // FIXME: could happen in the middle of streaming, so a h2o_start_response can bomb.
- LOG(ERROR) << "Rejecting write: could not find a leader.";
- response->set_500("Could not find a leader.");
- auto replication_arg = new AsyncIndexArg{request, response, nullptr};
- replication_arg->req->route_hash = static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED);
- return message_dispatcher->send_message(REPLICATION_MSG, replication_arg);
- }
-
- if (request->proxy_status == -1) {
- // indicates async request body of in-flight request
- //LOG(INFO) << "Inflight request, updating proxy status, body size: " << request->body.size();
- request->proxy_status = 0;
- return ;
- }
-
- const std::string & leader_addr = node->leader_id().to_string();
- LOG(INFO) << "Redirecting write to leader at: " << leader_addr;
-
- thread_pool->enqueue([leader_addr, request, response, this]() {
- auto raw_req = request->_req;
- std::string scheme = std::string(raw_req->scheme->name.base, raw_req->scheme->name.len);
- std::vector<std::string> addr_parts;
- StringUtils::split(leader_addr, addr_parts, ":");
- std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];
- const std::string & path = std::string(raw_req->path.base, raw_req->path.len);
- std::string url = scheme + "://" + leader_host_port + path;
-
- std::map<std::string, std::string> res_headers;
-
- if(request->http_method == "POST") {
- std::vector<std::string> path_parts;
- StringUtils::split(path, path_parts, "/");
-
- if(path_parts.back().rfind("import", 0) == 0) {
- // imports are handled asynchronously
- response->async_request_proceed = false;
- request->proxy_status = 1;
-
- long status = HttpClient::post_response_async(url, request, response);
- if(status == 500) {
- response->content_type_header = res_headers["content-type"];
- response->set_500("");
- } else {
- return ;
- }
- } else {
- std::string api_res;
- long status = HttpClient::post_response(url, request->body, api_res, res_headers);
- response->content_type_header = res_headers["content-type"];
- response->set_body(status, api_res);
- }
- } else if(request->http_method == "PUT") {
- std::string api_res;
- long status = HttpClient::put_response(url, request->body, api_res, res_headers);
- response->content_type_header = res_headers["content-type"];
- response->set_body(status, api_res);
- } else if(request->http_method == "DELETE") {
- std::string api_res;
- long status = HttpClient::delete_response(url, api_res, res_headers);
- response->content_type_header = res_headers["content-type"];
- response->set_body(status, api_res);
- } else {
- const std::string& err = "Forwarding for http method not implemented: " + request->http_method;
- LOG(ERROR) << err;
- response->set_500(err);
- }
-
- auto replication_arg = new AsyncIndexArg{request, response, nullptr};
- replication_arg->req->route_hash = static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED);
- message_dispatcher->send_message(REPLICATION_MSG, replication_arg);
- });
-
- return ;
+ return follower_write(request, response);
}
// Serialize request to replicated WAL so that all the nodes in the group receive it as well.
@@ -187,13 +112,111 @@ void ReplicationState::write(http_req* request, http_res* response) {
// To avoid ABA problem
task.expected_term = leader_term.load(butil::memory_order_relaxed);
- //LOG(INFO) << ":::" << "body size before apply: " << request->body.size();
+ LOG(INFO) << ":::" << "body size before apply: " << request->body.size();
// Now the task is applied to the group, waiting for the result.
return node->apply(task);
}
+size_t follower_write_count = 0;
+
+void ReplicationState::follower_write(http_req *request, http_res *response) const {
+ follower_write_count++;
+
+ LOG(INFO) << "follower_write_count: " << follower_write_count;
+
+ if(follower_write_count == 1) {
+ //LOG(INFO) << "follower_write, will sleep for 10 seconds...";
+ //std::this_thread::sleep_for(std::chrono::seconds(10));
+ }
+
+ if(node->leader_id().is_empty()) {
+ // Handle no leader scenario
+ LOG(ERROR) << "Rejecting write: could not find a leader.";
+
+ if(request->_req->proceed_req && request->promise) {
+ // streaming in progress: ensure graceful termination (cannot start response again)
+ LOG(ERROR) << "Terminating streaming request gracefully.";
+ request->promise->set_value(true);
+ request->promise = nullptr;
+ return ;
+ }
+
+ response->set_500("Could not find a leader.");
+ auto replication_arg = new AsyncIndexArg{request, response, nullptr};
+ replication_arg->req->route_hash = static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED);
+ return message_dispatcher->send_message(REPLICATION_MSG, replication_arg);
+ }
+
+ if (request->_req->proceed_req && request->promise) {
+ // indicates async request body of in-flight request
+ LOG(INFO) << "Inflight proxied request, returning control to caller, body_size=" << request->body.size();
+ request->promise->set_value(true);
+ request->promise = nullptr;
+ return ;
+ }
+
+ const std::string & leader_addr = node->leader_id().to_string();
+ LOG(INFO) << "Redirecting write to leader at: " << leader_addr;
+
+ thread_pool->enqueue([leader_addr, request, response, this]() {
+ auto raw_req = request->_req;
+ std::string scheme = std::string(raw_req->scheme->name.base, raw_req->scheme->name.len);
+ std::vector<std::string> addr_parts;
+ StringUtils::split(leader_addr, addr_parts, ":");
+ std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];
+ const std::string & path = std::string(raw_req->path.base, raw_req->path.len);
+ std::string url = scheme + "://" + leader_host_port + path;
+
+ std::map<std::string, std::string> res_headers;
+
+ if(request->http_method == "POST") {
+ std::vector<std::string> path_parts;
+ StringUtils::split(path, path_parts, "/");
+
+ if(path_parts.back().rfind("import", 0) == 0) {
+ // imports are handled asynchronously
+ response->proceed_req_after_write = false;
+
+ long status = HttpClient::post_response_async(url, request, response);
+ if(status == 500) {
+ response->content_type_header = res_headers["content-type"];
+ response->set_500("");
+ } else {
+ return ;
+ }
+ } else {
+ std::string api_res;
+ long status = HttpClient::post_response(url, request->body, api_res, res_headers);
+ response->content_type_header = res_headers["content-type"];
+ response->set_body(status, api_res);
+ }
+ } else if(request->http_method == "PUT") {
+ std::string api_res;
+ long status = HttpClient::put_response(url, request->body, api_res, res_headers);
+ response->content_type_header = res_headers["content-type"];
+ response->set_body(status, api_res);
+ } else if(request->http_method == "DELETE") {
+ std::string api_res;
+ long status = HttpClient::delete_response(url, api_res, res_headers);
+ response->content_type_header = res_headers["content-type"];
+ response->set_body(status, api_res);
+ } else {
+ const std::string& err = "Forwarding for http method not implemented: " + request->http_method;
+ LOG(ERROR) << err;
+ response->set_500(err);
+ }
+
+ auto replication_arg = new AsyncIndexArg{request, response, nullptr};
+ replication_arg->req->route_hash = static_cast<uint64_t>(ROUTE_CODES::ALREADY_HANDLED);
+ message_dispatcher->send_message(REPLICATION_MSG, replication_arg);
+ });
+}
+
void ReplicationState::on_apply(braft::Iterator& iter) {
+ LOG(INFO) << "ReplicationState::on_apply";
+
+ // NOTE: this is executed on a different thread and runs concurrent to http thread
// A batch of tasks are committed, which must be processed through
// |iter|
for (; iter.valid(); iter.next()) {
@@ -237,7 +260,9 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
std::future<bool> future = promise.get_future();
auto replication_arg = new AsyncIndexArg{request, response, &promise};
message_dispatcher->send_message(REPLICATION_MSG, replication_arg);
+ LOG(INFO) << "Raft write waiting for future";
future.get();
+ LOG(INFO) << "Raft write got the future";
}
}
@@ -325,7 +350,7 @@ int ReplicationState::init_db() {
}
int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
- CHECK(!node->is_leader()) << "Leader is not supposed to load snapshot";
+ CHECK(!node || !node->is_leader()) << "Leader is not supposed to load snapshot";
LOG(INFO) << "on_snapshot_load";
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index e09a12c2..4350d340 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -252,7 +252,7 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
LOG(INFO) << "Typesense peering service is going to quit.";
- // Stop counter before server
+ // Stop application before server
replication_state.shutdown();
raft_server.Stop(0);
@@ -349,6 +349,7 @@ int run_server(const Config & config, const std::string & version, void (*master
server->on(SEND_RESPONSE_MSG, on_send_response);
server->on(ReplicationState::REPLICATION_MSG, raft_write_send_response);
server->on(HttpServer::STREAM_RESPONSE_MESSAGE, HttpServer::on_stream_response_message);
+ server->on(HttpServer::REQUEST_PROCEED_MESSAGE, HttpServer::on_request_proceed_message);
// first we start the peering service