Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Clean shutdown.

kishorenc9f159b5d3d89
diff --git a/include/raft_server.h b/include/raft_server.h
index f6012098..a19efa8d 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -114,8 +114,6 @@ private:
bool create_init_db_snapshot;
- std::atomic<bool>& shut_down;
-
std::string raft_dir_path;
std::string ext_snapshot_path;
@@ -126,6 +124,9 @@ private:
std::condition_variable cv;
bool ready;
+ std::atomic<bool> shutting_down;
+ std::atomic<size_t> pending_writes;
+
public:
static constexpr const char* log_dir_name = "log";
@@ -134,7 +135,7 @@ public:
ReplicationState(Store* store, ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,
bool api_uses_ssl, size_t catchup_min_sequence_diff, size_t catch_up_threshold_percentage,
- bool create_init_db_snapshot, std::atomic<bool>& quit_service);
+ bool create_init_db_snapshot);
// Starts this node
int start(const butil::EndPoint & peering_endpoint, int api_port,
@@ -165,23 +166,7 @@ public:
uint64_t node_state() const;
// Shut this node down.
- void shutdown() {
- LOG(INFO) << "Replication state shutdown.";
- shut_down = true;
- if (node) {
- node->shutdown(nullptr);
- }
- }
-
- // Blocking this thread until the node is eventually down.
- void join() {
- std::unique_lock lock(mutex);
- if (node) {
- node->join();
- delete node;
- node = nullptr;
- }
- }
+ void shutdown();
int init_db();
diff --git a/include/threadpool.h b/include/threadpool.h
index ec98b9d2..dcfa4df4 100644
--- a/include/threadpool.h
+++ b/include/threadpool.h
@@ -11,7 +11,7 @@ public:
explicit ThreadPool(size_t);
template<class F, class... Args>
decltype(auto) enqueue(F&& f, Args&&... args);
- ~ThreadPool();
+ void shutdown();
private:
// need to keep track of threads so we can join them
std::vector< std::thread > workers;
@@ -39,9 +39,13 @@ inline ThreadPool::ThreadPool(size_t threads)
std::unique_lock<std::mutex> lock(this->queue_mutex);
this->condition.wait(lock,
[this]{ return this->stop || !this->tasks.empty(); });
- if(this->stop && this->tasks.empty()) {
+ if(this->stop) {
return;
}
+
+ if(this->tasks.empty()) {
+ continue;
+ }
task = std::move(this->tasks.front());
this->tasks.pop();
}
@@ -67,16 +71,15 @@ decltype(auto) ThreadPool::enqueue(F&& f, Args&&... args)
std::unique_lock<std::mutex> lock(queue_mutex);
// don't allow enqueueing after stopping the pool
- if(stop)
- throw std::runtime_error("enqueue on stopped ThreadPool");
-
- tasks.emplace(std::move(task));
+ if(!stop) {
+ tasks.emplace(std::move(task));
+ }
}
condition.notify_one();
return res;
}
-inline ThreadPool::~ThreadPool() {
+inline void ThreadPool::shutdown() {
{
std::unique_lock<std::mutex> lock(queue_mutex);
stop = true;
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 78906027..adb8f1d3 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1070,8 +1070,8 @@ bool del_key(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_re
}
bool raft_write_send_response(void *data) {
- //LOG(INFO) << "raft_write_send_response called";
request_response_t* index_arg = static_cast<request_response_t*>(data);
+ //LOG(INFO) << "raft_write_send_response " << index_arg->res.get();
std::unique_ptr<request_response_t> index_arg_guard(index_arg);
bool async_res = false;
diff --git a/src/http_server.cpp b/src/http_server.cpp
index f42e7bf7..cb433a7e 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -240,8 +240,8 @@ uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, con
}
void HttpServer::on_res_generator_dispose(void *self) {
- //LOG(INFO) << "on_res_generator_dispose fires";
h2o_custom_generator_t* custom_generator = *static_cast<h2o_custom_generator_t**>(self);
+ //LOG(INFO) << "on_res_generator_dispose fires " << custom_generator->res().get();
destroy_request_response(custom_generator->req(), custom_generator->res());
/*LOG(INFO) << "Deleting custom_generator, res: " << custom_generator->res();
@@ -506,6 +506,7 @@ int HttpServer::process_request(const std::shared_ptr<http_req>& request, const
handler->http_server->get_thread_pool()->enqueue([http_server, rpath, message_dispatcher,
request, response]() {
// call the API handler
+ //LOG(INFO) << "Wait for response " << response.get() << ", action: " << rpath->_get_action();
(rpath->handler)(request, response);
if(!rpath->async_res) {
@@ -513,6 +514,7 @@ int HttpServer::process_request(const std::shared_ptr<http_req>& request, const
message_dispatcher->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, &req_res);
response->wait();
}
+ //LOG(INFO) << "Response done " << response.get();
});
return 0;
@@ -521,6 +523,7 @@ int HttpServer::process_request(const std::shared_ptr<http_req>& request, const
void HttpServer::on_deferred_process_request(h2o_timer_t *entry) {
h2o_custom_timer_t* custom_timer = reinterpret_cast<h2o_custom_timer_t*>(entry);
deferred_req_res_t* deferred_req_res = static_cast<deferred_req_res_t*>(custom_timer->data);
+ //LOG(INFO) << "on_deferred_process_request " << deferred_req_res->res.get();
route_path* found_rpath = nullptr;
deferred_req_res->server->get_route(deferred_req_res->req->route_hash, &found_rpath);
@@ -531,6 +534,8 @@ void HttpServer::on_deferred_process_request(h2o_timer_t *entry) {
void HttpServer::defer_processing(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res,
size_t timeout_ms) {
+ //LOG(INFO) << "defer_processing, exit_loop: " << exit_loop << ", res: " << res.get();
+
if(req->defer_timer.data == nullptr) {
auto deferred_req_res = new deferred_req_res_t{req, res, this};
req->defer_timer.data = deferred_req_res;
@@ -541,8 +546,6 @@ void HttpServer::defer_processing(const std::shared_ptr<http_req>& req, const st
h2o_timer_link(ctx.loop, timeout_ms, &req->defer_timer.timer);
- //LOG(INFO) << "defer_processing, exit_loop: " << exit_loop << ", res->await: " << res->await;
-
if(exit_loop) {
// otherwise, replication thread could be stuck waiting on a future
req->_req = nullptr;
@@ -634,7 +637,7 @@ void HttpServer::response_proceed(h2o_generator_t *generator, h2o_req_t *req) {
}
void HttpServer::stream_response(const std::shared_ptr<http_req>& request, const std::shared_ptr<http_res>& response) {
- //LOG(INFO) << "stream_response called";
+ //LOG(INFO) << "stream_response called " << response.get();
if(request->_req == nullptr) {
// raft log replay or when underlying request is aborted
@@ -658,8 +661,8 @@ void HttpServer::stream_response(const std::shared_ptr<http_req>& request, const
h2o_start_response(req, &custom_generator->super);
}
- //LOG(INFO) << "stream_response, body_size: " << response.body.size() << ", response_final="
- // << custom_generator->response->final;
+ /*LOG(INFO) << "stream_response, body_size: " << response->body.size() << ", response_final="
+ << custom_generator->response->final;*/
h2o_iovec_t body = h2o_strdup(&req->pool, response->body.c_str(), SIZE_MAX);
response->body = "";
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 8ca37841..d0eb94a7 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -115,6 +115,10 @@ std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_end
}
void ReplicationState::write(const std::shared_ptr<http_req>& request, const std::shared_ptr<http_res>& response) {
+ if(shutting_down) {
+ return ;
+ }
+
std::shared_lock lock(mutex);
if(!node) {
@@ -142,6 +146,7 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const std
task.expected_term = leader_term.load(butil::memory_order_relaxed);
//LOG(INFO) << ":::" << "body size before apply: " << request->body.size();
+ pending_writes++;
// Now the task is applied to the group, waiting for the result.
return node->apply(task);
@@ -293,9 +298,14 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
request_response_t* replication_arg = new request_response_t{request_generated, response_generated};
message_dispatcher->send_message(REPLICATION_MSG, replication_arg);
- //LOG(INFO) << "Raft write waiting to proceed response->final=" << response_generated->final;
+ //LOG(INFO) << "Raft write pre wait " << response_generated.get();
response_generated->wait();
- //LOG(INFO) << "Raft write ready to proceed, response->final=" << response_generated->final;
+ //LOG(INFO) << "Raft write post wait " << response_generated.get();
+
+ if(iter.done()) {
+ pending_writes--;
+ //LOG(INFO) << "pending_writes: " << pending_writes;
+ }
}
}
@@ -527,12 +537,12 @@ void ReplicationState::refresh_nodes(const std::string & nodes) {
ReplicationState::ReplicationState(Store *store, ThreadPool* thread_pool, http_message_dispatcher *message_dispatcher,
bool api_uses_ssl, size_t catchup_min_sequence_diff, size_t catch_up_threshold_percentage,
- bool create_init_db_snapshot, std::atomic<bool>& quit_service):
+ bool create_init_db_snapshot):
node(nullptr), leader_term(-1), store(store), thread_pool(thread_pool),
message_dispatcher(message_dispatcher),
catchup_min_sequence_diff(catchup_min_sequence_diff), catch_up_threshold_percentage(catch_up_threshold_percentage),
- api_uses_ssl(api_uses_ssl), create_init_db_snapshot(create_init_db_snapshot), shut_down(quit_service),
- ready(false) {
+ api_uses_ssl(api_uses_ssl), create_init_db_snapshot(create_init_db_snapshot),
+ ready(false), shutting_down(false), pending_writes(0) {
}
@@ -621,6 +631,32 @@ Store* ReplicationState::get_store() {
return store;
}
+void ReplicationState::shutdown() {
+ LOG(INFO) << "Set shutting_down = true";
+ shutting_down = true;
+
+ // wait for pending writes to drop to zero
+ LOG(INFO) << "Waiting for in-flight writes to finish...";
+ while(pending_writes.load() != 0) {
+ LOG(INFO) << "pending_writes: " << pending_writes;
+ std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+ }
+
+ LOG(INFO) << "Replication state shutdown, store sequence: " << store->get_latest_seq_number();
+ std::unique_lock lock(mutex);
+
+ if (node) {
+ LOG(INFO) << "node->shutdown";
+ node->shutdown(nullptr);
+
+ // Blocking this thread until the node is eventually down.
+ LOG(INFO) << "node->join";
+ node->join();
+ delete node;
+ node = nullptr;
+ }
+}
+
void InitSnapshotClosure::Run() {
// Auto delete this after Run()
std::unique_ptr<InitSnapshotClosure> self_guard(this);
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 907e4d44..39f1629d 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -285,16 +285,15 @@ int start_raft_server(ReplicationState& replication_state, const std::string& st
// Stop application before server
replication_state.shutdown();
+
+ LOG(INFO) << "raft_server.stop()";
raft_server.Stop(0);
- // Wait until all the processing tasks are over.
- replication_state.join();
+ LOG(INFO) << "raft_server.join()";
raft_server.Join();
LOG(INFO) << "Typesense peering service has quit.";
- server->stop();
-
return 0;
}
@@ -400,15 +399,25 @@ int run_server(const Config & config, const std::string & version, void (*master
ReplicationState replication_state(&store, &app_thread_pool, server->get_message_dispatcher(),
ssl_enabled, config.get_catch_up_min_sequence_diff(),
config.get_catch_up_threshold_percentage(),
- create_init_db_snapshot, quit_raft_service);
+ create_init_db_snapshot);
- std::thread raft_thread([&replication_state, &config, &state_dir]() {
+ std::thread raft_thread([&replication_state, &config, &state_dir, &app_thread_pool, &server_thread_pool]() {
std::string path_to_nodes = config.get_nodes();
start_raft_server(replication_state, state_dir, path_to_nodes,
config.get_peering_address(),
config.get_peering_port(),
config.get_api_port(),
config.get_snapshot_interval_seconds());
+
+ LOG(INFO) << "Shutting down server_thread_pool";
+
+ server_thread_pool.shutdown();
+
+ LOG(INFO) << "Shutting down app_thread_pool.";
+
+ app_thread_pool.shutdown();
+
+ server->stop();
});
LOG(INFO) << "Starting API service...";
@@ -422,10 +431,19 @@ int run_server(const Config & config, const std::string & version, void (*master
quit_raft_service = true; // we set this once again in case API thread crashes instead of a signal
raft_thread.join();
+ LOG(INFO) << "CURL clean up";
+
curl_global_cleanup();
+ LOG(INFO) << "Deleting server";
+
delete server;
+
+ LOG(INFO) << "CollectionManager dispose, this might take some time...";
+
CollectionManager::get_instance().dispose();
+ LOG(INFO) << "Bye.";
+
return ret_code;
}