Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Recover gracefully from a crash during write.

Kishore Nallanaa1cd0acd19f
diff --git a/CMakeLists.txt b/CMakeLists.txt
index b5a71859..13171754 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -35,6 +35,8 @@ if (APPLE)
set(SNAPPY_ROOT_DIR /usr/local/opt/snappy)
set(ZLIB_ROOT /usr/local/opt/zlib)
set(CMAKE_PREFIX_PATH /usr/local/opt/curl-openssl /usr/local/opt/icu4c)
+
+ add_compile_definitions(USE_BACKWARD=1)
else()
add_compile_definitions(BACKWARD_HAS_DW=1)
endif()
diff --git a/cmake/Modules/FindJemalloc.cmake b/cmake/Modules/FindJemalloc.cmake
index f27b3b36..491ec4bd 100644
--- a/cmake/Modules/FindJemalloc.cmake
+++ b/cmake/Modules/FindJemalloc.cmake
@@ -28,7 +28,7 @@ find_library(JEMALLOC_LIBRARIES
NO_DEFAULT_PATH)
include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(jemalloc DEFAULT_MSG
+find_package_handle_standard_args(Jemalloc DEFAULT_MSG
JEMALLOC_LIBRARIES
JEMALLOC_INCLUDE_DIR)
diff --git a/cmake/Modules/FindNGHTTP2.cmake b/cmake/Modules/FindNGHTTP2.cmake
index 47f37ad4..5698541e 100644
--- a/cmake/Modules/FindNGHTTP2.cmake
+++ b/cmake/Modules/FindNGHTTP2.cmake
@@ -4,7 +4,7 @@ find_path(NGHTTP2_INCLUDE_DIR "nghttp2/nghttp2.h")
find_library(NGHTTP2_LIBRARY NAMES nghttp2)
-find_package_handle_standard_args(NGHTTP2
+find_package_handle_standard_args(ngHTTP2
FOUND_VAR
NGHTTP2_FOUND
REQUIRED_VARS
diff --git a/include/backward.hpp b/include/backward.hpp
index 3c418d48..14a58a69 100644
--- a/include/backward.hpp
+++ b/include/backward.hpp
@@ -4103,7 +4103,8 @@ private:
sizeof posix_signals / sizeof posix_signals[0]);
}
- SignalHandling(const std::vector<int> &posix_signals = make_default_signals())
+ SignalHandling(const std::vector<int> &posix_signals = make_default_signals(),
+ void (*callback)(StackTrace&) = nullptr)
: _loaded(false) {
bool success = true;
@@ -4147,6 +4148,8 @@ private:
bool loaded() const { return _loaded; }
+ static void (*_callback)(StackTrace&);
+
static void handleSignal(int, siginfo_t *info, void *_ctx) {
ucontext_t *uctx = static_cast<ucontext_t *>(_ctx);
@@ -4186,6 +4189,10 @@ private:
st.load_here(32, reinterpret_cast<void *>(uctx), info->si_addr);
}
+ if(_callback) {
+ _callback(st);
+ }
+
Printer printer;
printer.address = true;
//printer.print(st, stderr);
diff --git a/include/http_server.h b/include/http_server.h
index 3e384342..e402cb2c 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -217,4 +217,6 @@ public:
void do_snapshot(const std::string& snapshot_path, const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
bool trigger_vote();
+
+ void persist_applying_index();
};
diff --git a/include/raft_server.h b/include/raft_server.h
index aa3ef0d5..e84434ac 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -85,6 +85,7 @@ public:
class ReplicationState : public braft::StateMachine {
private:
static constexpr const char* db_snapshot_name = "db_snapshot";
+ static constexpr const char* SKIP_INDICES_PREFIX = "$XP";
mutable std::shared_mutex node_mutex;
@@ -93,10 +94,20 @@ private:
std::set<braft::PeerId> peers;
HttpServer* server;
- Store *store;
+
+ Store* store;
+ Store* meta_store;
+
ThreadPool* thread_pool;
http_message_dispatcher* message_dispatcher;
+ // Used to skip over a bad raft log entry which previously triggered a crash
+ const static int64_t UNSET_SKIP_INDEX = -9999;
+ std::atomic<int64_t> skip_index = UNSET_SKIP_INDEX;
+ rocksdb::Iterator* skip_index_iter = nullptr;
+
+ const bool api_uses_ssl;
+
const size_t read_max_lag;
const size_t write_max_lag;
@@ -106,8 +117,6 @@ private:
std::atomic<bool> read_caught_up;
std::atomic<bool> write_caught_up;
- const bool api_uses_ssl;
-
std::string raft_dir_path;
std::string ext_snapshot_path;
@@ -127,7 +136,8 @@ public:
static constexpr const char* meta_dir_name = "meta";
static constexpr const char* snapshot_dir_name = "snapshot";
- ReplicationState(HttpServer* server, Store* store, ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,
+ ReplicationState(HttpServer* server, Store* store, Store* meta_store,
+ ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,
bool api_uses_ssl, size_t read_max_lag, size_t write_max_lag,
size_t num_collections_parallel_load, size_t num_documents_parallel_load);
@@ -181,6 +191,8 @@ public:
const std::string& get_ext_snapshot_path() const;
+ void persist_applying_index();
+
http_message_dispatcher* get_message_dispatcher() const;
void wait() {
@@ -254,4 +266,6 @@ private:
std::string get_leader_url_path(const std::string& leader_addr, const std::string& path,
const std::string& protocol) const;
+
+ void populate_skip_index();
};
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 7b7395b9..d6e1c8f6 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -967,3 +967,7 @@ bool HttpServer::initialize_ssl_ctx(const char *cert_file, const char *key_file,
return true;
}
+void HttpServer::persist_applying_index() {
+ return replication_state->persist_applying_index();
+}
+
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 378afeeb..a2645912 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -72,6 +72,19 @@ void master_server_routes() {
server->post("/config", post_config, false, false);
}
+void (*backward::SignalHandling::_callback)(backward::StackTrace&) = nullptr;
+
+void crash_callback(backward::StackTrace& st) {
+ backward::TraceResolver tr; tr.load_stacktrace(st);
+ for (size_t i = 0; i < st.size(); ++i) {
+ backward::ResolvedTrace trace = tr.resolve(st[i]);
+ if(trace.object_function.find("ReplicationState::on_apply") != std::string::npos) {
+ server->persist_applying_index();
+ break;
+ }
+ }
+}
+
int main(int argc, char **argv) {
#ifdef __APPLE__
// On OS X, je_zone_register registers jemalloc with the system allocator.
@@ -109,12 +122,18 @@ int main(int argc, char **argv) {
}
#ifdef __APPLE__
- signal(SIGABRT, StackPrinter::bt_sighandler);
- signal(SIGFPE, StackPrinter::bt_sighandler);
- signal(SIGILL, StackPrinter::bt_sighandler);
- signal(SIGSEGV, StackPrinter::bt_sighandler);
+ #ifdef USE_BACKWARD
+ backward::SignalHandling sh;
+ sh._callback = crash_callback;
+ #else
+ signal(SIGABRT, StackPrinter::bt_sighandler);
+ signal(SIGFPE, StackPrinter::bt_sighandler);
+ signal(SIGILL, StackPrinter::bt_sighandler);
+ signal(SIGSEGV, StackPrinter::bt_sighandler);
+ #endif
#elif __linux__
backward::SignalHandling sh;
+ sh._callback = crash_callback;
#endif
// we can install new signal handlers only after overriding above
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index a44ac86d..afe23552 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -96,13 +96,31 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int
return -1;
}
- std::vector<std::string> peer_vec;
- StringUtils::split(actual_nodes_config, peer_vec, ",");
+ braft::NodeStatus node_status;
+ node->get_status(&node_status);
+
+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);
+ populate_skip_index();
+
+ LOG(INFO) << "Node last_index: " << node_status.last_index << ", skip_index: " << skip_index;
this->node = node;
return 0;
}
+void ReplicationState::populate_skip_index() {
+ if(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {
+ const std::string& index_value = skip_index_iter->value().ToString();
+ if(StringUtils::is_int64_t(index_value)) {
+ skip_index = std::stoll(index_value);
+ }
+
+ skip_index_iter->Next();
+ } else {
+ skip_index = UNSET_SKIP_INDEX;
+ }
+}
+
std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_endpoint, const int api_port,
const std::string& nodes_config) {
std::string actual_nodes_config = nodes_config;
@@ -272,6 +290,13 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
// Guard invokes replication_arg->done->Run() asynchronously to avoid the callback blocking the main thread
braft::AsyncClosureGuard closure_guard(iter.done());
+ if(iter.index() == skip_index) {
+ LOG(ERROR) << "Skipping write log index " << iter.index()
+ << " which seems to have triggered a crash previously.";
+ populate_skip_index();
+ continue;
+ }
+
//LOG(INFO) << "Init use count: " << dynamic_cast<ReplicationClosure*>(iter.done())->get_request().use_count();
const std::shared_ptr<http_req>& request_generated = iter.done() ?
@@ -403,6 +428,18 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Cl
ext_snapshot_path = "";
}
+ // we will also delete all the skip indices in meta store and flush that DB
+ // this will block raft writes, but should be pretty fast
+ delete skip_index_iter;
+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);
+
+ while(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {
+ meta_store->remove(skip_index_iter->key().ToString());
+ skip_index_iter->Next();
+ }
+
+ meta_store->flush();
+
// Start a new bthread to avoid blocking StateMachine for slower operations that don't need a blocking view
bthread_t tid;
bthread_start_urgent(&tid, NULL, save_snapshot, arg);
@@ -539,15 +576,16 @@ void ReplicationState::refresh_catchup_status(bool log_msg) {
}
}
-ReplicationState::ReplicationState(HttpServer* server, Store *store, ThreadPool* thread_pool,
+ReplicationState::ReplicationState(HttpServer* server, Store *store, Store* meta_store, ThreadPool* thread_pool,
http_message_dispatcher *message_dispatcher,
- bool api_uses_ssl, size_t read_max_lag, size_t write_max_lag,
+ bool api_uses_ssl,
+ size_t read_max_lag, size_t write_max_lag,
size_t num_collections_parallel_load, size_t num_documents_parallel_load):
- node(nullptr), leader_term(-1), server(server), store(store), thread_pool(thread_pool),
- message_dispatcher(message_dispatcher), read_max_lag(read_max_lag), write_max_lag(write_max_lag),
+ node(nullptr), leader_term(-1), server(server), store(store), meta_store(meta_store),
+ thread_pool(thread_pool), message_dispatcher(message_dispatcher), api_uses_ssl(api_uses_ssl),
+ read_max_lag(read_max_lag), write_max_lag(write_max_lag),
num_collections_parallel_load(num_collections_parallel_load),
num_documents_parallel_load(num_documents_parallel_load),
- api_uses_ssl(api_uses_ssl),
ready(false), shutting_down(false), pending_writes(0) {
}
@@ -666,6 +704,26 @@ void ReplicationState::shutdown() {
delete node;
node = nullptr;
}
+
+ delete skip_index_iter;
+}
+
+void ReplicationState::persist_applying_index() {
+ std::shared_lock lock(node_mutex);
+
+ if(node == nullptr) {
+ return ;
+ }
+
+ braft::NodeStatus node_status;
+ node->get_status(&node_status);
+
+ lock.unlock();
+
+ LOG(INFO) << "Saving currently applying index: " << node_status.applying_index;
+
+ std::string key = SKIP_INDICES_PREFIX + std::to_string(node_status.applying_index);
+ meta_store->insert(key, std::to_string(node_status.applying_index));
}
void OnDemandSnapshotClosure::Run() {
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 6adc338f..c34945ff 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -343,6 +343,8 @@ int run_server(const Config & config, const std::string & version, void (*master
std::string data_dir = config.get_data_dir();
std::string db_dir = config.get_data_dir() + "/db";
std::string state_dir = config.get_data_dir() + "/state";
+ std::string meta_dir = config.get_data_dir() + "/meta";
+
size_t thread_pool_size = config.get_thread_pool_size();
const size_t proc_count = std::max<size_t>(1, std::thread::hardware_concurrency());
@@ -355,8 +357,13 @@ int run_server(const Config & config, const std::string & version, void (*master
LOG(INFO) << "Thread pool size: " << num_threads;
ThreadPool app_thread_pool(num_threads);
ThreadPool server_thread_pool(num_threads);
+
+ // primary DB used for storing the documents: we will not use WAL since Raft provides that
Store store(db_dir);
+ // meta DB for storing house keeping things
+ Store meta_store(meta_dir, 24*60*60, 1024, false);
+
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(), config.get_api_key());
@@ -384,7 +391,7 @@ int run_server(const Config & config, const std::string & version, void (*master
// first we start the peering service
- ReplicationState replication_state(server, &store, &app_thread_pool, server->get_message_dispatcher(),
+ ReplicationState replication_state(server, &store, &meta_store, &app_thread_pool, server->get_message_dispatcher(),
ssl_enabled,
config.get_read_max_lag(),
config.get_write_max_lag(),