Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Recover gracefully from a crash during write.
Kishore Nallanaa1cd0acd19f
diff --git a/CMakeLists.txt b/CMakeLists.txtindex b5a71859..13171754 100644--- a/CMakeLists.txt+++ b/CMakeLists.txt@@ -35,6 +35,8 @@ if (APPLE)set(SNAPPY_ROOT_DIR /usr/local/opt/snappy)set(ZLIB_ROOT /usr/local/opt/zlib)set(CMAKE_PREFIX_PATH /usr/local/opt/curl-openssl /usr/local/opt/icu4c)++ add_compile_definitions(USE_BACKWARD=1)else()add_compile_definitions(BACKWARD_HAS_DW=1)endif()diff --git a/cmake/Modules/FindJemalloc.cmake b/cmake/Modules/FindJemalloc.cmakeindex f27b3b36..491ec4bd 100644--- a/cmake/Modules/FindJemalloc.cmake+++ b/cmake/Modules/FindJemalloc.cmake@@ -28,7 +28,7 @@ find_library(JEMALLOC_LIBRARIESNO_DEFAULT_PATH)include(FindPackageHandleStandardArgs)-find_package_handle_standard_args(jemalloc DEFAULT_MSG+find_package_handle_standard_args(Jemalloc DEFAULT_MSGJEMALLOC_LIBRARIESJEMALLOC_INCLUDE_DIR)diff --git a/cmake/Modules/FindNGHTTP2.cmake b/cmake/Modules/FindNGHTTP2.cmakeindex 47f37ad4..5698541e 100644--- a/cmake/Modules/FindNGHTTP2.cmake+++ b/cmake/Modules/FindNGHTTP2.cmake@@ -4,7 +4,7 @@ find_path(NGHTTP2_INCLUDE_DIR "nghttp2/nghttp2.h")find_library(NGHTTP2_LIBRARY NAMES nghttp2)-find_package_handle_standard_args(NGHTTP2+find_package_handle_standard_args(ngHTTP2FOUND_VARNGHTTP2_FOUNDREQUIRED_VARSdiff --git a/include/backward.hpp b/include/backward.hppindex 3c418d48..14a58a69 100644--- a/include/backward.hpp+++ b/include/backward.hpp@@ -4103,7 +4103,8 @@ private:sizeof posix_signals / sizeof posix_signals[0]);}- SignalHandling(const std::vector<int> &posix_signals = make_default_signals())+ SignalHandling(const std::vector<int> &posix_signals = make_default_signals(),+ void (*callback)(StackTrace&) = nullptr): _loaded(false) {bool success = true;@@ -4147,6 +4148,8 @@ private:bool loaded() const { return _loaded; }+ static void (*_callback)(StackTrace&);+static void handleSignal(int, siginfo_t *info, void *_ctx) {ucontext_t *uctx = static_cast<ucontext_t *>(_ctx);@@ -4186,6 +4189,10 @@ private:st.load_here(32, reinterpret_cast<void *>(uctx), info->si_addr);}+ if(_callback) {+ _callback(st);+ }+Printer printer;printer.address = true;//printer.print(st, stderr);diff --git a/include/http_server.h b/include/http_server.hindex 3e384342..e402cb2c 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -217,4 +217,6 @@ public:void do_snapshot(const std::string& snapshot_path, const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);bool trigger_vote();++ void persist_applying_index();};diff --git a/include/raft_server.h b/include/raft_server.hindex aa3ef0d5..e84434ac 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -85,6 +85,7 @@ public:class ReplicationState : public braft::StateMachine {private:static constexpr const char* db_snapshot_name = "db_snapshot";+ static constexpr const char* SKIP_INDICES_PREFIX = "$XP";mutable std::shared_mutex node_mutex;@@ -93,10 +94,20 @@ private:std::set<braft::PeerId> peers;HttpServer* server;- Store *store;++ Store* store;+ Store* meta_store;+ThreadPool* thread_pool;http_message_dispatcher* message_dispatcher;+ // Used to skip over a bad raft log entry which previously triggered a crash+ const static int64_t UNSET_SKIP_INDEX = -9999;+ std::atomic<int64_t> skip_index = UNSET_SKIP_INDEX;+ rocksdb::Iterator* skip_index_iter = nullptr;++ const bool api_uses_ssl;+const size_t read_max_lag;const size_t write_max_lag;@@ -106,8 +117,6 @@ private:std::atomic<bool> read_caught_up;std::atomic<bool> write_caught_up;- const bool api_uses_ssl;-std::string raft_dir_path;std::string ext_snapshot_path;@@ -127,7 +136,8 @@ public:static constexpr const char* meta_dir_name = "meta";static constexpr const char* snapshot_dir_name = "snapshot";- ReplicationState(HttpServer* server, Store* store, ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,+ ReplicationState(HttpServer* server, Store* store, Store* meta_store,+ ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,bool api_uses_ssl, size_t read_max_lag, size_t write_max_lag,size_t num_collections_parallel_load, size_t num_documents_parallel_load);@@ -181,6 +191,8 @@ public:const std::string& get_ext_snapshot_path() const;+ void persist_applying_index();+http_message_dispatcher* get_message_dispatcher() const;void wait() {@@ -254,4 +266,6 @@ private:std::string get_leader_url_path(const std::string& leader_addr, const std::string& path,const std::string& protocol) const;++ void populate_skip_index();};diff --git a/src/http_server.cpp b/src/http_server.cppindex 7b7395b9..d6e1c8f6 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -967,3 +967,7 @@ bool HttpServer::initialize_ssl_ctx(const char *cert_file, const char *key_file,return true;}+void HttpServer::persist_applying_index() {+ return replication_state->persist_applying_index();+}+diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 378afeeb..a2645912 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -72,6 +72,19 @@ void master_server_routes() {server->post("/config", post_config, false, false);}+void (*backward::SignalHandling::_callback)(backward::StackTrace&) = nullptr;++void crash_callback(backward::StackTrace& st) {+ backward::TraceResolver tr; tr.load_stacktrace(st);+ for (size_t i = 0; i < st.size(); ++i) {+ backward::ResolvedTrace trace = tr.resolve(st[i]);+ if(trace.object_function.find("ReplicationState::on_apply") != std::string::npos) {+ server->persist_applying_index();+ break;+ }+ }+}+int main(int argc, char **argv) {#ifdef __APPLE__// On OS X, je_zone_register registers jemalloc with the system allocator.@@ -109,12 +122,18 @@ int main(int argc, char **argv) {}#ifdef __APPLE__- signal(SIGABRT, StackPrinter::bt_sighandler);- signal(SIGFPE, StackPrinter::bt_sighandler);- signal(SIGILL, StackPrinter::bt_sighandler);- signal(SIGSEGV, StackPrinter::bt_sighandler);+ #ifdef USE_BACKWARD+ backward::SignalHandling sh;+ sh._callback = crash_callback;+ #else+ signal(SIGABRT, StackPrinter::bt_sighandler);+ signal(SIGFPE, StackPrinter::bt_sighandler);+ signal(SIGILL, StackPrinter::bt_sighandler);+ signal(SIGSEGV, StackPrinter::bt_sighandler);+ #endif#elif __linux__backward::SignalHandling sh;+ sh._callback = crash_callback;#endif// we can install new signal handlers only after overriding abovediff --git a/src/raft_server.cpp b/src/raft_server.cppindex a44ac86d..afe23552 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -96,13 +96,31 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const intreturn -1;}- std::vector<std::string> peer_vec;- StringUtils::split(actual_nodes_config, peer_vec, ",");+ braft::NodeStatus node_status;+ node->get_status(&node_status);++ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);+ populate_skip_index();++ LOG(INFO) << "Node last_index: " << node_status.last_index << ", skip_index: " << skip_index;this->node = node;return 0;}+void ReplicationState::populate_skip_index() {+ if(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {+ const std::string& index_value = skip_index_iter->value().ToString();+ if(StringUtils::is_int64_t(index_value)) {+ skip_index = std::stoll(index_value);+ }++ skip_index_iter->Next();+ } else {+ skip_index = UNSET_SKIP_INDEX;+ }+}+std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_endpoint, const int api_port,const std::string& nodes_config) {std::string actual_nodes_config = nodes_config;@@ -272,6 +290,13 @@ void ReplicationState::on_apply(braft::Iterator& iter) {// Guard invokes replication_arg->done->Run() asynchronously to avoid the callback blocking the main threadbraft::AsyncClosureGuard closure_guard(iter.done());+ if(iter.index() == skip_index) {+ LOG(ERROR) << "Skipping write log index " << iter.index()+ << " which seems to have triggered a crash previously.";+ populate_skip_index();+ continue;+ }+//LOG(INFO) << "Init use count: " << dynamic_cast<ReplicationClosure*>(iter.done())->get_request().use_count();const std::shared_ptr<http_req>& request_generated = iter.done() ?@@ -403,6 +428,18 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Clext_snapshot_path = "";}+ // we will also delete all the skip indices in meta store and flush that DB+ // this will block raft writes, but should be pretty fast+ delete skip_index_iter;+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);++ while(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {+ meta_store->remove(skip_index_iter->key().ToString());+ skip_index_iter->Next();+ }++ meta_store->flush();+// Start a new bthread to avoid blocking StateMachine for slower operations that don't need a blocking viewbthread_t tid;bthread_start_urgent(&tid, NULL, save_snapshot, arg);@@ -539,15 +576,16 @@ void ReplicationState::refresh_catchup_status(bool log_msg) {}}-ReplicationState::ReplicationState(HttpServer* server, Store *store, ThreadPool* thread_pool,+ReplicationState::ReplicationState(HttpServer* server, Store *store, Store* meta_store, ThreadPool* thread_pool,http_message_dispatcher *message_dispatcher,- bool api_uses_ssl, size_t read_max_lag, size_t write_max_lag,+ bool api_uses_ssl,+ size_t read_max_lag, size_t write_max_lag,size_t num_collections_parallel_load, size_t num_documents_parallel_load):- node(nullptr), leader_term(-1), server(server), store(store), thread_pool(thread_pool),- message_dispatcher(message_dispatcher), read_max_lag(read_max_lag), write_max_lag(write_max_lag),+ node(nullptr), leader_term(-1), server(server), store(store), meta_store(meta_store),+ thread_pool(thread_pool), message_dispatcher(message_dispatcher), api_uses_ssl(api_uses_ssl),+ read_max_lag(read_max_lag), write_max_lag(write_max_lag),num_collections_parallel_load(num_collections_parallel_load),num_documents_parallel_load(num_documents_parallel_load),- api_uses_ssl(api_uses_ssl),ready(false), shutting_down(false), pending_writes(0) {}@@ -666,6 +704,26 @@ void ReplicationState::shutdown() {delete node;node = nullptr;}++ delete skip_index_iter;+}++void ReplicationState::persist_applying_index() {+ std::shared_lock lock(node_mutex);++ if(node == nullptr) {+ return ;+ }++ braft::NodeStatus node_status;+ node->get_status(&node_status);++ lock.unlock();++ LOG(INFO) << "Saving currently applying index: " << node_status.applying_index;++ std::string key = SKIP_INDICES_PREFIX + std::to_string(node_status.applying_index);+ meta_store->insert(key, std::to_string(node_status.applying_index));}void OnDemandSnapshotClosure::Run() {diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 6adc338f..c34945ff 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -343,6 +343,8 @@ int run_server(const Config & config, const std::string & version, void (*masterstd::string data_dir = config.get_data_dir();std::string db_dir = config.get_data_dir() + "/db";std::string state_dir = config.get_data_dir() + "/state";+ std::string meta_dir = config.get_data_dir() + "/meta";+size_t thread_pool_size = config.get_thread_pool_size();const size_t proc_count = std::max<size_t>(1, std::thread::hardware_concurrency());@@ -355,8 +357,13 @@ int run_server(const Config & config, const std::string & version, void (*masterLOG(INFO) << "Thread pool size: " << num_threads;ThreadPool app_thread_pool(num_threads);ThreadPool server_thread_pool(num_threads);++ // primary DB used for storing the documents: we will not use WAL since Raft provides thatStore store(db_dir);+ // meta DB for storing house keeping things+ Store meta_store(meta_dir, 24*60*60, 1024, false);+CollectionManager & collectionManager = CollectionManager::get_instance();collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(), config.get_api_key());@@ -384,7 +391,7 @@ int run_server(const Config & config, const std::string & version, void (*master// first we start the peering service- ReplicationState replication_state(server, &store, &app_thread_pool, server->get_message_dispatcher(),+ ReplicationState replication_state(server, &store, &meta_store, &app_thread_pool, server->get_message_dispatcher(),ssl_enabled,config.get_read_max_lag(),config.get_write_max_lag(),