Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Move write log persistence logic to batched indexer.
Kishore Nallanb2acf289accd
diff --git a/include/batched_indexer.h b/include/batched_indexer.hindex 0aa5530e..60e162d4 100644--- a/include/batched_indexer.h+++ b/include/batched_indexer.h@@ -39,6 +39,7 @@ private:HttpServer* server;Store* store;+ Store* meta_store;const size_t num_threads;@@ -60,6 +61,12 @@ private:std::atomic<bool> quit;std::shared_mutex pause_mutex;+ // Used to skip over a bad raft log entry which previously triggered a crash+ const static int64_t UNSET_SKIP_INDEX = -9999;+ std::atomic<int64_t> skip_index = UNSET_SKIP_INDEX;+ rocksdb::Iterator* skip_index_iter = nullptr;+ static constexpr const char* SKIP_INDICES_PREFIX = "$XP";+static const size_t GC_INTERVAL_SECONDS = 60;static const size_t GC_PRUNE_MAX_SECONDS = 3600;@@ -69,7 +76,7 @@ public:static const constexpr char* RAFT_REQ_LOG_PREFIX = "$RL_";- BatchedIndexer(HttpServer* server, Store* store, size_t num_threads);+ BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, size_t num_threads);~BatchedIndexer();@@ -81,6 +88,12 @@ public:void stop();+ void populate_skip_index();++ void persist_applying_index();++ void clear_skip_indices();+// requires external synchronization!void serialize_state(nlohmann::json& state);diff --git a/include/raft_server.h b/include/raft_server.hindex a82196d6..c69344a3 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -86,7 +86,6 @@ public:class ReplicationState : public braft::StateMachine {private:static constexpr const char* db_snapshot_name = "db_snapshot";- static constexpr const char* SKIP_INDICES_PREFIX = "$XP";mutable std::shared_mutex node_mutex;@@ -97,16 +96,10 @@ private:BatchedIndexer* batched_indexer;Store* store;- Store* meta_store;ThreadPool* thread_pool;http_message_dispatcher* message_dispatcher;- // Used to skip over a bad raft log entry which previously triggered a crash- const static int64_t UNSET_SKIP_INDEX = -9999;- std::atomic<int64_t> skip_index = UNSET_SKIP_INDEX;- rocksdb::Iterator* skip_index_iter = nullptr;-const bool api_uses_ssl;const Config* config;@@ -136,7 +129,7 @@ public:static constexpr const char* meta_dir_name = "meta";static constexpr const char* snapshot_dir_name = "snapshot";- ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer, Store* store, Store* meta_store,+ ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer, Store* store,ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,bool api_uses_ssl, const Config* config,size_t num_collections_parallel_load, size_t num_documents_parallel_load);@@ -273,6 +266,4 @@ private:std::string get_leader_url_path(const std::string& leader_addr, const std::string& path,const std::string& protocol) const;-- void populate_skip_index();};diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cppindex cb5b5403..0dcc7fd2 100644--- a/src/batched_indexer.cpp+++ b/src/batched_indexer.cpp@@ -2,8 +2,8 @@#include "core_api.h"#include "thread_local_vars.h"-BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, const size_t num_threads):- server(server), store(store), num_threads(num_threads),+BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, const size_t num_threads):+ server(server), store(store), meta_store(meta_store), num_threads(num_threads),last_gc_run(std::chrono::high_resolution_clock::now()), quit(false) {queues.resize(num_threads);qmutuxes = new await_t[num_threads];@@ -118,6 +118,11 @@ void BatchedIndexer::run() {LOG(INFO) << "Starting batch indexer with " << num_threads << " threads.";ThreadPool* thread_pool = new ThreadPool(num_threads);+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);+ populate_skip_index();++ LOG(INFO) << "BatchedIndexer skip_index: " << skip_index;+for(size_t i = 0; i < num_threads; i++) {std::deque<uint64_t>& queue = queues[i];await_t& queue_mutex = qmutuxes[i];@@ -178,24 +183,32 @@ void BatchedIndexer::run() {// update thread local for reference during a crashwrite_log_index = orig_req->log_index;- //LOG(INFO) << "original request: " << orig_req_res.req << ", req: " << orig_req_res.req->req;-- if(route_found) {- async_res = found_rpath->async_res;- found_rpath->handler(orig_req, orig_res);- prev_body = orig_req->body;- } else {- orig_res->set_404();- }-- if(is_live_req && (!route_found ||!async_res)) {- // sync request get a response immediately- async_req_res_t* async_req_res = new async_req_res_t(orig_req, orig_res, true);- server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, async_req_res);+ if(write_log_index == skip_index) {+ LOG(ERROR) << "Skipping write log index " << write_log_index+ << " which seems to have triggered a crash previously.";+ populate_skip_index();}- if(!route_found) {- break;+ else {+ //LOG(INFO) << "original request: " << orig_req_res.req << ", req: " << orig_req_res.req->req;++ if(route_found) {+ async_res = found_rpath->async_res;+ found_rpath->handler(orig_req, orig_res);+ prev_body = orig_req->body;+ } else {+ orig_res->set_404();+ }++ if(is_live_req && (!route_found ||!async_res)) {+ // sync request get a response immediately+ async_req_res_t* async_req_res = new async_req_res_t(orig_req, orig_res, true);+ server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, async_req_res);+ }++ if(!route_found) {+ break;+ }}queued_writes--;@@ -280,6 +293,7 @@ std::string BatchedIndexer::get_req_prefix_key(uint64_t req_id) {BatchedIndexer::~BatchedIndexer() {delete [] qmutuxes;+ delete skip_index_iter;}void BatchedIndexer::stop() {@@ -290,6 +304,25 @@ int64_t BatchedIndexer::get_queued_writes() {return queued_writes;}+void BatchedIndexer::populate_skip_index() {+ if(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {+ const std::string& index_value = skip_index_iter->value().ToString();+ if(StringUtils::is_int64_t(index_value)) {+ skip_index = std::stoll(index_value);+ }++ skip_index_iter->Next();+ } else {+ skip_index = UNSET_SKIP_INDEX;+ }+}++void BatchedIndexer::persist_applying_index() {+ LOG(INFO) << "Saving currently applying index: " << write_log_index;+ std::string key = SKIP_INDICES_PREFIX + std::to_string(write_log_index);+ meta_store->insert(key, std::to_string(write_log_index));+}+void BatchedIndexer::serialize_state(nlohmann::json& state) {// requires external synchronization!state["queued_writes"] = queued_writes.load();@@ -367,3 +400,15 @@ void BatchedIndexer::load_state(const nlohmann::json& state) {std::shared_mutex& BatchedIndexer::get_pause_mutex() {return pause_mutex;}++void BatchedIndexer::clear_skip_indices() {+ delete skip_index_iter;+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);++ while(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {+ meta_store->remove(skip_index_iter->key().ToString());+ skip_index_iter->Next();+ }++ meta_store->flush();+}diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 467ac85a..dc5622f6 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -124,28 +124,12 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const intbraft::NodeStatus node_status;node->get_status(&node_status);- skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);- populate_skip_index();-- LOG(INFO) << "Node last_index: " << node_status.last_index << ", skip_index: " << skip_index;+ LOG(INFO) << "Node last_index: " << node_status.last_index;this->node = node;return 0;}-void ReplicationState::populate_skip_index() {- if(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {- const std::string& index_value = skip_index_iter->value().ToString();- if(StringUtils::is_int64_t(index_value)) {- skip_index = std::stoll(index_value);- }-- skip_index_iter->Next();- } else {- skip_index = UNSET_SKIP_INDEX;- }-}-std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_endpoint, const int api_port,const std::string& nodes_config) {if(nodes_config.empty()) {@@ -345,13 +329,6 @@ void ReplicationState::on_apply(braft::Iterator& iter) {// Guard invokes replication_arg->done->Run() asynchronously to avoid the callback blocking the main threadbraft::AsyncClosureGuard closure_guard(iter.done());- if(iter.index() == skip_index) {- LOG(ERROR) << "Skipping write log index " << iter.index()- << " which seems to have triggered a crash previously.";- populate_skip_index();- continue;- }-//LOG(INFO) << "Apply entry";const std::shared_ptr<http_req>& request_generated = iter.done() ?@@ -457,6 +434,10 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Clbatched_indexer->serialize_state(batch_index_state);store->insert(CollectionManager::BATCHED_INDEXER_STATE_KEY, batch_index_state.dump());+ // we will delete all the skip indices in meta store and flush that DB+ // this will block writes, but should be pretty fast+ batched_indexer->clear_skip_indices();+rocksdb::Checkpoint* checkpoint = nullptr;rocksdb::Status status = store->create_check_point(&checkpoint, db_snapshot_path);std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);@@ -479,18 +460,6 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Clext_snapshot_path = "";}- // we will also delete all the skip indices in meta store and flush that DB- // this will block raft writes, but should be pretty fast- delete skip_index_iter;- skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);-- while(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {- meta_store->remove(skip_index_iter->key().ToString());- skip_index_iter->Next();- }-- meta_store->flush();-// Start a new bthread to avoid blocking StateMachine for slower operations that don't need a blocking viewbthread_t tid;bthread_start_urgent(&tid, NULL, save_snapshot, arg);@@ -650,12 +619,12 @@ void ReplicationState::refresh_catchup_status(bool log_msg) {}ReplicationState::ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer,- Store *store, Store* meta_store, ThreadPool* thread_pool,+ Store *store, ThreadPool* thread_pool,http_message_dispatcher *message_dispatcher,bool api_uses_ssl, const Config* config,size_t num_collections_parallel_load, size_t num_documents_parallel_load):node(nullptr), leader_term(-1), server(server), batched_indexer(batched_indexer),- store(store), meta_store(meta_store),+ store(store),thread_pool(thread_pool), message_dispatcher(message_dispatcher), api_uses_ssl(api_uses_ssl),config(config),num_collections_parallel_load(num_collections_parallel_load),@@ -767,8 +736,6 @@ void ReplicationState::shutdown() {delete node;node = nullptr;}-- delete skip_index_iter;}void ReplicationState::persist_applying_index() {@@ -780,10 +747,7 @@ void ReplicationState::persist_applying_index() {lock.unlock();- LOG(INFO) << "Saving currently applying index: " << write_log_index;-- std::string key = SKIP_INDICES_PREFIX + std::to_string(write_log_index);- meta_store->insert(key, std::to_string(write_log_index));+ batched_indexer->persist_applying_index();}int64_t ReplicationState::get_num_queued_writes() {diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 9c6ce5e0..0eaedd60 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -388,7 +388,7 @@ int run_server(const Config & config, const std::string & version, void (*masterbool ssl_enabled = (!config.get_ssl_cert().empty() && !config.get_ssl_cert_key().empty());- BatchedIndexer* batch_indexer = new BatchedIndexer(server, &store, num_threads);+ BatchedIndexer* batch_indexer = new BatchedIndexer(server, &store, &meta_store, num_threads);CollectionManager & collectionManager = CollectionManager::get_instance();collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),@@ -396,7 +396,7 @@ int run_server(const Config & config, const std::string & version, void (*master// first we start the peering service- ReplicationState replication_state(server, batch_indexer, &store, &meta_store,+ ReplicationState replication_state(server, batch_indexer, &store,&app_thread_pool, server->get_message_dispatcher(),ssl_enabled,&config,