Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Move write log persistence logic to batched indexer.

Kishore Nallanb2acf289accd
diff --git a/include/batched_indexer.h b/include/batched_indexer.h
index 0aa5530e..60e162d4 100644
--- a/include/batched_indexer.h
+++ b/include/batched_indexer.h
@@ -39,6 +39,7 @@ private:
HttpServer* server;
Store* store;
+ Store* meta_store;
const size_t num_threads;
@@ -60,6 +61,12 @@ private:
std::atomic<bool> quit;
std::shared_mutex pause_mutex;
+ // Used to skip over a bad raft log entry which previously triggered a crash
+ const static int64_t UNSET_SKIP_INDEX = -9999;
+ std::atomic<int64_t> skip_index = UNSET_SKIP_INDEX;
+ rocksdb::Iterator* skip_index_iter = nullptr;
+ static constexpr const char* SKIP_INDICES_PREFIX = "$XP";
+
static const size_t GC_INTERVAL_SECONDS = 60;
static const size_t GC_PRUNE_MAX_SECONDS = 3600;
@@ -69,7 +76,7 @@ public:
static const constexpr char* RAFT_REQ_LOG_PREFIX = "$RL_";
- BatchedIndexer(HttpServer* server, Store* store, size_t num_threads);
+ BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, size_t num_threads);
~BatchedIndexer();
@@ -81,6 +88,12 @@ public:
void stop();
+ void populate_skip_index();
+
+ void persist_applying_index();
+
+ void clear_skip_indices();
+
// requires external synchronization!
void serialize_state(nlohmann::json& state);
diff --git a/include/raft_server.h b/include/raft_server.h
index a82196d6..c69344a3 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -86,7 +86,6 @@ public:
class ReplicationState : public braft::StateMachine {
private:
static constexpr const char* db_snapshot_name = "db_snapshot";
- static constexpr const char* SKIP_INDICES_PREFIX = "$XP";
mutable std::shared_mutex node_mutex;
@@ -97,16 +96,10 @@ private:
BatchedIndexer* batched_indexer;
Store* store;
- Store* meta_store;
ThreadPool* thread_pool;
http_message_dispatcher* message_dispatcher;
- // Used to skip over a bad raft log entry which previously triggered a crash
- const static int64_t UNSET_SKIP_INDEX = -9999;
- std::atomic<int64_t> skip_index = UNSET_SKIP_INDEX;
- rocksdb::Iterator* skip_index_iter = nullptr;
-
const bool api_uses_ssl;
const Config* config;
@@ -136,7 +129,7 @@ public:
static constexpr const char* meta_dir_name = "meta";
static constexpr const char* snapshot_dir_name = "snapshot";
- ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer, Store* store, Store* meta_store,
+ ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer, Store* store,
ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,
bool api_uses_ssl, const Config* config,
size_t num_collections_parallel_load, size_t num_documents_parallel_load);
@@ -273,6 +266,4 @@ private:
std::string get_leader_url_path(const std::string& leader_addr, const std::string& path,
const std::string& protocol) const;
-
- void populate_skip_index();
};
diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cpp
index cb5b5403..0dcc7fd2 100644
--- a/src/batched_indexer.cpp
+++ b/src/batched_indexer.cpp
@@ -2,8 +2,8 @@
#include "core_api.h"
#include "thread_local_vars.h"
-BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, const size_t num_threads):
- server(server), store(store), num_threads(num_threads),
+BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, Store* meta_store, const size_t num_threads):
+ server(server), store(store), meta_store(meta_store), num_threads(num_threads),
last_gc_run(std::chrono::high_resolution_clock::now()), quit(false) {
queues.resize(num_threads);
qmutuxes = new await_t[num_threads];
@@ -118,6 +118,11 @@ void BatchedIndexer::run() {
LOG(INFO) << "Starting batch indexer with " << num_threads << " threads.";
ThreadPool* thread_pool = new ThreadPool(num_threads);
+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);
+ populate_skip_index();
+
+ LOG(INFO) << "BatchedIndexer skip_index: " << skip_index;
+
for(size_t i = 0; i < num_threads; i++) {
std::deque<uint64_t>& queue = queues[i];
await_t& queue_mutex = qmutuxes[i];
@@ -178,24 +183,32 @@ void BatchedIndexer::run() {
// update thread local for reference during a crash
write_log_index = orig_req->log_index;
- //LOG(INFO) << "original request: " << orig_req_res.req << ", req: " << orig_req_res.req->req;
-
- if(route_found) {
- async_res = found_rpath->async_res;
- found_rpath->handler(orig_req, orig_res);
- prev_body = orig_req->body;
- } else {
- orig_res->set_404();
- }
-
- if(is_live_req && (!route_found ||!async_res)) {
- // sync request get a response immediately
- async_req_res_t* async_req_res = new async_req_res_t(orig_req, orig_res, true);
- server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, async_req_res);
+ if(write_log_index == skip_index) {
+ LOG(ERROR) << "Skipping write log index " << write_log_index
+ << " which seems to have triggered a crash previously.";
+ populate_skip_index();
}
- if(!route_found) {
- break;
+ else {
+ //LOG(INFO) << "original request: " << orig_req_res.req << ", req: " << orig_req_res.req->req;
+
+ if(route_found) {
+ async_res = found_rpath->async_res;
+ found_rpath->handler(orig_req, orig_res);
+ prev_body = orig_req->body;
+ } else {
+ orig_res->set_404();
+ }
+
+ if(is_live_req && (!route_found ||!async_res)) {
+ // sync request get a response immediately
+ async_req_res_t* async_req_res = new async_req_res_t(orig_req, orig_res, true);
+ server->get_message_dispatcher()->send_message(HttpServer::STREAM_RESPONSE_MESSAGE, async_req_res);
+ }
+
+ if(!route_found) {
+ break;
+ }
}
queued_writes--;
@@ -280,6 +293,7 @@ std::string BatchedIndexer::get_req_prefix_key(uint64_t req_id) {
BatchedIndexer::~BatchedIndexer() {
delete [] qmutuxes;
+ delete skip_index_iter;
}
void BatchedIndexer::stop() {
@@ -290,6 +304,25 @@ int64_t BatchedIndexer::get_queued_writes() {
return queued_writes;
}
+void BatchedIndexer::populate_skip_index() {
+ if(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {
+ const std::string& index_value = skip_index_iter->value().ToString();
+ if(StringUtils::is_int64_t(index_value)) {
+ skip_index = std::stoll(index_value);
+ }
+
+ skip_index_iter->Next();
+ } else {
+ skip_index = UNSET_SKIP_INDEX;
+ }
+}
+
+void BatchedIndexer::persist_applying_index() {
+ LOG(INFO) << "Saving currently applying index: " << write_log_index;
+ std::string key = SKIP_INDICES_PREFIX + std::to_string(write_log_index);
+ meta_store->insert(key, std::to_string(write_log_index));
+}
+
void BatchedIndexer::serialize_state(nlohmann::json& state) {
// requires external synchronization!
state["queued_writes"] = queued_writes.load();
@@ -367,3 +400,15 @@ void BatchedIndexer::load_state(const nlohmann::json& state) {
std::shared_mutex& BatchedIndexer::get_pause_mutex() {
return pause_mutex;
}
+
+void BatchedIndexer::clear_skip_indices() {
+ delete skip_index_iter;
+ skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);
+
+ while(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {
+ meta_store->remove(skip_index_iter->key().ToString());
+ skip_index_iter->Next();
+ }
+
+ meta_store->flush();
+}
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 467ac85a..dc5622f6 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -124,28 +124,12 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int
braft::NodeStatus node_status;
node->get_status(&node_status);
- skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);
- populate_skip_index();
-
- LOG(INFO) << "Node last_index: " << node_status.last_index << ", skip_index: " << skip_index;
+ LOG(INFO) << "Node last_index: " << node_status.last_index;
this->node = node;
return 0;
}
-void ReplicationState::populate_skip_index() {
- if(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {
- const std::string& index_value = skip_index_iter->value().ToString();
- if(StringUtils::is_int64_t(index_value)) {
- skip_index = std::stoll(index_value);
- }
-
- skip_index_iter->Next();
- } else {
- skip_index = UNSET_SKIP_INDEX;
- }
-}
-
std::string ReplicationState::to_nodes_config(const butil::EndPoint& peering_endpoint, const int api_port,
const std::string& nodes_config) {
if(nodes_config.empty()) {
@@ -345,13 +329,6 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
// Guard invokes replication_arg->done->Run() asynchronously to avoid the callback blocking the main thread
braft::AsyncClosureGuard closure_guard(iter.done());
- if(iter.index() == skip_index) {
- LOG(ERROR) << "Skipping write log index " << iter.index()
- << " which seems to have triggered a crash previously.";
- populate_skip_index();
- continue;
- }
-
//LOG(INFO) << "Apply entry";
const std::shared_ptr<http_req>& request_generated = iter.done() ?
@@ -457,6 +434,10 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Cl
batched_indexer->serialize_state(batch_index_state);
store->insert(CollectionManager::BATCHED_INDEXER_STATE_KEY, batch_index_state.dump());
+ // we will delete all the skip indices in meta store and flush that DB
+ // this will block writes, but should be pretty fast
+ batched_indexer->clear_skip_indices();
+
rocksdb::Checkpoint* checkpoint = nullptr;
rocksdb::Status status = store->create_check_point(&checkpoint, db_snapshot_path);
std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);
@@ -479,18 +460,6 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Cl
ext_snapshot_path = "";
}
- // we will also delete all the skip indices in meta store and flush that DB
- // this will block raft writes, but should be pretty fast
- delete skip_index_iter;
- skip_index_iter = meta_store->scan(SKIP_INDICES_PREFIX);
-
- while(skip_index_iter->Valid() && skip_index_iter->key().starts_with(SKIP_INDICES_PREFIX)) {
- meta_store->remove(skip_index_iter->key().ToString());
- skip_index_iter->Next();
- }
-
- meta_store->flush();
-
// Start a new bthread to avoid blocking StateMachine for slower operations that don't need a blocking view
bthread_t tid;
bthread_start_urgent(&tid, NULL, save_snapshot, arg);
@@ -650,12 +619,12 @@ void ReplicationState::refresh_catchup_status(bool log_msg) {
}
ReplicationState::ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer,
- Store *store, Store* meta_store, ThreadPool* thread_pool,
+ Store *store, ThreadPool* thread_pool,
http_message_dispatcher *message_dispatcher,
bool api_uses_ssl, const Config* config,
size_t num_collections_parallel_load, size_t num_documents_parallel_load):
node(nullptr), leader_term(-1), server(server), batched_indexer(batched_indexer),
- store(store), meta_store(meta_store),
+ store(store),
thread_pool(thread_pool), message_dispatcher(message_dispatcher), api_uses_ssl(api_uses_ssl),
config(config),
num_collections_parallel_load(num_collections_parallel_load),
@@ -767,8 +736,6 @@ void ReplicationState::shutdown() {
delete node;
node = nullptr;
}
-
- delete skip_index_iter;
}
void ReplicationState::persist_applying_index() {
@@ -780,10 +747,7 @@ void ReplicationState::persist_applying_index() {
lock.unlock();
- LOG(INFO) << "Saving currently applying index: " << write_log_index;
-
- std::string key = SKIP_INDICES_PREFIX + std::to_string(write_log_index);
- meta_store->insert(key, std::to_string(write_log_index));
+ batched_indexer->persist_applying_index();
}
int64_t ReplicationState::get_num_queued_writes() {
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 9c6ce5e0..0eaedd60 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -388,7 +388,7 @@ int run_server(const Config & config, const std::string & version, void (*master
bool ssl_enabled = (!config.get_ssl_cert().empty() && !config.get_ssl_cert_key().empty());
- BatchedIndexer* batch_indexer = new BatchedIndexer(server, &store, num_threads);
+ BatchedIndexer* batch_indexer = new BatchedIndexer(server, &store, &meta_store, num_threads);
CollectionManager & collectionManager = CollectionManager::get_instance();
collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),
@@ -396,7 +396,7 @@ int run_server(const Config & config, const std::string & version, void (*master
// first we start the peering service
- ReplicationState replication_state(server, batch_indexer, &store, &meta_store,
+ ReplicationState replication_state(server, batch_indexer, &store,
&app_thread_pool, server->get_message_dispatcher(),
ssl_enabled,
&config,