Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Restore store persisted raft log from snapshot.

Kishore Nallanbaaf7ea65db8
diff --git a/include/batched_indexer.h b/include/batched_indexer.h
index f1d41da8..ba6a9eac 100644
--- a/include/batched_indexer.h
+++ b/include/batched_indexer.h
@@ -9,50 +9,60 @@
class BatchedIndexer {
private:
- static const constexpr char* RAFT_REQ_LOG_PREFIX = "$RL_";
-
struct req_res_t {
+ std::string prev_req_body; // used to handle partial JSON documents caused by chunking
std::shared_ptr<http_req> req;
std::shared_ptr<http_res> res;
uint64_t batch_begin_ts;
- req_res_t(const std::shared_ptr<http_req>& req,
- const std::shared_ptr<http_res>& res, uint64_t batch_begin_ts):
- req(req), res(res), batch_begin_ts(batch_begin_ts) {
+ uint32_t num_chunks;
+ uint32_t next_chunk_index; // index where next read must begin
+ bool is_complete; // whether the req has been written to store fully
- }
-
- req_res_t() {
+ req_res_t(const std::string& prev_req_body,
+ const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res,
+ uint64_t batch_begin_ts, uint32_t num_chunks, uint32_t next_chunk_index, bool is_complete):
+ prev_req_body(prev_req_body), req(req), res(res), batch_begin_ts(batch_begin_ts),
+ num_chunks(num_chunks), next_chunk_index(next_chunk_index), is_complete(is_complete) {
}
+
+ req_res_t(): req(nullptr), res(nullptr), batch_begin_ts(0), num_chunks(0),
+ next_chunk_index(0), is_complete(false) {};
};
HttpServer* server;
Store* store;
- ThreadPool* thread_pool;
const size_t num_threads;
- std::vector<std::deque<uint64_t>> queues;
std::mutex* qmutuxes;
+ std::vector<std::deque<uint64_t>> queues;
+
+ /* Variables to be serialized on snapshot /
+ --------------------------------------------------------- */
std::mutex mutex;
- std::unordered_map<uint64_t, uint32_t> request_to_chunk;
std::unordered_map<uint64_t, req_res_t> req_res_map;
+ std::atomic<int64_t> queued_writes = 0;
+
+ /* ------------------------------------------------------- */
+
std::chrono::high_resolution_clock::time_point last_gc_run;
std::atomic<bool> quit;
+ std::shared_mutex pause_mutex;
static const size_t GC_INTERVAL_SECONDS = 60;
static const size_t GC_PRUNE_MAX_SECONDS = 3600;
static std::string get_req_prefix_key(uint64_t req_id);
- std::atomic<int64_t> queued_writes = 0;
-
public:
+ static const constexpr char* RAFT_REQ_LOG_PREFIX = "$RL_";
+
BatchedIndexer(HttpServer* server, Store* store, size_t num_threads);
~BatchedIndexer();
@@ -64,4 +74,13 @@ public:
void run();
void stop();
-};
\ No newline at end of file
+
+ // requires external synchronization!
+ void serialize_state(nlohmann::json& state);
+
+ void load_state(const nlohmann::json& state);
+
+ std::string get_collection_name(const std::shared_ptr<http_req>& req);
+
+ std::shared_mutex& get_pause_mutex();
+};
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 0d5b31a8..9de2ecf7 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -8,6 +8,7 @@
#include "collection.h"
#include "auth_manager.h"
#include "threadpool.h"
+#include "batched_indexer.h"
template<typename ResourceType>
struct locked_resource_view_t {
@@ -73,6 +74,8 @@ private:
std::atomic<bool>* quit;
+ BatchedIndexer* batch_indexer;
+
CollectionManager();
~CollectionManager() = default;
@@ -92,6 +95,7 @@ public:
static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";
static constexpr const char* SYMLINK_PREFIX = "$SL";
+ static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";
static CollectionManager & get_instance() {
static CollectionManager instance;
@@ -120,7 +124,7 @@ public:
// PUBLICLY EXPOSED API
void init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio,
- const std::string & auth_key, std::atomic<bool>& quit);
+ const std::string & auth_key, std::atomic<bool>& quit, BatchedIndexer* batch_indexer);
// only for tests!
void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit);
diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cpp
index 17c30014..a652650b 100644
--- a/src/batched_indexer.cpp
+++ b/src/batched_indexer.cpp
@@ -5,7 +5,6 @@
BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, const size_t num_threads):
server(server), store(store), num_threads(num_threads),
last_gc_run(std::chrono::high_resolution_clock::now()), quit(false) {
- thread_pool = new ThreadPool(num_threads);
queues.resize(num_threads);
qmutuxes = new std::mutex[num_threads];
}
@@ -15,68 +14,49 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh
// NOTE: it's ok to access `req` and `res` in this function without synchronization
// because the read thread for *this* request is paused now and resumes only messaged at the end
- std::string& coll_name = req->params["collection"];
-
- if(coll_name.empty()) {
- route_path* rpath;
- server->get_route(req->route_hash, &rpath);
-
- // ensure that collection creation is sent to the same queue as writes to that collection
- if(rpath->handler == post_create_collection) {
- nlohmann::json obj = nlohmann::json::parse(req->body, nullptr, false);
-
- if(obj != nlohmann::json::value_t::discarded && obj.is_object() &&
- obj.count("name") != 0 && obj["name"].is_string()) {
- coll_name = obj["name"];
- }
- }
- }
-
- uint64_t queue_id = StringUtils::hash_wy(coll_name.c_str(), coll_name.size()) % num_threads;
-
+ //LOG(INFO) << "BatchedIndexer::enqueue";
uint32_t chunk_sequence = 0;
{
std::unique_lock lk(mutex);
- chunk_sequence = request_to_chunk[req->start_ts];
- request_to_chunk[req->start_ts] += 1;
+ auto req_res_map_it = req_res_map.find(req->start_ts);
+
+ if(req_res_map_it == req_res_map.end()) {
+ // first chunk
+ uint64_t batch_begin_ts = std::chrono::duration_cast<std::chrono::seconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ req_res_t req_res("", req, res, batch_begin_ts, 1, 0, false);
+ req_res_map.emplace(req->start_ts, req_res);
+ } else {
+ chunk_sequence = req_res_map_it->second.num_chunks;
+ req_res_map_it->second.num_chunks += 1;
+ }
}
const std::string& req_key_prefix = get_req_prefix_key(req->start_ts);
const std::string& request_chunk_key = req_key_prefix + StringUtils::serialize_uint32_t(chunk_sequence);
- //LOG(INFO) << "req_id: " << req->start_ts << ", chunk_sequence: " << chunk_sequence;
+ //LOG(INFO) << "insert request_chunk_key: " << request_chunk_key;
store->insert(request_chunk_key, req->to_json());
req->body = "";
- {
- std::unique_lock lk(mutex);
- auto req_res_map_it = req_res_map.find(req->start_ts);
- if(req_res_map_it == req_res_map.end()) {
- uint64_t batch_begin_ts = std::chrono::duration_cast<std::chrono::seconds>(
- std::chrono::system_clock::now().time_since_epoch()).count();
-
- req_res_t req_res{req, res, batch_begin_ts};
- req_res_map[req->start_ts] = req_res;
- }
- }
-
bool is_old_serialized_request = (req->start_ts == 0);
bool read_more_input = (req->_req != nullptr && req->_req->proceed_req);
if(req->last_chunk_aggregate) {
- //LOG(INFO) << "Last chunk for req_id: " << req->start_ts << ", queue_id: " << queue_id;
-
+ //LOG(INFO) << "Last chunk for req_id: " << req->start_ts;
queued_writes += (chunk_sequence + 1);
- std::unique_lock lk(mutex);
- request_to_chunk.erase(req->start_ts);
- lk.unlock();
-
{
- std::unique_lock lk(qmutuxes[queue_id]);
+ const std::string& coll_name = get_collection_name(req);
+ uint64_t queue_id = StringUtils::hash_wy(coll_name.c_str(), coll_name.size()) % num_threads;
+ std::unique_lock lk1(qmutuxes[queue_id]);
queues[queue_id].emplace_back(req->start_ts);
+
+ std::unique_lock lk2(mutex);
+ req_res_map[req->start_ts].is_complete = true;
}
// IMPORTANT: must not read `req` variables (except _req) henceforth to prevent data races with indexing thread
@@ -86,11 +66,12 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh
// We can only do serial writes as we cannot reliably distinguish one streaming request from another.
// So, wait for `req_res_map` to be empty before proceeding
while(true) {
- lk.lock();
- if(req_res_map.empty()) {
- break;
+ {
+ std::unique_lock lk(mutex);
+ if(req_res_map.empty()) {
+ break;
+ }
}
- lk.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds (10));
}
@@ -104,8 +85,30 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh
}
}
+std::string BatchedIndexer::get_collection_name(const std::shared_ptr<http_req>& req) {
+ std::string& coll_name = req->params["collection"];
+
+ if(coll_name.empty()) {
+ route_path* rpath;
+ server->get_route(req->route_hash, &rpath);
+
+ // ensure that collection creation is sent to the same queue as writes to that collection
+ if(rpath->handler == post_create_collection) {
+ nlohmann::json obj = nlohmann::json::parse(req->body, nullptr, false);
+
+ if(obj != nlohmann::json::value_t::discarded && obj.is_object() &&
+ obj.count("name") != 0 && obj["name"].is_string()) {
+ coll_name = obj["name"];
+ }
+ }
+ }
+
+ return coll_name;
+}
+
void BatchedIndexer::run() {
LOG(INFO) << "Starting batch indexer with " << num_threads << " threads.";
+ ThreadPool* thread_pool = new ThreadPool(num_threads);
for(size_t i = 0; i < num_threads; i++) {
std::deque<uint64_t>& queue = queues[i];
@@ -129,14 +132,24 @@ void BatchedIndexer::run() {
// scan db for all logs associated with request
const std::string& req_key_prefix = get_req_prefix_key(req_id);
- rocksdb::Iterator* iter = store->scan(req_key_prefix);
- std::string prev_body; // used to handle partial JSON documents caused by chunking
+ const std::string& req_key_start_prefix = get_req_prefix_key(req_id) +
+ StringUtils::serialize_uint32_t(orig_req_res.next_chunk_index);
+
+ rocksdb::Iterator* iter = store->scan(req_key_start_prefix);
+
+ // used to handle partial JSON documents caused by chunking
+ std::string& prev_body = orig_req_res.prev_req_body;
- const std::shared_ptr<http_res>& orig_res = orig_req_res.res;
const std::shared_ptr<http_req>& orig_req = orig_req_res.req;
+ const std::shared_ptr<http_res>& orig_res = orig_req_res.res;
bool is_live_req = orig_res->is_alive;
+ route_path* found_rpath = nullptr;
+ bool route_found = server->get_route(orig_req->route_hash, &found_rpath);
+ bool async_res = false;
+
while(iter->Valid() && iter->key().starts_with(req_key_prefix)) {
+ std::shared_lock slk(pause_mutex); // used for snapshot
orig_req->body = prev_body;
orig_req->load_from_json(iter->value().ToString());
@@ -145,10 +158,6 @@ void BatchedIndexer::run() {
//LOG(INFO) << "original request: " << orig_req_res.req << ", req: " << orig_req_res.req->req;
- route_path* found_rpath = nullptr;
- bool route_found = server->get_route(orig_req->route_hash, &found_rpath);
- bool async_res = false;
-
if(route_found) {
async_res = found_rpath->async_res;
found_rpath->handler(orig_req, orig_res);
@@ -168,6 +177,7 @@ void BatchedIndexer::run() {
}
queued_writes--;
+ orig_req_res.next_chunk_index++;
iter->Next();
if(quit) {
@@ -177,7 +187,7 @@ void BatchedIndexer::run() {
delete iter;
- //LOG(INFO) << "Erasing request data from disk and memory for request " << req_res.req->start_ts;
+ //LOG(INFO) << "Erasing request data from disk and memory for request " << orig_req_res.req->start_ts;
// we can delete the buffered request content
store->delete_range(req_key_prefix, req_key_prefix + StringUtils::serialize_uint32_t(UINT32_MAX));
@@ -216,7 +226,6 @@ void BatchedIndexer::run() {
LOG(INFO) << "Deleting partial upload for req id " << it->second.req->start_ts;
const std::string& req_key_prefix = get_req_prefix_key(it->second.req->start_ts);
store->delete_range(req_key_prefix, req_key_prefix + StringUtils::serialize_uint32_t(UINT32_MAX));
- request_to_chunk.erase(it->second.req->start_ts);
it = req_res_map.erase(it);
} else {
it++;
@@ -229,6 +238,7 @@ void BatchedIndexer::run() {
LOG(INFO) << "Batched indexer threadpool shutdown...";
thread_pool->shutdown();
+ delete thread_pool;
}
std::string BatchedIndexer::get_req_prefix_key(uint64_t req_id) {
@@ -240,7 +250,6 @@ std::string BatchedIndexer::get_req_prefix_key(uint64_t req_id) {
BatchedIndexer::~BatchedIndexer() {
delete [] qmutuxes;
- delete thread_pool;
}
void BatchedIndexer::stop() {
@@ -250,3 +259,79 @@ void BatchedIndexer::stop() {
int64_t BatchedIndexer::get_queued_writes() {
return queued_writes;
}
+
+void BatchedIndexer::serialize_state(nlohmann::json& state) {
+ // requires external synchronization!
+ state["queued_writes"] = queued_writes.load();
+ state["req_res_map"] = nlohmann::json();
+
+ size_t num_reqs_stored = 0;
+ std::unique_lock lk(mutex);
+
+ for(auto& kv: req_res_map) {
+ std::string req_key = std::to_string(kv.first);
+ state["req_res_map"].emplace(req_key, nlohmann::json());
+ nlohmann::json& req_res = state["req_res_map"][req_key];
+ req_res["batch_begin_ts"] = kv.second.batch_begin_ts;
+ req_res["num_chunks"] = kv.second.num_chunks;
+ req_res["next_chunk_index"] = kv.second.next_chunk_index;
+ req_res["is_complete"] = kv.second.is_complete;
+ req_res["req"] = kv.second.req->to_json();
+ req_res["prev_req_body"] = kv.second.prev_req_body;
+ num_reqs_stored++;
+
+ //LOG(INFO) << "req_key: " << req_key << ", next_chunk_index: " << kv.second.next_chunk_index;
+ }
+
+ LOG(INFO) << "Serialized " << num_reqs_stored << " in-flight requests for snapshot.";
+}
+
+void BatchedIndexer::load_state(const nlohmann::json& state) {
+ queued_writes = state["queued_writes"].get<int64_t>();
+
+ size_t num_reqs_restored = 0;
+ std::vector<uint64_t> queue_ids;
+
+ for(auto& kv: state["req_res_map"].items()) {
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ req->load_from_json(kv.value()["req"].get<std::string>());
+
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+ req_res_t req_res(kv.value()["prev_req_body"].get<std::string>(), req, res,
+ kv.value()["batch_begin_ts"].get<uint64_t>(),
+ kv.value()["num_chunks"].get<uint32_t>(),
+ kv.value()["next_chunk_index"].get<uint32_t>(),
+ kv.value()["is_complete"].get<bool>());
+
+ {
+ std::unique_lock mlk(mutex);
+ req_res_map.emplace(std::stoull(kv.key()), req_res);
+ }
+
+ // add only completed requests to their respective collection-based queues
+ // the rest will be added by enqueue() when raft log is completely read
+
+ if(req_res.is_complete) {
+ LOG(INFO) << "req_res.next_chunk_index: " << req_res.next_chunk_index;
+ const std::string& coll_name = get_collection_name(req);
+ uint64_t queue_id = StringUtils::hash_wy(coll_name.c_str(), coll_name.size()) % num_threads;
+ queue_ids.push_back(queue_id);
+ std::unique_lock qlk(qmutuxes[queue_id]);
+ queues[queue_id].emplace_back(req->start_ts);
+ }
+
+ num_reqs_restored++;
+ }
+
+ // need to sort on `start_ts` to preserve original order
+ for(auto queue_id: queue_ids) {
+ std::unique_lock lk(qmutuxes[queue_id]);
+ std::sort(queues[queue_id].begin(), queues[queue_id].end());
+ }
+
+ LOG(INFO) << "Restored " << num_reqs_restored << " in-flight requests from snapshot.";
+}
+
+std::shared_mutex& BatchedIndexer::get_pause_mutex() {
+ return pause_mutex;
+}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 115b550d..eb70c2d4 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -3,6 +3,7 @@
#include <json.hpp>
#include <app_metrics.h>
#include "collection_manager.h"
+#include "batched_indexer.h"
#include "logger.h"
constexpr const size_t CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;
@@ -92,7 +93,8 @@ void CollectionManager::add_to_collections(Collection* collection) {
void CollectionManager::init(Store *store, ThreadPool* thread_pool,
const float max_memory_ratio,
const std::string & auth_key,
- std::atomic<bool>& quit) {
+ std::atomic<bool>& quit,
+ BatchedIndexer* batch_indexer) {
std::unique_lock lock(mutex);
this->store = store;
@@ -100,13 +102,14 @@ void CollectionManager::init(Store *store, ThreadPool* thread_pool,
this->bootstrap_auth_key = auth_key;
this->max_memory_ratio = max_memory_ratio;
this->quit = &quit;
+ this->batch_indexer = batch_indexer;
}
// used only in tests!
void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key,
std::atomic<bool>& quit) {
ThreadPool* thread_pool = new ThreadPool(8);
- init(store, thread_pool, max_memory_ratio, auth_key, quit);
+ init(store, thread_pool, max_memory_ratio, auth_key, quit, nullptr);
}
Option<bool> CollectionManager::load(const size_t collection_batch_size, const size_t document_batch_size) {
@@ -198,6 +201,16 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const s
loading_pool.shutdown();
+ LOG(INFO) << "Initializing batched indexer from snapshot state...";
+ if(batch_indexer != nullptr) {
+ std::string batched_indexer_state_str;
+ StoreStatus s = store->get(BATCHED_INDEXER_STATE_KEY, batched_indexer_state_str);
+ if(s == FOUND) {
+ nlohmann::json batch_indexer_state = nlohmann::json::parse(batched_indexer_state_str);
+ batch_indexer->load_state(batch_indexer_state);
+ }
+ }
+
return Option<bool>(true);
}
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 05b27cc9..467ac85a 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -447,13 +447,24 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Cl
LOG(INFO) << "on_snapshot_save";
std::string db_snapshot_path = writer->get_path() + "/" + db_snapshot_name;
- rocksdb::Checkpoint* checkpoint = nullptr;
- rocksdb::Status status = store->create_check_point(&checkpoint, db_snapshot_path);
- std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);
- if(!status.ok()) {
- LOG(ERROR) << "Failure during checkpoint creation, msg:" << status.ToString();
- done->status().set_error(EIO, "Checkpoint creation failure.");
+ {
+ // grab batch indexer lock so that we can take a clean snapshot
+ std::shared_mutex& pause_mutex = batched_indexer->get_pause_mutex();
+ std::unique_lock lk(pause_mutex);
+
+ nlohmann::json batch_index_state;
+ batched_indexer->serialize_state(batch_index_state);
+ store->insert(CollectionManager::BATCHED_INDEXER_STATE_KEY, batch_index_state.dump());
+
+ rocksdb::Checkpoint* checkpoint = nullptr;
+ rocksdb::Status status = store->create_check_point(&checkpoint, db_snapshot_path);
+ std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);
+
+ if(!status.ok()) {
+ LOG(ERROR) << "Failure during checkpoint creation, msg:" << status.ToString();
+ done->status().set_error(EIO, "Checkpoint creation failure.");
+ }
}
SnapshotArg* arg = new SnapshotArg;
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index dd5dbe51..9c6ce5e0 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -365,10 +365,6 @@ int run_server(const Config & config, const std::string & version, void (*master
// meta DB for storing house keeping things
Store meta_store(meta_dir, 24*60*60, 1024, false);
- CollectionManager & collectionManager = CollectionManager::get_instance();
- collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),
- config.get_api_key(), quit_raft_service);
-
curl_global_init(CURL_GLOBAL_SSL);
HttpClient & httpClient = HttpClient::get_instance();
httpClient.init(config.get_api_key());
@@ -394,6 +390,10 @@ int run_server(const Config & config, const std::string & version, void (*master
BatchedIndexer* batch_indexer = new BatchedIndexer(server, &store, num_threads);
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),
+ config.get_api_key(), quit_raft_service, batch_indexer);
+
// first we start the peering service
ReplicationState replication_state(server, batch_indexer, &store, &meta_store,