Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Restore store persisted raft log from snapshot.
Kishore Nallanbaaf7ea65db8
diff --git a/include/batched_indexer.h b/include/batched_indexer.hindex f1d41da8..ba6a9eac 100644--- a/include/batched_indexer.h+++ b/include/batched_indexer.h@@ -9,50 +9,60 @@class BatchedIndexer {private:- static const constexpr char* RAFT_REQ_LOG_PREFIX = "$RL_";-struct req_res_t {+ std::string prev_req_body; // used to handle partial JSON documents caused by chunkingstd::shared_ptr<http_req> req;std::shared_ptr<http_res> res;uint64_t batch_begin_ts;- req_res_t(const std::shared_ptr<http_req>& req,- const std::shared_ptr<http_res>& res, uint64_t batch_begin_ts):- req(req), res(res), batch_begin_ts(batch_begin_ts) {+ uint32_t num_chunks;+ uint32_t next_chunk_index; // index where next read must begin+ bool is_complete; // whether the req has been written to store fully- }-- req_res_t() {+ req_res_t(const std::string& prev_req_body,+ const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res,+ uint64_t batch_begin_ts, uint32_t num_chunks, uint32_t next_chunk_index, bool is_complete):+ prev_req_body(prev_req_body), req(req), res(res), batch_begin_ts(batch_begin_ts),+ num_chunks(num_chunks), next_chunk_index(next_chunk_index), is_complete(is_complete) {}++ req_res_t(): req(nullptr), res(nullptr), batch_begin_ts(0), num_chunks(0),+ next_chunk_index(0), is_complete(false) {};};HttpServer* server;Store* store;- ThreadPool* thread_pool;const size_t num_threads;- std::vector<std::deque<uint64_t>> queues;std::mutex* qmutuxes;+ std::vector<std::deque<uint64_t>> queues;++ /* Variables to be serialized on snapshot /+ --------------------------------------------------------- */std::mutex mutex;- std::unordered_map<uint64_t, uint32_t> request_to_chunk;std::unordered_map<uint64_t, req_res_t> req_res_map;+ std::atomic<int64_t> queued_writes = 0;++ /* ------------------------------------------------------- */+std::chrono::high_resolution_clock::time_point last_gc_run;std::atomic<bool> quit;+ std::shared_mutex pause_mutex;static const size_t GC_INTERVAL_SECONDS = 60;static const size_t GC_PRUNE_MAX_SECONDS = 3600;static std::string get_req_prefix_key(uint64_t req_id);- std::atomic<int64_t> queued_writes = 0;-public:+ static const constexpr char* RAFT_REQ_LOG_PREFIX = "$RL_";+BatchedIndexer(HttpServer* server, Store* store, size_t num_threads);~BatchedIndexer();@@ -64,4 +74,13 @@ public:void run();void stop();-};\ No newline at end of file++ // requires external synchronization!+ void serialize_state(nlohmann::json& state);++ void load_state(const nlohmann::json& state);++ std::string get_collection_name(const std::shared_ptr<http_req>& req);++ std::shared_mutex& get_pause_mutex();+};diff --git a/include/collection_manager.h b/include/collection_manager.hindex 0d5b31a8..9de2ecf7 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -8,6 +8,7 @@#include "collection.h"#include "auth_manager.h"#include "threadpool.h"+#include "batched_indexer.h"template<typename ResourceType>struct locked_resource_view_t {@@ -73,6 +74,8 @@ private:std::atomic<bool>* quit;+ BatchedIndexer* batch_indexer;+CollectionManager();~CollectionManager() = default;@@ -92,6 +95,7 @@ public:static constexpr const char* NEXT_COLLECTION_ID_KEY = "$CI";static constexpr const char* SYMLINK_PREFIX = "$SL";+ static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";static CollectionManager & get_instance() {static CollectionManager instance;@@ -120,7 +124,7 @@ public:// PUBLICLY EXPOSED APIvoid init(Store *store, ThreadPool* thread_pool, const float max_memory_ratio,- const std::string & auth_key, std::atomic<bool>& quit);+ const std::string & auth_key, std::atomic<bool>& quit, BatchedIndexer* batch_indexer);// only for tests!void init(Store *store, const float max_memory_ratio, const std::string & auth_key, std::atomic<bool>& exit);diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cppindex 17c30014..a652650b 100644--- a/src/batched_indexer.cpp+++ b/src/batched_indexer.cpp@@ -5,7 +5,6 @@BatchedIndexer::BatchedIndexer(HttpServer* server, Store* store, const size_t num_threads):server(server), store(store), num_threads(num_threads),last_gc_run(std::chrono::high_resolution_clock::now()), quit(false) {- thread_pool = new ThreadPool(num_threads);queues.resize(num_threads);qmutuxes = new std::mutex[num_threads];}@@ -15,68 +14,49 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh// NOTE: it's ok to access `req` and `res` in this function without synchronization// because the read thread for *this* request is paused now and resumes only messaged at the end- std::string& coll_name = req->params["collection"];-- if(coll_name.empty()) {- route_path* rpath;- server->get_route(req->route_hash, &rpath);-- // ensure that collection creation is sent to the same queue as writes to that collection- if(rpath->handler == post_create_collection) {- nlohmann::json obj = nlohmann::json::parse(req->body, nullptr, false);-- if(obj != nlohmann::json::value_t::discarded && obj.is_object() &&- obj.count("name") != 0 && obj["name"].is_string()) {- coll_name = obj["name"];- }- }- }-- uint64_t queue_id = StringUtils::hash_wy(coll_name.c_str(), coll_name.size()) % num_threads;-+ //LOG(INFO) << "BatchedIndexer::enqueue";uint32_t chunk_sequence = 0;{std::unique_lock lk(mutex);- chunk_sequence = request_to_chunk[req->start_ts];- request_to_chunk[req->start_ts] += 1;+ auto req_res_map_it = req_res_map.find(req->start_ts);++ if(req_res_map_it == req_res_map.end()) {+ // first chunk+ uint64_t batch_begin_ts = std::chrono::duration_cast<std::chrono::seconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ req_res_t req_res("", req, res, batch_begin_ts, 1, 0, false);+ req_res_map.emplace(req->start_ts, req_res);+ } else {+ chunk_sequence = req_res_map_it->second.num_chunks;+ req_res_map_it->second.num_chunks += 1;+ }}const std::string& req_key_prefix = get_req_prefix_key(req->start_ts);const std::string& request_chunk_key = req_key_prefix + StringUtils::serialize_uint32_t(chunk_sequence);- //LOG(INFO) << "req_id: " << req->start_ts << ", chunk_sequence: " << chunk_sequence;+ //LOG(INFO) << "insert request_chunk_key: " << request_chunk_key;store->insert(request_chunk_key, req->to_json());req->body = "";- {- std::unique_lock lk(mutex);- auto req_res_map_it = req_res_map.find(req->start_ts);- if(req_res_map_it == req_res_map.end()) {- uint64_t batch_begin_ts = std::chrono::duration_cast<std::chrono::seconds>(- std::chrono::system_clock::now().time_since_epoch()).count();-- req_res_t req_res{req, res, batch_begin_ts};- req_res_map[req->start_ts] = req_res;- }- }-bool is_old_serialized_request = (req->start_ts == 0);bool read_more_input = (req->_req != nullptr && req->_req->proceed_req);if(req->last_chunk_aggregate) {- //LOG(INFO) << "Last chunk for req_id: " << req->start_ts << ", queue_id: " << queue_id;-+ //LOG(INFO) << "Last chunk for req_id: " << req->start_ts;queued_writes += (chunk_sequence + 1);- std::unique_lock lk(mutex);- request_to_chunk.erase(req->start_ts);- lk.unlock();-{- std::unique_lock lk(qmutuxes[queue_id]);+ const std::string& coll_name = get_collection_name(req);+ uint64_t queue_id = StringUtils::hash_wy(coll_name.c_str(), coll_name.size()) % num_threads;+ std::unique_lock lk1(qmutuxes[queue_id]);queues[queue_id].emplace_back(req->start_ts);++ std::unique_lock lk2(mutex);+ req_res_map[req->start_ts].is_complete = true;}// IMPORTANT: must not read `req` variables (except _req) henceforth to prevent data races with indexing thread@@ -86,11 +66,12 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh// We can only do serial writes as we cannot reliably distinguish one streaming request from another.// So, wait for `req_res_map` to be empty before proceedingwhile(true) {- lk.lock();- if(req_res_map.empty()) {- break;+ {+ std::unique_lock lk(mutex);+ if(req_res_map.empty()) {+ break;+ }}- lk.unlock();std::this_thread::sleep_for(std::chrono::milliseconds (10));}@@ -104,8 +85,30 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh}}+std::string BatchedIndexer::get_collection_name(const std::shared_ptr<http_req>& req) {+ std::string& coll_name = req->params["collection"];++ if(coll_name.empty()) {+ route_path* rpath;+ server->get_route(req->route_hash, &rpath);++ // ensure that collection creation is sent to the same queue as writes to that collection+ if(rpath->handler == post_create_collection) {+ nlohmann::json obj = nlohmann::json::parse(req->body, nullptr, false);++ if(obj != nlohmann::json::value_t::discarded && obj.is_object() &&+ obj.count("name") != 0 && obj["name"].is_string()) {+ coll_name = obj["name"];+ }+ }+ }++ return coll_name;+}+void BatchedIndexer::run() {LOG(INFO) << "Starting batch indexer with " << num_threads << " threads.";+ ThreadPool* thread_pool = new ThreadPool(num_threads);for(size_t i = 0; i < num_threads; i++) {std::deque<uint64_t>& queue = queues[i];@@ -129,14 +132,24 @@ void BatchedIndexer::run() {// scan db for all logs associated with requestconst std::string& req_key_prefix = get_req_prefix_key(req_id);- rocksdb::Iterator* iter = store->scan(req_key_prefix);- std::string prev_body; // used to handle partial JSON documents caused by chunking+ const std::string& req_key_start_prefix = get_req_prefix_key(req_id) ++ StringUtils::serialize_uint32_t(orig_req_res.next_chunk_index);++ rocksdb::Iterator* iter = store->scan(req_key_start_prefix);++ // used to handle partial JSON documents caused by chunking+ std::string& prev_body = orig_req_res.prev_req_body;- const std::shared_ptr<http_res>& orig_res = orig_req_res.res;const std::shared_ptr<http_req>& orig_req = orig_req_res.req;+ const std::shared_ptr<http_res>& orig_res = orig_req_res.res;bool is_live_req = orig_res->is_alive;+ route_path* found_rpath = nullptr;+ bool route_found = server->get_route(orig_req->route_hash, &found_rpath);+ bool async_res = false;+while(iter->Valid() && iter->key().starts_with(req_key_prefix)) {+ std::shared_lock slk(pause_mutex); // used for snapshotorig_req->body = prev_body;orig_req->load_from_json(iter->value().ToString());@@ -145,10 +158,6 @@ void BatchedIndexer::run() {//LOG(INFO) << "original request: " << orig_req_res.req << ", req: " << orig_req_res.req->req;- route_path* found_rpath = nullptr;- bool route_found = server->get_route(orig_req->route_hash, &found_rpath);- bool async_res = false;-if(route_found) {async_res = found_rpath->async_res;found_rpath->handler(orig_req, orig_res);@@ -168,6 +177,7 @@ void BatchedIndexer::run() {}queued_writes--;+ orig_req_res.next_chunk_index++;iter->Next();if(quit) {@@ -177,7 +187,7 @@ void BatchedIndexer::run() {delete iter;- //LOG(INFO) << "Erasing request data from disk and memory for request " << req_res.req->start_ts;+ //LOG(INFO) << "Erasing request data from disk and memory for request " << orig_req_res.req->start_ts;// we can delete the buffered request contentstore->delete_range(req_key_prefix, req_key_prefix + StringUtils::serialize_uint32_t(UINT32_MAX));@@ -216,7 +226,6 @@ void BatchedIndexer::run() {LOG(INFO) << "Deleting partial upload for req id " << it->second.req->start_ts;const std::string& req_key_prefix = get_req_prefix_key(it->second.req->start_ts);store->delete_range(req_key_prefix, req_key_prefix + StringUtils::serialize_uint32_t(UINT32_MAX));- request_to_chunk.erase(it->second.req->start_ts);it = req_res_map.erase(it);} else {it++;@@ -229,6 +238,7 @@ void BatchedIndexer::run() {LOG(INFO) << "Batched indexer threadpool shutdown...";thread_pool->shutdown();+ delete thread_pool;}std::string BatchedIndexer::get_req_prefix_key(uint64_t req_id) {@@ -240,7 +250,6 @@ std::string BatchedIndexer::get_req_prefix_key(uint64_t req_id) {BatchedIndexer::~BatchedIndexer() {delete [] qmutuxes;- delete thread_pool;}void BatchedIndexer::stop() {@@ -250,3 +259,79 @@ void BatchedIndexer::stop() {int64_t BatchedIndexer::get_queued_writes() {return queued_writes;}++void BatchedIndexer::serialize_state(nlohmann::json& state) {+ // requires external synchronization!+ state["queued_writes"] = queued_writes.load();+ state["req_res_map"] = nlohmann::json();++ size_t num_reqs_stored = 0;+ std::unique_lock lk(mutex);++ for(auto& kv: req_res_map) {+ std::string req_key = std::to_string(kv.first);+ state["req_res_map"].emplace(req_key, nlohmann::json());+ nlohmann::json& req_res = state["req_res_map"][req_key];+ req_res["batch_begin_ts"] = kv.second.batch_begin_ts;+ req_res["num_chunks"] = kv.second.num_chunks;+ req_res["next_chunk_index"] = kv.second.next_chunk_index;+ req_res["is_complete"] = kv.second.is_complete;+ req_res["req"] = kv.second.req->to_json();+ req_res["prev_req_body"] = kv.second.prev_req_body;+ num_reqs_stored++;++ //LOG(INFO) << "req_key: " << req_key << ", next_chunk_index: " << kv.second.next_chunk_index;+ }++ LOG(INFO) << "Serialized " << num_reqs_stored << " in-flight requests for snapshot.";+}++void BatchedIndexer::load_state(const nlohmann::json& state) {+ queued_writes = state["queued_writes"].get<int64_t>();++ size_t num_reqs_restored = 0;+ std::vector<uint64_t> queue_ids;++ for(auto& kv: state["req_res_map"].items()) {+ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ req->load_from_json(kv.value()["req"].get<std::string>());++ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);+ req_res_t req_res(kv.value()["prev_req_body"].get<std::string>(), req, res,+ kv.value()["batch_begin_ts"].get<uint64_t>(),+ kv.value()["num_chunks"].get<uint32_t>(),+ kv.value()["next_chunk_index"].get<uint32_t>(),+ kv.value()["is_complete"].get<bool>());++ {+ std::unique_lock mlk(mutex);+ req_res_map.emplace(std::stoull(kv.key()), req_res);+ }++ // add only completed requests to their respective collection-based queues+ // the rest will be added by enqueue() when raft log is completely read++ if(req_res.is_complete) {+ LOG(INFO) << "req_res.next_chunk_index: " << req_res.next_chunk_index;+ const std::string& coll_name = get_collection_name(req);+ uint64_t queue_id = StringUtils::hash_wy(coll_name.c_str(), coll_name.size()) % num_threads;+ queue_ids.push_back(queue_id);+ std::unique_lock qlk(qmutuxes[queue_id]);+ queues[queue_id].emplace_back(req->start_ts);+ }++ num_reqs_restored++;+ }++ // need to sort on `start_ts` to preserve original order+ for(auto queue_id: queue_ids) {+ std::unique_lock lk(qmutuxes[queue_id]);+ std::sort(queues[queue_id].begin(), queues[queue_id].end());+ }++ LOG(INFO) << "Restored " << num_reqs_restored << " in-flight requests from snapshot.";+}++std::shared_mutex& BatchedIndexer::get_pause_mutex() {+ return pause_mutex;+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 115b550d..eb70c2d4 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -3,6 +3,7 @@#include <json.hpp>#include <app_metrics.h>#include "collection_manager.h"+#include "batched_indexer.h"#include "logger.h"constexpr const size_t CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;@@ -92,7 +93,8 @@ void CollectionManager::add_to_collections(Collection* collection) {void CollectionManager::init(Store *store, ThreadPool* thread_pool,const float max_memory_ratio,const std::string & auth_key,- std::atomic<bool>& quit) {+ std::atomic<bool>& quit,+ BatchedIndexer* batch_indexer) {std::unique_lock lock(mutex);this->store = store;@@ -100,13 +102,14 @@ void CollectionManager::init(Store *store, ThreadPool* thread_pool,this->bootstrap_auth_key = auth_key;this->max_memory_ratio = max_memory_ratio;this->quit = &quit;+ this->batch_indexer = batch_indexer;}// used only in tests!void CollectionManager::init(Store *store, const float max_memory_ratio, const std::string & auth_key,std::atomic<bool>& quit) {ThreadPool* thread_pool = new ThreadPool(8);- init(store, thread_pool, max_memory_ratio, auth_key, quit);+ init(store, thread_pool, max_memory_ratio, auth_key, quit, nullptr);}Option<bool> CollectionManager::load(const size_t collection_batch_size, const size_t document_batch_size) {@@ -198,6 +201,16 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const sloading_pool.shutdown();+ LOG(INFO) << "Initializing batched indexer from snapshot state...";+ if(batch_indexer != nullptr) {+ std::string batched_indexer_state_str;+ StoreStatus s = store->get(BATCHED_INDEXER_STATE_KEY, batched_indexer_state_str);+ if(s == FOUND) {+ nlohmann::json batch_indexer_state = nlohmann::json::parse(batched_indexer_state_str);+ batch_indexer->load_state(batch_indexer_state);+ }+ }+return Option<bool>(true);}diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 05b27cc9..467ac85a 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -447,13 +447,24 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::ClLOG(INFO) << "on_snapshot_save";std::string db_snapshot_path = writer->get_path() + "/" + db_snapshot_name;- rocksdb::Checkpoint* checkpoint = nullptr;- rocksdb::Status status = store->create_check_point(&checkpoint, db_snapshot_path);- std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);- if(!status.ok()) {- LOG(ERROR) << "Failure during checkpoint creation, msg:" << status.ToString();- done->status().set_error(EIO, "Checkpoint creation failure.");+ {+ // grab batch indexer lock so that we can take a clean snapshot+ std::shared_mutex& pause_mutex = batched_indexer->get_pause_mutex();+ std::unique_lock lk(pause_mutex);++ nlohmann::json batch_index_state;+ batched_indexer->serialize_state(batch_index_state);+ store->insert(CollectionManager::BATCHED_INDEXER_STATE_KEY, batch_index_state.dump());++ rocksdb::Checkpoint* checkpoint = nullptr;+ rocksdb::Status status = store->create_check_point(&checkpoint, db_snapshot_path);+ std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);++ if(!status.ok()) {+ LOG(ERROR) << "Failure during checkpoint creation, msg:" << status.ToString();+ done->status().set_error(EIO, "Checkpoint creation failure.");+ }}SnapshotArg* arg = new SnapshotArg;diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex dd5dbe51..9c6ce5e0 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -365,10 +365,6 @@ int run_server(const Config & config, const std::string & version, void (*master// meta DB for storing house keeping thingsStore meta_store(meta_dir, 24*60*60, 1024, false);- CollectionManager & collectionManager = CollectionManager::get_instance();- collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),- config.get_api_key(), quit_raft_service);-curl_global_init(CURL_GLOBAL_SSL);HttpClient & httpClient = HttpClient::get_instance();httpClient.init(config.get_api_key());@@ -394,6 +390,10 @@ int run_server(const Config & config, const std::string & version, void (*masterBatchedIndexer* batch_indexer = new BatchedIndexer(server, &store, num_threads);+ CollectionManager & collectionManager = CollectionManager::get_instance();+ collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),+ config.get_api_key(), quit_raft_service, batch_indexer);+// first we start the peering serviceReplicationState replication_state(server, batch_indexer, &store, &meta_store,