Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
add analytics db to snapshot
krunala21a2dfdddf9
diff --git a/include/raft_server.h b/include/raft_server.hindex d5fcfec7..0c0035ec 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -99,6 +99,7 @@ public:class ReplicationState : public braft::StateMachine {private:static constexpr const char* db_snapshot_name = "db_snapshot";+ static constexpr const char* analytics_db_snapshot_name = "analytics_db_snapshot";mutable std::shared_mutex node_mutex;@@ -109,6 +110,7 @@ private:BatchedIndexer* batched_indexer;Store* store;+ Store* analytics_store;ThreadPool* thread_pool;http_message_dispatcher* message_dispatcher;@@ -148,7 +150,7 @@ public:static constexpr const char* meta_dir_name = "meta";static constexpr const char* snapshot_dir_name = "snapshot";- ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer, Store* store,+ ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer, Store* store, Store* analytics_store,ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,bool api_uses_ssl, const Config* config,size_t num_collections_parallel_load, size_t num_documents_parallel_load);@@ -253,6 +255,7 @@ private:braft::SnapshotWriter* writer;std::string state_dir_path;std::string db_snapshot_path;+ std::string analytics_db_snapshot_path;std::string ext_snapshot_path;braft::Closure* done;};diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 102df461..e5d65fd9 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -493,6 +493,16 @@ void* ReplicationState::save_snapshot(void* arg) {}}+ //add analytics db snapshot files to writer state+ butil::FileEnumerator analytics_dir_enum(butil::FilePath(sa->analytics_db_snapshot_path), false, butil::FileEnumerator::FILES);+ for (butil::FilePath file = analytics_dir_enum.Next(); !file.empty(); file = analytics_dir_enum.Next()) {+ auto file_name = std::string(analytics_db_snapshot_name) + "/" + file.BaseName().value();+ if (sa->writer->add_file(file_name) != 0) {+ sa->done->status().set_error(EIO, "Fail to add analytics file to writer.");+ return nullptr;+ }+ }+const std::string& temp_snapshot_dir = sa->writer->get_path();sa->done->Run();@@ -538,6 +548,7 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::ClLOG(INFO) << "on_snapshot_save";std::string db_snapshot_path = writer->get_path() + "/" + db_snapshot_name;+ std::string analytics_db_snapshot_path = writer->get_path() + "/" + analytics_db_snapshot_name;{// grab batch indexer lock so that we can take a clean snapshot@@ -560,6 +571,18 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::ClLOG(ERROR) << "Failure during checkpoint creation, msg:" << status.ToString();done->status().set_error(EIO, "Checkpoint creation failure.");}++ if(analytics_store) {+ analytics_store->insert(CollectionManager::BATCHED_INDEXER_STATE_KEY, batch_index_state.dump());+ rocksdb::Checkpoint* checkpoint2 = nullptr;+ status = analytics_store->create_check_point(&checkpoint2, analytics_db_snapshot_path);+ std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint2);++ if(!status.ok()) {+ LOG(ERROR) << "AnalyticsStore : Failure during checkpoint creation, msg:" << status.ToString();+ done->status().set_error(EIO, "AnalyticsStore : Checkpoint creation failure.");+ }+ }}SnapshotArg* arg = new SnapshotArg;@@ -567,6 +590,7 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Clarg->writer = writer;arg->state_dir_path = raft_dir_path;arg->db_snapshot_path = db_snapshot_path;+ arg->analytics_db_snapshot_path = analytics_db_snapshot_path;arg->done = done;if(!ext_snapshot_path.empty()) {@@ -609,6 +633,17 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {// Load snapshot from leader, replacing the running StateMachinestd::string snapshot_path = reader->get_path();++ if(analytics_store) {+ snapshot_path.append(std::string("/") + analytics_db_snapshot_name);+ int reload_store = analytics_store->reload(true, snapshot_path);+ if (reload_store != 0) {+ LOG(ERROR) << "Failed to reload analytics db snapshot.";+ return reload_store;+ }+ }++ snapshot_path = reader->get_path();snapshot_path.append(std::string("/") + db_snapshot_name);int reload_store = store->reload(true, snapshot_path);@@ -776,12 +811,12 @@ void ReplicationState::refresh_catchup_status(bool log_msg) {}ReplicationState::ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer,- Store *store, ThreadPool* thread_pool,+ Store *store, Store* analytics_store, ThreadPool* thread_pool,http_message_dispatcher *message_dispatcher,bool api_uses_ssl, const Config* config,size_t num_collections_parallel_load, size_t num_documents_parallel_load):node(nullptr), leader_term(-1), server(server), batched_indexer(batched_indexer),- store(store),+ store(store), analytics_store(analytics_store),thread_pool(thread_pool), message_dispatcher(message_dispatcher), api_uses_ssl(api_uses_ssl),config(config),num_collections_parallel_load(num_collections_parallel_load),diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex b6f0afb9..24d20d42 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -467,7 +467,7 @@ int run_server(const Config & config, const std::string & version, void (*master// first we start the peering service- ReplicationState replication_state(server, batch_indexer, &store,+ ReplicationState replication_state(server, batch_indexer, &store, analytics_store.get(),&replication_thread_pool, server->get_message_dispatcher(),ssl_enabled,&config,