Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

add analytics db to snapshot

krunala21a2dfdddf9
diff --git a/include/raft_server.h b/include/raft_server.h
index d5fcfec7..0c0035ec 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -99,6 +99,7 @@ public:
class ReplicationState : public braft::StateMachine {
private:
static constexpr const char* db_snapshot_name = "db_snapshot";
+ static constexpr const char* analytics_db_snapshot_name = "analytics_db_snapshot";
mutable std::shared_mutex node_mutex;
@@ -109,6 +110,7 @@ private:
BatchedIndexer* batched_indexer;
Store* store;
+ Store* analytics_store;
ThreadPool* thread_pool;
http_message_dispatcher* message_dispatcher;
@@ -148,7 +150,7 @@ public:
static constexpr const char* meta_dir_name = "meta";
static constexpr const char* snapshot_dir_name = "snapshot";
- ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer, Store* store,
+ ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer, Store* store, Store* analytics_store,
ThreadPool* thread_pool, http_message_dispatcher* message_dispatcher,
bool api_uses_ssl, const Config* config,
size_t num_collections_parallel_load, size_t num_documents_parallel_load);
@@ -253,6 +255,7 @@ private:
braft::SnapshotWriter* writer;
std::string state_dir_path;
std::string db_snapshot_path;
+ std::string analytics_db_snapshot_path;
std::string ext_snapshot_path;
braft::Closure* done;
};
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 102df461..e5d65fd9 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -493,6 +493,16 @@ void* ReplicationState::save_snapshot(void* arg) {
}
}
+ //add analytics db snapshot files to writer state
+ butil::FileEnumerator analytics_dir_enum(butil::FilePath(sa->analytics_db_snapshot_path), false, butil::FileEnumerator::FILES);
+ for (butil::FilePath file = analytics_dir_enum.Next(); !file.empty(); file = analytics_dir_enum.Next()) {
+ auto file_name = std::string(analytics_db_snapshot_name) + "/" + file.BaseName().value();
+ if (sa->writer->add_file(file_name) != 0) {
+ sa->done->status().set_error(EIO, "Fail to add analytics file to writer.");
+ return nullptr;
+ }
+ }
+
const std::string& temp_snapshot_dir = sa->writer->get_path();
sa->done->Run();
@@ -538,6 +548,7 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Cl
LOG(INFO) << "on_snapshot_save";
std::string db_snapshot_path = writer->get_path() + "/" + db_snapshot_name;
+ std::string analytics_db_snapshot_path = writer->get_path() + "/" + analytics_db_snapshot_name;
{
// grab batch indexer lock so that we can take a clean snapshot
@@ -560,6 +571,18 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Cl
LOG(ERROR) << "Failure during checkpoint creation, msg:" << status.ToString();
done->status().set_error(EIO, "Checkpoint creation failure.");
}
+
+ if(analytics_store) {
+ analytics_store->insert(CollectionManager::BATCHED_INDEXER_STATE_KEY, batch_index_state.dump());
+ rocksdb::Checkpoint* checkpoint2 = nullptr;
+ status = analytics_store->create_check_point(&checkpoint2, analytics_db_snapshot_path);
+ std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint2);
+
+ if(!status.ok()) {
+ LOG(ERROR) << "AnalyticsStore : Failure during checkpoint creation, msg:" << status.ToString();
+ done->status().set_error(EIO, "AnalyticsStore : Checkpoint creation failure.");
+ }
+ }
}
SnapshotArg* arg = new SnapshotArg;
@@ -567,6 +590,7 @@ void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Cl
arg->writer = writer;
arg->state_dir_path = raft_dir_path;
arg->db_snapshot_path = db_snapshot_path;
+ arg->analytics_db_snapshot_path = analytics_db_snapshot_path;
arg->done = done;
if(!ext_snapshot_path.empty()) {
@@ -609,6 +633,17 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
// Load snapshot from leader, replacing the running StateMachine
std::string snapshot_path = reader->get_path();
+
+ if(analytics_store) {
+ snapshot_path.append(std::string("/") + analytics_db_snapshot_name);
+ int reload_store = analytics_store->reload(true, snapshot_path);
+ if (reload_store != 0) {
+ LOG(ERROR) << "Failed to reload analytics db snapshot.";
+ return reload_store;
+ }
+ }
+
+ snapshot_path = reader->get_path();
snapshot_path.append(std::string("/") + db_snapshot_name);
int reload_store = store->reload(true, snapshot_path);
@@ -776,12 +811,12 @@ void ReplicationState::refresh_catchup_status(bool log_msg) {
}
ReplicationState::ReplicationState(HttpServer* server, BatchedIndexer* batched_indexer,
- Store *store, ThreadPool* thread_pool,
+ Store *store, Store* analytics_store, ThreadPool* thread_pool,
http_message_dispatcher *message_dispatcher,
bool api_uses_ssl, const Config* config,
size_t num_collections_parallel_load, size_t num_documents_parallel_load):
node(nullptr), leader_term(-1), server(server), batched_indexer(batched_indexer),
- store(store),
+ store(store), analytics_store(analytics_store),
thread_pool(thread_pool), message_dispatcher(message_dispatcher), api_uses_ssl(api_uses_ssl),
config(config),
num_collections_parallel_load(num_collections_parallel_load),
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index b6f0afb9..24d20d42 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -467,7 +467,7 @@ int run_server(const Config & config, const std::string & version, void (*master
// first we start the peering service
- ReplicationState replication_state(server, batch_indexer, &store,
+ ReplicationState replication_state(server, batch_indexer, &store, analytics_store.get(),
&replication_thread_pool, server->get_message_dispatcher(),
ssl_enabled,
&config,