Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

On demand snapshot API.

kishorenc8a310bbb8267
diff --git a/include/core_api.h b/include/core_api.h
index 1851385c..ee0bbb25 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -79,10 +79,14 @@ bool get_debug(http_req& req, http_res& res);
bool get_health(http_req& req, http_res& res);
+bool post_health(http_req& req, http_res& res);
+
bool get_metrics_json(http_req& req, http_res& res);
bool get_log_sequence(http_req& req, http_res& res);
+bool post_snapshot(http_req& req, http_res& res);
+
// Misc helpers
bool raft_write_send_response(void *data);
diff --git a/include/http_server.h b/include/http_server.h
index 847cecef..51f8bade 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -58,8 +58,6 @@ private:
bool exit_loop = false;
-private:
-
std::string version;
// must be a vector since order of routes matter
@@ -173,4 +171,6 @@ public:
static void on_deferred_process_request(h2o_timer_t *entry);
void defer_processing(http_req& req, http_res& res, size_t timeout_ms);
+
+ void do_snapshot(const std::string& snapshot_path, http_req& req, http_res& res);
};
diff --git a/include/raft_server.h b/include/raft_server.h
index 4bfceef9..b5e11693 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -72,6 +72,23 @@ public:
void Run();
};
+// Closure that fires when requested
+class OnDemandSnapshotClosure : public braft::Closure {
+private:
+ ReplicationState* replication_state;
+ http_req& req;
+ http_res& res;
+
+public:
+
+ OnDemandSnapshotClosure(ReplicationState* replication_state, http_req& req, http_res& res):
+ replication_state(replication_state), req(req), res(res) {}
+
+ ~OnDemandSnapshotClosure() {}
+
+ void Run();
+};
+
// Implements braft::StateMachine.
class ReplicationState : public braft::StateMachine {
@@ -96,6 +113,10 @@ private:
std::atomic<bool>& shut_down;
+ std::string raft_dir_path;
+
+ std::string ext_snapshot_path;
+
public:
static constexpr const char* log_dir_name = "log";
@@ -154,9 +175,15 @@ public:
void reset_db();
+ void do_snapshot(const std::string& snapshot_path, http_req& req, http_res& res);
+
static std::string to_nodes_config(const butil::EndPoint &peering_endpoint, const int api_port,
const std::string &nodes_config);
+ void set_ext_snapshot_path(const std::string &snapshot_path);
+
+ const std::string& get_ext_snapshot_path() const;
+
static constexpr const char* REPLICATION_MSG = "raft_replication";
private:
@@ -167,8 +194,12 @@ private:
void on_apply(braft::Iterator& iter);
struct SnapshotArg {
- rocksdb::DB* db;
+ ReplicationState* replication_state;
braft::SnapshotWriter* writer;
+ std::string state_dir_path;
+ std::string db_dir_path;
+ std::string db_snapshot_path;
+ std::string ext_snapshot_path;
braft::Closure* done;
};
@@ -217,5 +248,10 @@ private:
LOG(INFO) << "Node stops following " << ctx;
}
- void follower_write(http_req *request, http_res *response) const;
+ void write_to_leader(http_req *request, http_res *response) const;
+
+ void do_dummy_write();
+
+ std::string get_leader_url_path(const std::string& leader_addr, const std::string& path,
+ const std::string& protocol) const;
};
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 7270671d..97e647d6 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -203,6 +203,7 @@ bool get_health(http_req & req, http_res & res) {
nlohmann::json result;
bool alive = server->is_alive();
result["ok"] = alive;
+
if(alive) {
res.set_body(200, result.dump());
} else {
@@ -212,6 +213,19 @@ bool get_health(http_req & req, http_res & res) {
return alive;
}
+bool post_health(http_req & req, http_res & res) {
+ nlohmann::json result;
+ bool alive = server->is_alive();
+ result["ok"] = alive;
+
+ if(alive) {
+ res.set_body(200, result.dump());
+ } else {
+ res.set_body(503, result.dump());
+ }
+
+ return alive;
+}
bool get_metrics_json(http_req &req, http_res &res) {
nlohmann::json result;
@@ -1289,6 +1303,25 @@ bool raft_write_send_response(void *data) {
return true;
}
+bool post_snapshot(http_req& req, http_res& res) {
+ const std::string SNAPSHOT_PATH = "snapshot_path";
+
+ res.status_code = 201;
+ res.content_type_header = "application/json";
+
+ if(req.params.count(SNAPSHOT_PATH) == 0) {
+ req.last_chunk_aggregate = true;
+ res.final = true;
+ res.set_400(std::string("Parameter `") + SNAPSHOT_PATH + "` is required.");
+ HttpServer::stream_response(req, res);
+ return false;
+ }
+
+ server->do_snapshot(req.params[SNAPSHOT_PATH], req, res);
+
+ return true;
+}
+
bool get_synonyms(http_req &req, http_res &res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
Collection *collection = collectionManager.get_collection(req.params["collection"]);
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 4932fadf..25741a39 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -245,7 +245,8 @@ std::map<std::string, std::string> HttpServer::parse_query(const std::string& qu
return query_map;
}
-uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, const std::string & http_method, route_path** found_rpath) {
+uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, const std::string & http_method,
+ route_path** found_rpath) {
for (const auto& index_route : routes) {
const route_path & rpath = index_route.second;
@@ -492,10 +493,14 @@ int HttpServer::process_request(http_req* request, http_res* response, route_pat
const h2o_custom_req_handler_t *handler) {
//LOG(INFO) << "process_request called";
+ bool operations_path = (rpath->path_parts.size() != 0 && rpath->path_parts[0] == "operations");
+
+ //LOG(INFO) << "operations_path: " << operations_path;
// for writes, we delegate to replication_state to handle response
- if(rpath->http_method == "POST" || rpath->http_method == "PUT" || rpath->http_method == "DELETE" ||
- rpath->http_method == "PATCH") {
+ if(!operations_path &&
+ (rpath->http_method == "POST" || rpath->http_method == "PUT" ||
+ rpath->http_method == "DELETE" || rpath->http_method == "PATCH")) {
handler->http_server->get_replication_state()->write(request, response);
return 0;
}
@@ -800,3 +805,7 @@ bool HttpServer::on_request_proceed_message(void *data) {
bool HttpServer::has_exited() const {
return exit_loop;
}
+
+void HttpServer::do_snapshot(const std::string& snapshot_path, http_req& req, http_res& res) {
+ return replication_state->do_snapshot(snapshot_path, req, res);
+}
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index b4d52def..38b396c1 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -13,23 +13,21 @@ extern "C" {
#endif
void master_server_routes() {
- // collection management
- server->post("/collections", post_create_collection);
- server->get("/collections", get_collections);
- server->del("/collections/:collection", del_drop_collection);
- server->get("/collections/:collection", get_collection_summary);
+ // collection operations
+ // NOTE: placing this first to score an immediate hit on O(N) route search
+ server->get("/collections/:collection/documents/search", get_search);
- // document management - `/documents/:id` end-points must be placed last in the list
+ // document management
+ // NOTE:`/documents/:id` end-points must be placed last in the list
server->post("/collections/:collection/documents", post_add_document);
- server->patch("/collections/:collection/documents/:id", patch_update_document);
- server->get("/collections/:collection/documents/search", get_search);
+ server->del("/collections/:collection/documents", del_remove_documents, false, true);
server->post("/collections/:collection/documents/import", post_import_documents, true, true);
server->get("/collections/:collection/documents/export", get_export_documents, false, true);
server->get("/collections/:collection/documents/:id", get_fetch_document);
+ server->patch("/collections/:collection/documents/:id", patch_update_document);
server->del("/collections/:collection/documents/:id", del_remove_document);
- server->del("/collections/:collection/documents", del_remove_documents, false, true);
server->get("/collections/:collection/overrides", get_overrides);
server->get("/collections/:collection/overrides/:id", get_override);
@@ -41,6 +39,12 @@ void master_server_routes() {
server->put("/collections/:collection/synonyms/:id", put_synonym);
server->del("/collections/:collection/synonyms/:id", del_synonym);
+ // collection management
+ server->post("/collections", post_create_collection);
+ server->get("/collections", get_collections);
+ server->del("/collections/:collection", del_drop_collection);
+ server->get("/collections/:collection", get_collection_summary);
+
server->get("/aliases", get_aliases);
server->get("/aliases/:alias", get_alias);
server->put("/aliases/:alias", put_upsert_alias);
@@ -55,7 +59,9 @@ void master_server_routes() {
server->get("/metrics.json", get_metrics_json);
server->get("/debug", get_debug);
server->get("/health", get_health);
+ server->post("/health", post_health);
server->get("/sequence", get_log_sequence);
+ server->post("/operations/snapshot", post_snapshot, false, true);
}
int main(int argc, char **argv) {
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index fd618050..25fad5e0 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -25,6 +25,8 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const int
int election_timeout_ms, int snapshot_interval_s,
const std::string & raft_dir, const std::string & nodes) {
+ this->raft_dir_path = raft_dir;
+
braft::NodeOptions node_options;
std::string actual_nodes_config = to_nodes_config(peering_endpoint, api_port, nodes);
@@ -105,7 +107,7 @@ void ReplicationState::write(http_req* request, http_res* response) {
}
if (!node->is_leader()) {
- return follower_write(request, response);
+ return write_to_leader(request, response);
}
// Serialize request to replicated WAL so that all the nodes in the group receive it as well.
@@ -130,7 +132,7 @@ void ReplicationState::write(http_req* request, http_res* response) {
return node->apply(task);
}
-void ReplicationState::follower_write(http_req *request, http_res *response) const {
+void ReplicationState::write_to_leader(http_req *request, http_res *response) const {
if(node->leader_id().is_empty()) {
// Handle no leader scenario
LOG(ERROR) << "Rejecting write: could not find a leader.";
@@ -163,12 +165,9 @@ void ReplicationState::follower_write(http_req *request, http_res *response) con
thread_pool->enqueue([leader_addr, request, response, server, this]() {
auto raw_req = request->_req;
- std::string scheme = std::string(raw_req->scheme->name.base, raw_req->scheme->name.len);
- std::vector<std::string> addr_parts;
- StringUtils::split(leader_addr, addr_parts, ":");
- std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];
- const std::string & path = std::string(raw_req->path.base, raw_req->path.len);
- std::string url = scheme + "://" + leader_host_port + path;
+ const std::string& path = std::string(raw_req->path.base, raw_req->path.len);
+ const std::string& scheme = std::string(raw_req->scheme->name.base, raw_req->scheme->name.len);
+ const std::string url = get_leader_url_path(leader_addr, path, scheme);
std::map<std::string, std::string> res_headers;
@@ -222,6 +221,15 @@ void ReplicationState::follower_write(http_req *request, http_res *response) con
});
}
+std::string ReplicationState::get_leader_url_path(const std::string& leader_addr, const std::string& path,
+ const std::string& protocol) const {
+ std::vector<std::string> addr_parts;
+ StringUtils::split(leader_addr, addr_parts, ":");
+ std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];
+ std::string url = protocol + "://" + leader_host_port + path;
+ return url;
+}
+
void ReplicationState::on_apply(braft::Iterator& iter) {
//LOG(INFO) << "ReplicationState::on_apply";
@@ -297,44 +305,73 @@ void* ReplicationState::save_snapshot(void* arg) {
std::unique_ptr<SnapshotArg> arg_guard(sa);
brpc::ClosureGuard done_guard(sa->done);
- std::string snapshot_path = sa->writer->get_path() + "/" + db_snapshot_name;
-
- rocksdb::Checkpoint* checkpoint = nullptr;
- rocksdb::Status status = rocksdb::Checkpoint::Create(sa->db, &checkpoint);
-
- if(!status.ok()) {
- LOG(WARNING) << "Checkpoint Create failed, msg:" << status.ToString();
- return nullptr;
- }
-
- std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);
- status = checkpoint->CreateCheckpoint(snapshot_path);
- if(!status.ok()) {
- LOG(WARNING) << "Checkpoint CreateCheckpoint failed, msg:" << status.ToString();
- return nullptr;
- }
+ // add the db snapshot files to writer state
+ butil::FileEnumerator dir_enum(butil::FilePath(sa->db_snapshot_path), false, butil::FileEnumerator::FILES);
- butil::FileEnumerator dir_enum(butil::FilePath(snapshot_path),false, butil::FileEnumerator::FILES);
- for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {
- std::string file_name = std::string(db_snapshot_name) + "/" + name.BaseName().value();
+ for (butil::FilePath file = dir_enum.Next(); !file.empty(); file = dir_enum.Next()) {
+ std::string file_name = std::string(db_snapshot_name) + "/" + file.BaseName().value();
if (sa->writer->add_file(file_name) != 0) {
sa->done->status().set_error(EIO, "Fail to add file to writer.");
return nullptr;
}
}
+ // if an external snapshot is requested, copy both state and data directory into that
+ if(!sa->ext_snapshot_path.empty()) {
+ LOG(INFO) << "Copying system snapshot to external snapshot directory at " << sa->ext_snapshot_path;
+ if(!butil::DirectoryExists(butil::FilePath(sa->ext_snapshot_path))) {
+ butil::CreateDirectory(butil::FilePath(sa->ext_snapshot_path), true);
+ }
+ butil::CopyDirectory(butil::FilePath(sa->state_dir_path), butil::FilePath(sa->ext_snapshot_path), true);
+ butil::CopyDirectory(butil::FilePath(sa->db_dir_path), butil::FilePath(sa->ext_snapshot_path), true);
+ }
+
+ // NOTE: *must* do a dummy write here since snapshots cannot be triggered if no write has happened since the
+ // last snapshot. By doing a dummy write right after a snapshot, we ensure that this can never be the case.
+ sa->replication_state->do_dummy_write();
+
LOG(INFO) << "save_snapshot done";
return nullptr;
}
+// this method is serial to on_apply so guarantees a snapshot view of the state machine
void ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done) {
LOG(INFO) << "on_snapshot_save";
- // Start a new bthread to avoid blocking StateMachine since it could be slow to write data to disk
+
+ rocksdb::Checkpoint* checkpoint = nullptr;
+ rocksdb::Status status = rocksdb::Checkpoint::Create(store->_get_db_unsafe(), &checkpoint);
+
+ if(!status.ok()) {
+ LOG(WARNING) << "Checkpoint Create failed, msg:" << status.ToString();
+ done->status().set_error(EIO, "Checkpoint Create failed.");
+ }
+
+ std::string db_snapshot_path = writer->get_path() + "/" + db_snapshot_name;
+
+ std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);
+ status = checkpoint->CreateCheckpoint(db_snapshot_path);
+
+ if(!status.ok()) {
+ LOG(WARNING) << "Checkpoint CreateCheckpoint failed at snapshot path: "
+ << db_snapshot_path << ", msg:" << status.ToString();
+ done->status().set_error(EIO, "CreateCheckpoint failed.");
+ }
+
SnapshotArg* arg = new SnapshotArg;
- arg->db = store->_get_db_unsafe();
+ arg->replication_state = this;
arg->writer = writer;
+ arg->state_dir_path = raft_dir_path;
+ arg->db_dir_path = store->get_state_dir_path();
+ arg->db_snapshot_path = db_snapshot_path;
arg->done = done;
+
+ if(!ext_snapshot_path.empty()) {
+ arg->ext_snapshot_path = ext_snapshot_path;
+ ext_snapshot_path = "";
+ }
+
+ // Start a new bthread to avoid blocking StateMachine since it could be slow to write data to disk
bthread_t tid;
bthread_start_urgent(&tid, NULL, save_snapshot, arg);
}
@@ -507,6 +544,38 @@ uint64_t ReplicationState::node_state() const {
return node_status.state;
}
+void ReplicationState::do_snapshot(const std::string& snapshot_path, http_req& req, http_res& res) {
+ LOG(INFO) << "Triggerring an on demand snapshot...";
+ OnDemandSnapshotClosure* snapshot_closure = new OnDemandSnapshotClosure(this, req, res);
+ ext_snapshot_path = snapshot_path;
+ node->snapshot(snapshot_closure);
+}
+
+void ReplicationState::set_ext_snapshot_path(const std::string& snapshot_path) {
+ this->ext_snapshot_path = snapshot_path;
+}
+
+const std::string &ReplicationState::get_ext_snapshot_path() const {
+ return ext_snapshot_path;
+}
+
+void ReplicationState::do_dummy_write() {
+ if(node->leader_id().is_empty()) {
+ LOG(ERROR) << "Could not do a dummy write, as node does not have a leader";
+ return ;
+ }
+
+ const std::string & leader_addr = node->leader_id().to_string();
+ const std::string protocol = api_uses_ssl ? "https" : "http";
+ std::string url = get_leader_url_path(leader_addr, "/health", protocol);
+
+ std::string api_res;
+ std::map<std::string, std::string> res_headers;
+ long status_code = HttpClient::post_response(url, "", api_res, res_headers);
+
+ LOG(INFO) << "Dummy write to " << url << ", status = " << status_code << ", response = " << api_res;
+}
+
void InitSnapshotClosure::Run() {
// Auto delete this after Run()
std::unique_ptr<InitSnapshotClosure> self_guard(this);
@@ -516,6 +585,35 @@ void InitSnapshotClosure::Run() {
replication_state->reset_db();
replication_state->init_db();
} else {
- LOG(ERROR) << "Init snapshot failed, error: " << status().error_str();
+ LOG(ERROR) << "Init snapshot failed, error: " << status().error_str() << ", code: " << status().error_code();
}
}
+
+void OnDemandSnapshotClosure::Run() {
+ // Auto delete this after Run()
+ std::unique_ptr<OnDemandSnapshotClosure> self_guard(this);
+
+ replication_state->set_ext_snapshot_path("");
+
+ req.last_chunk_aggregate = true;
+ res.final = true;
+
+ nlohmann::json response;
+ uint32_t status_code;
+
+ if(status().ok()) {
+ LOG(INFO) << "On demand snapshot succeeded!";
+ status_code = 201;
+ response["success"] = true;
+ } else {
+ LOG(ERROR) << "On demand snapshot failed, error: " << status().error_str() << ", code: " << status().error_code();
+ status_code = 500;
+ response["success"] = false;
+ response["error"] = status().error_str();
+ }
+
+ res.status_code = status_code;
+ res.body = response.dump();
+
+ HttpServer::stream_response(req, res);
+}