Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
On demand snapshot API.
kishorenc8a310bbb8267
diff --git a/include/core_api.h b/include/core_api.hindex 1851385c..ee0bbb25 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -79,10 +79,14 @@ bool get_debug(http_req& req, http_res& res);bool get_health(http_req& req, http_res& res);+bool post_health(http_req& req, http_res& res);+bool get_metrics_json(http_req& req, http_res& res);bool get_log_sequence(http_req& req, http_res& res);+bool post_snapshot(http_req& req, http_res& res);+// Misc helpersbool raft_write_send_response(void *data);diff --git a/include/http_server.h b/include/http_server.hindex 847cecef..51f8bade 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -58,8 +58,6 @@ private:bool exit_loop = false;-private:-std::string version;// must be a vector since order of routes matter@@ -173,4 +171,6 @@ public:static void on_deferred_process_request(h2o_timer_t *entry);void defer_processing(http_req& req, http_res& res, size_t timeout_ms);++ void do_snapshot(const std::string& snapshot_path, http_req& req, http_res& res);};diff --git a/include/raft_server.h b/include/raft_server.hindex 4bfceef9..b5e11693 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -72,6 +72,23 @@ public:void Run();};+// Closure that fires when requested+class OnDemandSnapshotClosure : public braft::Closure {+private:+ ReplicationState* replication_state;+ http_req& req;+ http_res& res;++public:++ OnDemandSnapshotClosure(ReplicationState* replication_state, http_req& req, http_res& res):+ replication_state(replication_state), req(req), res(res) {}++ ~OnDemandSnapshotClosure() {}++ void Run();+};+// Implements braft::StateMachine.class ReplicationState : public braft::StateMachine {@@ -96,6 +113,10 @@ private:std::atomic<bool>& shut_down;+ std::string raft_dir_path;++ std::string ext_snapshot_path;+public:static constexpr const char* log_dir_name = "log";@@ -154,9 +175,15 @@ public:void reset_db();+ void do_snapshot(const std::string& snapshot_path, http_req& req, http_res& res);+static std::string to_nodes_config(const butil::EndPoint &peering_endpoint, const int api_port,const std::string &nodes_config);+ void set_ext_snapshot_path(const std::string &snapshot_path);++ const std::string& get_ext_snapshot_path() const;+static constexpr const char* REPLICATION_MSG = "raft_replication";private:@@ -167,8 +194,12 @@ private:void on_apply(braft::Iterator& iter);struct SnapshotArg {- rocksdb::DB* db;+ ReplicationState* replication_state;braft::SnapshotWriter* writer;+ std::string state_dir_path;+ std::string db_dir_path;+ std::string db_snapshot_path;+ std::string ext_snapshot_path;braft::Closure* done;};@@ -217,5 +248,10 @@ private:LOG(INFO) << "Node stops following " << ctx;}- void follower_write(http_req *request, http_res *response) const;+ void write_to_leader(http_req *request, http_res *response) const;++ void do_dummy_write();++ std::string get_leader_url_path(const std::string& leader_addr, const std::string& path,+ const std::string& protocol) const;};diff --git a/src/core_api.cpp b/src/core_api.cppindex 7270671d..97e647d6 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -203,6 +203,7 @@ bool get_health(http_req & req, http_res & res) {nlohmann::json result;bool alive = server->is_alive();result["ok"] = alive;+if(alive) {res.set_body(200, result.dump());} else {@@ -212,6 +213,19 @@ bool get_health(http_req & req, http_res & res) {return alive;}+bool post_health(http_req & req, http_res & res) {+ nlohmann::json result;+ bool alive = server->is_alive();+ result["ok"] = alive;++ if(alive) {+ res.set_body(200, result.dump());+ } else {+ res.set_body(503, result.dump());+ }++ return alive;+}bool get_metrics_json(http_req &req, http_res &res) {nlohmann::json result;@@ -1289,6 +1303,25 @@ bool raft_write_send_response(void *data) {return true;}+bool post_snapshot(http_req& req, http_res& res) {+ const std::string SNAPSHOT_PATH = "snapshot_path";++ res.status_code = 201;+ res.content_type_header = "application/json";++ if(req.params.count(SNAPSHOT_PATH) == 0) {+ req.last_chunk_aggregate = true;+ res.final = true;+ res.set_400(std::string("Parameter `") + SNAPSHOT_PATH + "` is required.");+ HttpServer::stream_response(req, res);+ return false;+ }++ server->do_snapshot(req.params[SNAPSHOT_PATH], req, res);++ return true;+}+bool get_synonyms(http_req &req, http_res &res) {CollectionManager & collectionManager = CollectionManager::get_instance();Collection *collection = collectionManager.get_collection(req.params["collection"]);diff --git a/src/http_server.cpp b/src/http_server.cppindex 4932fadf..25741a39 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -245,7 +245,8 @@ std::map<std::string, std::string> HttpServer::parse_query(const std::string& qureturn query_map;}-uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, const std::string & http_method, route_path** found_rpath) {+uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, const std::string & http_method,+ route_path** found_rpath) {for (const auto& index_route : routes) {const route_path & rpath = index_route.second;@@ -492,10 +493,14 @@ int HttpServer::process_request(http_req* request, http_res* response, route_patconst h2o_custom_req_handler_t *handler) {//LOG(INFO) << "process_request called";+ bool operations_path = (rpath->path_parts.size() != 0 && rpath->path_parts[0] == "operations");++ //LOG(INFO) << "operations_path: " << operations_path;// for writes, we delegate to replication_state to handle response- if(rpath->http_method == "POST" || rpath->http_method == "PUT" || rpath->http_method == "DELETE" ||- rpath->http_method == "PATCH") {+ if(!operations_path &&+ (rpath->http_method == "POST" || rpath->http_method == "PUT" ||+ rpath->http_method == "DELETE" || rpath->http_method == "PATCH")) {handler->http_server->get_replication_state()->write(request, response);return 0;}@@ -800,3 +805,7 @@ bool HttpServer::on_request_proceed_message(void *data) {bool HttpServer::has_exited() const {return exit_loop;}++void HttpServer::do_snapshot(const std::string& snapshot_path, http_req& req, http_res& res) {+ return replication_state->do_snapshot(snapshot_path, req, res);+}diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex b4d52def..38b396c1 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -13,23 +13,21 @@ extern "C" {#endifvoid master_server_routes() {- // collection management- server->post("/collections", post_create_collection);- server->get("/collections", get_collections);- server->del("/collections/:collection", del_drop_collection);- server->get("/collections/:collection", get_collection_summary);+ // collection operations+ // NOTE: placing this first to score an immediate hit on O(N) route search+ server->get("/collections/:collection/documents/search", get_search);- // document management - `/documents/:id` end-points must be placed last in the list+ // document management+ // NOTE:`/documents/:id` end-points must be placed last in the listserver->post("/collections/:collection/documents", post_add_document);- server->patch("/collections/:collection/documents/:id", patch_update_document);- server->get("/collections/:collection/documents/search", get_search);+ server->del("/collections/:collection/documents", del_remove_documents, false, true);server->post("/collections/:collection/documents/import", post_import_documents, true, true);server->get("/collections/:collection/documents/export", get_export_documents, false, true);server->get("/collections/:collection/documents/:id", get_fetch_document);+ server->patch("/collections/:collection/documents/:id", patch_update_document);server->del("/collections/:collection/documents/:id", del_remove_document);- server->del("/collections/:collection/documents", del_remove_documents, false, true);server->get("/collections/:collection/overrides", get_overrides);server->get("/collections/:collection/overrides/:id", get_override);@@ -41,6 +39,12 @@ void master_server_routes() {server->put("/collections/:collection/synonyms/:id", put_synonym);server->del("/collections/:collection/synonyms/:id", del_synonym);+ // collection management+ server->post("/collections", post_create_collection);+ server->get("/collections", get_collections);+ server->del("/collections/:collection", del_drop_collection);+ server->get("/collections/:collection", get_collection_summary);+server->get("/aliases", get_aliases);server->get("/aliases/:alias", get_alias);server->put("/aliases/:alias", put_upsert_alias);@@ -55,7 +59,9 @@ void master_server_routes() {server->get("/metrics.json", get_metrics_json);server->get("/debug", get_debug);server->get("/health", get_health);+ server->post("/health", post_health);server->get("/sequence", get_log_sequence);+ server->post("/operations/snapshot", post_snapshot, false, true);}int main(int argc, char **argv) {diff --git a/src/raft_server.cpp b/src/raft_server.cppindex fd618050..25fad5e0 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -25,6 +25,8 @@ int ReplicationState::start(const butil::EndPoint & peering_endpoint, const intint election_timeout_ms, int snapshot_interval_s,const std::string & raft_dir, const std::string & nodes) {+ this->raft_dir_path = raft_dir;+braft::NodeOptions node_options;std::string actual_nodes_config = to_nodes_config(peering_endpoint, api_port, nodes);@@ -105,7 +107,7 @@ void ReplicationState::write(http_req* request, http_res* response) {}if (!node->is_leader()) {- return follower_write(request, response);+ return write_to_leader(request, response);}// Serialize request to replicated WAL so that all the nodes in the group receive it as well.@@ -130,7 +132,7 @@ void ReplicationState::write(http_req* request, http_res* response) {return node->apply(task);}-void ReplicationState::follower_write(http_req *request, http_res *response) const {+void ReplicationState::write_to_leader(http_req *request, http_res *response) const {if(node->leader_id().is_empty()) {// Handle no leader scenarioLOG(ERROR) << "Rejecting write: could not find a leader.";@@ -163,12 +165,9 @@ void ReplicationState::follower_write(http_req *request, http_res *response) conthread_pool->enqueue([leader_addr, request, response, server, this]() {auto raw_req = request->_req;- std::string scheme = std::string(raw_req->scheme->name.base, raw_req->scheme->name.len);- std::vector<std::string> addr_parts;- StringUtils::split(leader_addr, addr_parts, ":");- std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];- const std::string & path = std::string(raw_req->path.base, raw_req->path.len);- std::string url = scheme + "://" + leader_host_port + path;+ const std::string& path = std::string(raw_req->path.base, raw_req->path.len);+ const std::string& scheme = std::string(raw_req->scheme->name.base, raw_req->scheme->name.len);+ const std::string url = get_leader_url_path(leader_addr, path, scheme);std::map<std::string, std::string> res_headers;@@ -222,6 +221,15 @@ void ReplicationState::follower_write(http_req *request, http_res *response) con});}+std::string ReplicationState::get_leader_url_path(const std::string& leader_addr, const std::string& path,+ const std::string& protocol) const {+ std::vector<std::string> addr_parts;+ StringUtils::split(leader_addr, addr_parts, ":");+ std::string leader_host_port = addr_parts[0] + ":" + addr_parts[2];+ std::string url = protocol + "://" + leader_host_port + path;+ return url;+}+void ReplicationState::on_apply(braft::Iterator& iter) {//LOG(INFO) << "ReplicationState::on_apply";@@ -297,44 +305,73 @@ void* ReplicationState::save_snapshot(void* arg) {std::unique_ptr<SnapshotArg> arg_guard(sa);brpc::ClosureGuard done_guard(sa->done);- std::string snapshot_path = sa->writer->get_path() + "/" + db_snapshot_name;-- rocksdb::Checkpoint* checkpoint = nullptr;- rocksdb::Status status = rocksdb::Checkpoint::Create(sa->db, &checkpoint);-- if(!status.ok()) {- LOG(WARNING) << "Checkpoint Create failed, msg:" << status.ToString();- return nullptr;- }-- std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);- status = checkpoint->CreateCheckpoint(snapshot_path);- if(!status.ok()) {- LOG(WARNING) << "Checkpoint CreateCheckpoint failed, msg:" << status.ToString();- return nullptr;- }+ // add the db snapshot files to writer state+ butil::FileEnumerator dir_enum(butil::FilePath(sa->db_snapshot_path), false, butil::FileEnumerator::FILES);- butil::FileEnumerator dir_enum(butil::FilePath(snapshot_path),false, butil::FileEnumerator::FILES);- for (butil::FilePath name = dir_enum.Next(); !name.empty(); name = dir_enum.Next()) {- std::string file_name = std::string(db_snapshot_name) + "/" + name.BaseName().value();+ for (butil::FilePath file = dir_enum.Next(); !file.empty(); file = dir_enum.Next()) {+ std::string file_name = std::string(db_snapshot_name) + "/" + file.BaseName().value();if (sa->writer->add_file(file_name) != 0) {sa->done->status().set_error(EIO, "Fail to add file to writer.");return nullptr;}}+ // if an external snapshot is requested, copy both state and data directory into that+ if(!sa->ext_snapshot_path.empty()) {+ LOG(INFO) << "Copying system snapshot to external snapshot directory at " << sa->ext_snapshot_path;+ if(!butil::DirectoryExists(butil::FilePath(sa->ext_snapshot_path))) {+ butil::CreateDirectory(butil::FilePath(sa->ext_snapshot_path), true);+ }+ butil::CopyDirectory(butil::FilePath(sa->state_dir_path), butil::FilePath(sa->ext_snapshot_path), true);+ butil::CopyDirectory(butil::FilePath(sa->db_dir_path), butil::FilePath(sa->ext_snapshot_path), true);+ }++ // NOTE: *must* do a dummy write here since snapshots cannot be triggered if no write has happened since the+ // last snapshot. By doing a dummy write right after a snapshot, we ensure that this can never be the case.+ sa->replication_state->do_dummy_write();+LOG(INFO) << "save_snapshot done";return nullptr;}+// this method is serial to on_apply so guarantees a snapshot view of the state machinevoid ReplicationState::on_snapshot_save(braft::SnapshotWriter* writer, braft::Closure* done) {LOG(INFO) << "on_snapshot_save";- // Start a new bthread to avoid blocking StateMachine since it could be slow to write data to disk++ rocksdb::Checkpoint* checkpoint = nullptr;+ rocksdb::Status status = rocksdb::Checkpoint::Create(store->_get_db_unsafe(), &checkpoint);++ if(!status.ok()) {+ LOG(WARNING) << "Checkpoint Create failed, msg:" << status.ToString();+ done->status().set_error(EIO, "Checkpoint Create failed.");+ }++ std::string db_snapshot_path = writer->get_path() + "/" + db_snapshot_name;++ std::unique_ptr<rocksdb::Checkpoint> checkpoint_guard(checkpoint);+ status = checkpoint->CreateCheckpoint(db_snapshot_path);++ if(!status.ok()) {+ LOG(WARNING) << "Checkpoint CreateCheckpoint failed at snapshot path: "+ << db_snapshot_path << ", msg:" << status.ToString();+ done->status().set_error(EIO, "CreateCheckpoint failed.");+ }+SnapshotArg* arg = new SnapshotArg;- arg->db = store->_get_db_unsafe();+ arg->replication_state = this;arg->writer = writer;+ arg->state_dir_path = raft_dir_path;+ arg->db_dir_path = store->get_state_dir_path();+ arg->db_snapshot_path = db_snapshot_path;arg->done = done;++ if(!ext_snapshot_path.empty()) {+ arg->ext_snapshot_path = ext_snapshot_path;+ ext_snapshot_path = "";+ }++ // Start a new bthread to avoid blocking StateMachine since it could be slow to write data to diskbthread_t tid;bthread_start_urgent(&tid, NULL, save_snapshot, arg);}@@ -507,6 +544,38 @@ uint64_t ReplicationState::node_state() const {return node_status.state;}+void ReplicationState::do_snapshot(const std::string& snapshot_path, http_req& req, http_res& res) {+ LOG(INFO) << "Triggerring an on demand snapshot...";+ OnDemandSnapshotClosure* snapshot_closure = new OnDemandSnapshotClosure(this, req, res);+ ext_snapshot_path = snapshot_path;+ node->snapshot(snapshot_closure);+}++void ReplicationState::set_ext_snapshot_path(const std::string& snapshot_path) {+ this->ext_snapshot_path = snapshot_path;+}++const std::string &ReplicationState::get_ext_snapshot_path() const {+ return ext_snapshot_path;+}++void ReplicationState::do_dummy_write() {+ if(node->leader_id().is_empty()) {+ LOG(ERROR) << "Could not do a dummy write, as node does not have a leader";+ return ;+ }++ const std::string & leader_addr = node->leader_id().to_string();+ const std::string protocol = api_uses_ssl ? "https" : "http";+ std::string url = get_leader_url_path(leader_addr, "/health", protocol);++ std::string api_res;+ std::map<std::string, std::string> res_headers;+ long status_code = HttpClient::post_response(url, "", api_res, res_headers);++ LOG(INFO) << "Dummy write to " << url << ", status = " << status_code << ", response = " << api_res;+}+void InitSnapshotClosure::Run() {// Auto delete this after Run()std::unique_ptr<InitSnapshotClosure> self_guard(this);@@ -516,6 +585,35 @@ void InitSnapshotClosure::Run() {replication_state->reset_db();replication_state->init_db();} else {- LOG(ERROR) << "Init snapshot failed, error: " << status().error_str();+ LOG(ERROR) << "Init snapshot failed, error: " << status().error_str() << ", code: " << status().error_code();}}++void OnDemandSnapshotClosure::Run() {+ // Auto delete this after Run()+ std::unique_ptr<OnDemandSnapshotClosure> self_guard(this);++ replication_state->set_ext_snapshot_path("");++ req.last_chunk_aggregate = true;+ res.final = true;++ nlohmann::json response;+ uint32_t status_code;++ if(status().ok()) {+ LOG(INFO) << "On demand snapshot succeeded!";+ status_code = 201;+ response["success"] = true;+ } else {+ LOG(ERROR) << "On demand snapshot failed, error: " << status().error_str() << ", code: " << status().error_code();+ status_code = 500;+ response["success"] = false;+ response["error"] = status().error_str();+ }++ res.status_code = status_code;+ res.body = response.dump();++ HttpServer::stream_response(req, res);+}