Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Expose write stats.
Kishore Nallan6da39ff51ca6
diff --git a/include/app_metrics.h b/include/app_metrics.hindex cf5bbbf2..caffeae1 100644--- a/include/app_metrics.h+++ b/include/app_metrics.h@@ -10,8 +10,6 @@ class AppMetrics {private:mutable std::shared_mutex mutex;- static inline const std::string SEARCH_LABEL = "search";-// stores last complete windowspp::sparse_hash_map<std::string, uint64_t>* counts;spp::sparse_hash_map<std::string, uint64_t>* durations;@@ -37,6 +35,10 @@ private:}public:+ static inline const std::string SEARCH_LABEL = "search";+ static inline const std::string DOC_WRITE_LABEL = "write";+ static inline const std::string IMPORT_LABEL = "import";+ static inline const std::string DOC_DELETE_LABEL = "delete";static const uint64_t METRICS_REFRESH_INTERVAL_MS = 10 * 1000;@@ -53,71 +55,14 @@ public:(*current_counts)[identifier] += count;}- void increment_search_count(uint64_t count) {- std::unique_lock lock(mutex);- (*current_counts)[SEARCH_LABEL] += count;- }-void increment_duration(const std::string& identifier, uint64_t duration) {std::unique_lock lock(mutex);(*current_durations)[identifier] += duration;}- void increment_search_duration(uint64_t duration) {- std::unique_lock lock(mutex);- (*current_durations)[SEARCH_LABEL] += duration;- }+ void increment_write_metrics(uint64_t route_hash, uint64_t duration);- void window_reset() {- std::unique_lock lock(mutex);+ void window_reset();- delete counts;- counts = current_counts;- current_counts = new spp::sparse_hash_map<std::string, uint64_t>();-- delete durations;- durations = current_durations;- current_durations = new spp::sparse_hash_map<std::string, uint64_t>();- }-- void get(const std::string& rps_key, const std::string& latency_key, nlohmann::json &result) const {- std::shared_lock lock(mutex);-- uint64_t total_counts = 0;- auto SEARCH_RPS_KEY = SEARCH_LABEL + "_" + rps_key;- auto SEARCH_LATENCY_KEY = SEARCH_LABEL + "_" + latency_key;-- result[rps_key] = nlohmann::json::object();- for(const auto& kv: *counts) {- if(kv.first == SEARCH_LABEL) {- result[SEARCH_RPS_KEY] = double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000);- } else {- result[rps_key][kv.first] = (double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000));- total_counts += kv.second;- }- }-- result["total_" + rps_key] = double(total_counts) / (METRICS_REFRESH_INTERVAL_MS / 1000);-- result[latency_key] = nlohmann::json::object();-- for(const auto& kv: *durations) {- auto counter_it = counts->find(kv.first);- if(counter_it != counts->end() && counter_it->second != 0) {- if(kv.first == SEARCH_LABEL) {- result[SEARCH_LATENCY_KEY] = (double(kv.second) / counter_it->second);- } else {- result[latency_key][kv.first] = (double(kv.second) / counter_it->second);- }- }- }-- if(!result.contains(SEARCH_RPS_KEY)) {- result[SEARCH_RPS_KEY] = 0;- }-- if(!result.contains(SEARCH_LATENCY_KEY)) {- result[SEARCH_LATENCY_KEY] = 0;- }- }+ void get(const std::string& rps_key, const std::string& latency_key, nlohmann::json &result) const;};\ No newline at end of filediff --git a/include/core_api.h b/include/core_api.hindex 470f7cdf..006d21b6 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -102,3 +102,8 @@ bool post_config(const std::shared_ptr<http_req>& req, const std::shared_ptr<httvoid get_collections_for_auth(std::map<std::string, std::string> &req_params, const std::string &body,const route_path &rpath, std::vector<std::string> &collections);+bool is_doc_import_route(uint64_t route_hash);++bool is_doc_write_route(uint64_t route_hash);++bool is_doc_del_route(uint64_t route_hash);diff --git a/include/http_data.h b/include/http_data.hindex 5dbde5ff..1072fef2 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -33,33 +33,6 @@ enum class ROUTE_CODES {ALREADY_HANDLED = 2,};-class await_t {-private:-- std::mutex mcv;- std::condition_variable cv;- bool ready;--public:-- await_t(): ready(false) {}-- void notify() {- // Ideally we don't need lock over notify but it is needed here because- // the parent object could be deleted after lock on mutex is released but- // before notify can be called on condition variable.- std::lock_guard<std::mutex> lk(mcv);- ready = true;- cv.notify_all();- }-- void wait() {- auto lk = std::unique_lock<std::mutex>(mcv);- cv.wait(lk, [&] { return ready; });- ready = false;- }-};-struct http_res {uint32_t status_code;std::string content_type_header;@@ -264,16 +237,16 @@ struct http_req {~http_req() {//LOG(INFO) << "~http_req " << this;-if(!deserialized_request) {uint64_t now = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();-uint64_t ms_since_start = (now - start_ts) / 1000;- std::string metric_identifier = http_method + " " + path_without_query;+ std::string metric_identifier = http_method + " " + path_without_query;AppMetrics::get_instance().increment_duration(metric_identifier, ms_since_start);+ AppMetrics::get_instance().increment_write_metrics(route_hash, ms_since_start);+// log slow request if logging is enabledConfig& config = Config::get_instance();std::string query_string = "?";@@ -309,7 +282,7 @@ struct http_req {// NOTE: we don't ser/de all fields, only ones needed for write forwarding// Take care to check for existence of key to ensure backward compatibility during upgrade- void deserialize(const std::string& serialized_content) {+ void load_from_json(const std::string& serialized_content, const bool is_deserialized) {nlohmann::json content = nlohmann::json::parse(serialized_content);route_hash = content["route_hash"];body += content["body"];@@ -325,10 +298,10 @@ struct http_req {log_index = content.count("log_index") != 0 ? content["log_index"].get<int64_t>() : 0;_req = nullptr;- deserialized_request = true;+ deserialized_request = is_deserialized;}- std::string serialize() const {+ std::string to_json() const {nlohmann::json content;content["route_hash"] = route_hash;content["params"] = params;diff --git a/include/http_server.h b/include/http_server.hindex 5f73ee9c..51427a43 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -91,8 +91,11 @@ private:std::string version;- // must be a vector since order of routes matter- std::vector<std::pair<uint64_t, route_path>> routes;+ // must be a vector since order of routes entered matter+ std::vector<std::pair<uint64_t, route_path>> route_hash_to_path;++ // also have a hashmap for quick lookup of individual routes+ std::unordered_map<uint64_t, route_path> route_hash_to_path_map;const std::string listen_address;@@ -156,6 +159,8 @@ public:bool is_alive() const;+ bool is_leader() const;+uint64_t node_state() const;void set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params, const std::string& body,diff --git a/include/raft_server.h b/include/raft_server.hindex f09b96f8..a82196d6 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -212,6 +212,8 @@ public:int64_t get_num_queued_writes();+ bool is_leader();+private:friend class ReplicationClosure;diff --git a/include/string_utils.h b/include/string_utils.hindex 7538c8b8..3d386a68 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -317,4 +317,6 @@ struct StringUtils {const std::string& replace);static std::string trim_curly_spaces(const std::string& str);++ static bool ends_with(std::string const &str, std::string const &ending);};\ No newline at end of filediff --git a/src/app_metrics.cpp b/src/app_metrics.cppnew file mode 100644index 00000000..0d6dbc3d--- /dev/null+++ b/src/app_metrics.cpp@@ -0,0 +1,114 @@+#include "app_metrics.h"+#include "core_api.h"++void AppMetrics::increment_write_metrics(uint64_t route_hash, uint64_t duration) {+ if(is_doc_import_route(route_hash)) {+ AppMetrics::get_instance().increment_duration(AppMetrics::IMPORT_LABEL, duration);+ AppMetrics::get_instance().increment_count(AppMetrics::IMPORT_LABEL, 1);+ }++ else if(is_doc_write_route(route_hash)) {+ AppMetrics::get_instance().increment_duration(AppMetrics::DOC_WRITE_LABEL, duration);+ AppMetrics::get_instance().increment_count(AppMetrics::DOC_WRITE_LABEL, 1);+ }++ else if(is_doc_del_route(route_hash)) {+ AppMetrics::get_instance().increment_duration(AppMetrics::DOC_DELETE_LABEL, duration);+ AppMetrics::get_instance().increment_count(AppMetrics::DOC_DELETE_LABEL, 1);+ }+}++void AppMetrics::get(const std::string& rps_key, const std::string& latency_key, nlohmann::json& result) const {+ std::shared_lock lock(mutex);++ uint64_t total_counts = 0;++ auto SEARCH_RPS_KEY = SEARCH_LABEL + "_" + rps_key;+ auto SEARCH_LATENCY_KEY = SEARCH_LABEL + "_" + latency_key;++ auto IMPORT_RPS_KEY = IMPORT_LABEL + "_" + rps_key;+ auto IMPORT_LATENCY_KEY = IMPORT_LABEL + "_" + latency_key;++ auto DOC_WRITE_RPS_KEY = DOC_WRITE_LABEL + "_" + rps_key;+ auto DOC_WRITE_LATENCY_KEY = DOC_WRITE_LABEL + "_" + latency_key;++ auto DOC_DELETE_RPS_KEY = DOC_DELETE_LABEL + "_" + rps_key;+ auto DOC_DELETE_LATENCY_KEY = DOC_DELETE_LABEL + "_" + latency_key;++ result[rps_key] = nlohmann::json::object();+ for(const auto& kv: *counts) {+ if(kv.first == SEARCH_LABEL) {+ result[SEARCH_RPS_KEY] = double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000);+ }++ else if(kv.first == IMPORT_LABEL) {+ result[IMPORT_RPS_KEY] = double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000);+ }++ else if(kv.first == DOC_WRITE_LABEL) {+ result[DOC_WRITE_RPS_KEY] = double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000);+ }++ else if(kv.first == DOC_DELETE_LABEL) {+ result[DOC_DELETE_RPS_KEY] = double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000);+ }++ else {+ result[rps_key][kv.first] = (double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000));+ total_counts += kv.second;+ }+ }++ result["total_" + rps_key] = double(total_counts) / (METRICS_REFRESH_INTERVAL_MS / 1000);++ result[latency_key] = nlohmann::json::object();++ for(const auto& kv: *durations) {+ auto counter_it = counts->find(kv.first);+ if(counter_it != counts->end() && counter_it->second != 0) {+ if(kv.first == SEARCH_LABEL) {+ result[SEARCH_LATENCY_KEY] = (double(kv.second) / counter_it->second);+ }++ else if(kv.first == IMPORT_LABEL) {+ result[IMPORT_LATENCY_KEY] = (double(kv.second) / counter_it->second);+ }++ else if(kv.first == DOC_WRITE_LABEL) {+ result[DOC_WRITE_LATENCY_KEY] = (double(kv.second) / counter_it->second);+ }++ else if(kv.first == DOC_DELETE_LABEL) {+ result[DOC_DELETE_LATENCY_KEY] = (double(kv.second) / counter_it->second);+ }++ else {+ result[latency_key][kv.first] = (double(kv.second) / counter_it->second);+ }+ }+ }++ std::vector<std::string> keys_to_check = {+ SEARCH_RPS_KEY, IMPORT_RPS_KEY, DOC_WRITE_RPS_KEY, DOC_DELETE_RPS_KEY,+ SEARCH_LATENCY_KEY, IMPORT_LATENCY_KEY, DOC_WRITE_LATENCY_KEY, DOC_DELETE_LATENCY_KEY+ };++ for(auto& key: keys_to_check) {+ if(!result.contains(key)) {+ result[key] = 0;+ }+ }+}++void AppMetrics::window_reset() {+ std::unique_lock lock(mutex);++ delete counts;+ counts = current_counts;+ current_counts = new spp::sparse_hash_map<std::string, uint64_t>();++ delete durations;+ durations = current_durations;+ current_durations = new spp::sparse_hash_map<std::string, uint64_t>();+}+diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cppindex 9593e765..c749b418 100644--- a/src/batched_indexer.cpp+++ b/src/batched_indexer.cpp@@ -43,7 +43,7 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh//LOG(INFO) << "req_id: " << req->start_ts << ", chunk_sequence: " << chunk_sequence;- store->insert(request_chunk_key, req->serialize());+ store->insert(request_chunk_key, req->to_json());req->body = "";{@@ -110,7 +110,7 @@ void BatchedIndexer::run() {std::shared_ptr<http_req>& orig_req = orig_req_res.req;auto _req = orig_req->_req;orig_req->body = prev_body;- orig_req->deserialize(iter->value().ToString());+ orig_req->load_from_json(iter->value().ToString(), _req == nullptr);orig_req->_req = _req;// update thread local for reference during a crashdiff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 0034c71b..18edba3b 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -829,8 +829,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reuint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();- AppMetrics::get_instance().increment_search_count(1);- AppMetrics::get_instance().increment_search_duration(timeMillis);+ AppMetrics::get_instance().increment_count(AppMetrics::SEARCH_LABEL, 1);+ AppMetrics::get_instance().increment_duration(AppMetrics::SEARCH_LABEL, timeMillis);if(!result_op.ok()) {return Option<bool>(result_op.code(), result_op.error());diff --git a/src/core_api.cpp b/src/core_api.cppindex 6d89dc51..730e9649 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1,4 +1,3 @@-#include <regex>#include <chrono>#include <thread>#include <app_metrics.h>@@ -1407,3 +1406,21 @@ bool del_synonym(const std::shared_ptr<http_req>& req, const std::shared_ptr<httres->set_200(res_json.dump());return true;}++bool is_doc_import_route(uint64_t route_hash) {+ route_path* rpath;+ bool found = server->get_route(route_hash, &rpath);+ return found && (rpath->handler == post_import_documents);+}++bool is_doc_write_route(uint64_t route_hash) {+ route_path* rpath;+ bool found = server->get_route(route_hash, &rpath);+ return found && (rpath->handler == post_add_document || rpath->handler == patch_update_document);+}++bool is_doc_del_route(uint64_t route_hash) {+ route_path* rpath;+ bool found = server->get_route(route_hash, &rpath);+ return found && (rpath->handler == del_remove_document || rpath->handler == del_remove_documents);+}diff --git a/src/http_server.cpp b/src/http_server.cppindex f97d1f94..c5434b7e 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -230,7 +230,7 @@ h2o_pathconf_t* HttpServer::register_handler(h2o_hostconf_t *hostconf, const chauint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, const std::string & http_method,route_path** found_rpath) {- for (const auto& index_route : routes) {+ for (const auto& index_route : route_hash_to_path) {const route_path & rpath = index_route.second;if(rpath.path_parts.size() != path_parts.size() || rpath.http_method != http_method) {@@ -817,35 +817,40 @@ void HttpServer::get(const std::string & path, bool (*handler)(const std::sharedstd::vector<std::string> path_parts;StringUtils::split(path, path_parts, "/");route_path rpath("GET", path_parts, handler, async_req, async_res);- routes.emplace_back(rpath.route_hash(), rpath);+ route_hash_to_path.emplace_back(rpath.route_hash(), rpath);+ route_hash_to_path_map.emplace(rpath.route_hash(), rpath);}void HttpServer::post(const std::string & path, bool (*handler)(const std::shared_ptr<http_req>&, const std::shared_ptr<http_res>&), bool async_req, bool async_res) {std::vector<std::string> path_parts;StringUtils::split(path, path_parts, "/");route_path rpath("POST", path_parts, handler, async_req, async_res);- routes.emplace_back(rpath.route_hash(), rpath);+ route_hash_to_path.emplace_back(rpath.route_hash(), rpath);+ route_hash_to_path_map.emplace(rpath.route_hash(), rpath);}void HttpServer::put(const std::string & path, bool (*handler)(const std::shared_ptr<http_req>&, const std::shared_ptr<http_res>&), bool async_req, bool async_res) {std::vector<std::string> path_parts;StringUtils::split(path, path_parts, "/");route_path rpath("PUT", path_parts, handler, async_req, async_res);- routes.emplace_back(rpath.route_hash(), rpath);+ route_hash_to_path.emplace_back(rpath.route_hash(), rpath);+ route_hash_to_path_map.emplace(rpath.route_hash(), rpath);}void HttpServer::patch(const std::string & path, bool (*handler)(const std::shared_ptr<http_req>&, const std::shared_ptr<http_res>&), bool async_req, bool async_res) {std::vector<std::string> path_parts;StringUtils::split(path, path_parts, "/");route_path rpath("PATCH", path_parts, handler, async_req, async_res);- routes.emplace_back(rpath.route_hash(), rpath);+ route_hash_to_path.emplace_back(rpath.route_hash(), rpath);+ route_hash_to_path_map.emplace(rpath.route_hash(), rpath);}void HttpServer::del(const std::string & path, bool (*handler)(const std::shared_ptr<http_req>&, const std::shared_ptr<http_res>&), bool async_req, bool async_res) {std::vector<std::string> path_parts;StringUtils::split(path, path_parts, "/");route_path rpath("DELETE", path_parts, handler, async_req, async_res);- routes.emplace_back(rpath.route_hash(), rpath);+ route_hash_to_path.emplace_back(rpath.route_hash(), rpath);+ route_hash_to_path_map.emplace(rpath.route_hash(), rpath);}void HttpServer::on(const std::string & message, bool (*handler)(void*)) {@@ -896,14 +901,14 @@ bool HttpServer::is_alive() const {}bool HttpServer::get_route(uint64_t hash, route_path** found_rpath) {- for (auto& hash_route : routes) {- if(hash_route.first == hash) {- *found_rpath = &hash_route.second;- return true;- }+ auto route_hash_it = route_hash_to_path_map.find(hash);++ if(route_hash_it == route_hash_to_path_map.end()) {+ return false;}- return false;+ *found_rpath = &route_hash_it->second;+ return true;}uint64_t HttpServer::node_state() const {@@ -1028,3 +1033,7 @@ void HttpServer::persist_applying_index() {int64_t HttpServer::get_num_queued_writes() {return replication_state->get_num_queued_writes();}++bool HttpServer::is_leader() const {+ return replication_state->is_leader();+}diff --git a/src/raft_server.cpp b/src/raft_server.cppindex d6391be8..4958796d 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -212,7 +212,7 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const std// NOTE: actual write must be done only on the `on_apply` method to maintain consistency.butil::IOBufBuilder bufBuilder;- bufBuilder << request->serialize();+ bufBuilder << request->to_json();//LOG(INFO) << "write() pre request ref count " << request.use_count();@@ -364,7 +364,7 @@ void ReplicationState::on_apply(braft::Iterator& iter) {if(!iter.done()) {// indicates log serialized request- request_generated->deserialize(iter.data().to_string());+ request_generated->load_from_json(iter.data().to_string(), true);}request_generated->log_index = iter.index();@@ -779,6 +779,16 @@ int64_t ReplicationState::get_num_queued_writes() {return batched_indexer->get_queued_writes();}+bool ReplicationState::is_leader() {+ std::shared_lock lock(node_mutex);++ if(!node) {+ return false;+ }++ return node->is_leader();+}+void OnDemandSnapshotClosure::Run() {// Auto delete this after Done()std::unique_ptr<OnDemandSnapshotClosure> self_guard(this);diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 6a525304..0811b3fc 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -291,6 +291,14 @@ std::string StringUtils::trim_curly_spaces(const std::string& str) {return right_trimmed;}+bool StringUtils::ends_with(const std::string& str, const std::string& ending) {+ if (str.length() >= ending.length()) {+ return (0 == str.compare (str.length() - ending.length(), ending.length(), ending));+ } else {+ return false;+ }+}+/*size_t StringUtils::unicode_length(const std::string& bytes) {std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;return utf8conv.from_bytes(bytes).size();diff --git a/test/app_metrics_test.cpp b/test/app_metrics_test.cppindex 61d1710e..440ef222 100644--- a/test/app_metrics_test.cpp+++ b/test/app_metrics_test.cpp@@ -24,10 +24,10 @@ TEST_F(AppMetricsTest, StatefulRemoveDocs) {metrics.increment_duration("GET /collections", 4);metrics.increment_duration("GET /operations/vote", 5);- metrics.increment_search_count(1);- metrics.increment_search_count(1);- metrics.increment_search_duration(16);- metrics.increment_search_duration(12);+ metrics.increment_count(AppMetrics::SEARCH_LABEL, 1);+ metrics.increment_count(AppMetrics::SEARCH_LABEL, 1);+ metrics.increment_duration(AppMetrics::SEARCH_LABEL, 16);+ metrics.increment_duration(AppMetrics::SEARCH_LABEL, 12);metrics.window_reset();