Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Expose write stats.

Kishore Nallan6da39ff51ca6
diff --git a/include/app_metrics.h b/include/app_metrics.h
index cf5bbbf2..caffeae1 100644
--- a/include/app_metrics.h
+++ b/include/app_metrics.h
@@ -10,8 +10,6 @@ class AppMetrics {
private:
mutable std::shared_mutex mutex;
- static inline const std::string SEARCH_LABEL = "search";
-
// stores last complete window
spp::sparse_hash_map<std::string, uint64_t>* counts;
spp::sparse_hash_map<std::string, uint64_t>* durations;
@@ -37,6 +35,10 @@ private:
}
public:
+ static inline const std::string SEARCH_LABEL = "search";
+ static inline const std::string DOC_WRITE_LABEL = "write";
+ static inline const std::string IMPORT_LABEL = "import";
+ static inline const std::string DOC_DELETE_LABEL = "delete";
static const uint64_t METRICS_REFRESH_INTERVAL_MS = 10 * 1000;
@@ -53,71 +55,14 @@ public:
(*current_counts)[identifier] += count;
}
- void increment_search_count(uint64_t count) {
- std::unique_lock lock(mutex);
- (*current_counts)[SEARCH_LABEL] += count;
- }
-
void increment_duration(const std::string& identifier, uint64_t duration) {
std::unique_lock lock(mutex);
(*current_durations)[identifier] += duration;
}
- void increment_search_duration(uint64_t duration) {
- std::unique_lock lock(mutex);
- (*current_durations)[SEARCH_LABEL] += duration;
- }
+ void increment_write_metrics(uint64_t route_hash, uint64_t duration);
- void window_reset() {
- std::unique_lock lock(mutex);
+ void window_reset();
- delete counts;
- counts = current_counts;
- current_counts = new spp::sparse_hash_map<std::string, uint64_t>();
-
- delete durations;
- durations = current_durations;
- current_durations = new spp::sparse_hash_map<std::string, uint64_t>();
- }
-
- void get(const std::string& rps_key, const std::string& latency_key, nlohmann::json &result) const {
- std::shared_lock lock(mutex);
-
- uint64_t total_counts = 0;
- auto SEARCH_RPS_KEY = SEARCH_LABEL + "_" + rps_key;
- auto SEARCH_LATENCY_KEY = SEARCH_LABEL + "_" + latency_key;
-
- result[rps_key] = nlohmann::json::object();
- for(const auto& kv: *counts) {
- if(kv.first == SEARCH_LABEL) {
- result[SEARCH_RPS_KEY] = double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000);
- } else {
- result[rps_key][kv.first] = (double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000));
- total_counts += kv.second;
- }
- }
-
- result["total_" + rps_key] = double(total_counts) / (METRICS_REFRESH_INTERVAL_MS / 1000);
-
- result[latency_key] = nlohmann::json::object();
-
- for(const auto& kv: *durations) {
- auto counter_it = counts->find(kv.first);
- if(counter_it != counts->end() && counter_it->second != 0) {
- if(kv.first == SEARCH_LABEL) {
- result[SEARCH_LATENCY_KEY] = (double(kv.second) / counter_it->second);
- } else {
- result[latency_key][kv.first] = (double(kv.second) / counter_it->second);
- }
- }
- }
-
- if(!result.contains(SEARCH_RPS_KEY)) {
- result[SEARCH_RPS_KEY] = 0;
- }
-
- if(!result.contains(SEARCH_LATENCY_KEY)) {
- result[SEARCH_LATENCY_KEY] = 0;
- }
- }
+ void get(const std::string& rps_key, const std::string& latency_key, nlohmann::json &result) const;
};
\ No newline at end of file
diff --git a/include/core_api.h b/include/core_api.h
index 470f7cdf..006d21b6 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -102,3 +102,8 @@ bool post_config(const std::shared_ptr<http_req>& req, const std::shared_ptr<htt
void get_collections_for_auth(std::map<std::string, std::string> &req_params, const std::string &body,
const route_path &rpath, std::vector<std::string> &collections);
+bool is_doc_import_route(uint64_t route_hash);
+
+bool is_doc_write_route(uint64_t route_hash);
+
+bool is_doc_del_route(uint64_t route_hash);
diff --git a/include/http_data.h b/include/http_data.h
index 5dbde5ff..1072fef2 100644
--- a/include/http_data.h
+++ b/include/http_data.h
@@ -33,33 +33,6 @@ enum class ROUTE_CODES {
ALREADY_HANDLED = 2,
};
-class await_t {
-private:
-
- std::mutex mcv;
- std::condition_variable cv;
- bool ready;
-
-public:
-
- await_t(): ready(false) {}
-
- void notify() {
- // Ideally we don't need lock over notify but it is needed here because
- // the parent object could be deleted after lock on mutex is released but
- // before notify can be called on condition variable.
- std::lock_guard<std::mutex> lk(mcv);
- ready = true;
- cv.notify_all();
- }
-
- void wait() {
- auto lk = std::unique_lock<std::mutex>(mcv);
- cv.wait(lk, [&] { return ready; });
- ready = false;
- }
-};
-
struct http_res {
uint32_t status_code;
std::string content_type_header;
@@ -264,16 +237,16 @@ struct http_req {
~http_req() {
//LOG(INFO) << "~http_req " << this;
-
if(!deserialized_request) {
uint64_t now = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
-
uint64_t ms_since_start = (now - start_ts) / 1000;
- std::string metric_identifier = http_method + " " + path_without_query;
+ std::string metric_identifier = http_method + " " + path_without_query;
AppMetrics::get_instance().increment_duration(metric_identifier, ms_since_start);
+ AppMetrics::get_instance().increment_write_metrics(route_hash, ms_since_start);
+
// log slow request if logging is enabled
Config& config = Config::get_instance();
std::string query_string = "?";
@@ -309,7 +282,7 @@ struct http_req {
// NOTE: we don't ser/de all fields, only ones needed for write forwarding
// Take care to check for existence of key to ensure backward compatibility during upgrade
- void deserialize(const std::string& serialized_content) {
+ void load_from_json(const std::string& serialized_content, const bool is_deserialized) {
nlohmann::json content = nlohmann::json::parse(serialized_content);
route_hash = content["route_hash"];
body += content["body"];
@@ -325,10 +298,10 @@ struct http_req {
log_index = content.count("log_index") != 0 ? content["log_index"].get<int64_t>() : 0;
_req = nullptr;
- deserialized_request = true;
+ deserialized_request = is_deserialized;
}
- std::string serialize() const {
+ std::string to_json() const {
nlohmann::json content;
content["route_hash"] = route_hash;
content["params"] = params;
diff --git a/include/http_server.h b/include/http_server.h
index 5f73ee9c..51427a43 100644
--- a/include/http_server.h
+++ b/include/http_server.h
@@ -91,8 +91,11 @@ private:
std::string version;
- // must be a vector since order of routes matter
- std::vector<std::pair<uint64_t, route_path>> routes;
+ // must be a vector since order of routes entered matter
+ std::vector<std::pair<uint64_t, route_path>> route_hash_to_path;
+
+ // also have a hashmap for quick lookup of individual routes
+ std::unordered_map<uint64_t, route_path> route_hash_to_path_map;
const std::string listen_address;
@@ -156,6 +159,8 @@ public:
bool is_alive() const;
+ bool is_leader() const;
+
uint64_t node_state() const;
void set_auth_handler(bool (*handler)(std::map<std::string, std::string>& params, const std::string& body,
diff --git a/include/raft_server.h b/include/raft_server.h
index f09b96f8..a82196d6 100644
--- a/include/raft_server.h
+++ b/include/raft_server.h
@@ -212,6 +212,8 @@ public:
int64_t get_num_queued_writes();
+ bool is_leader();
+
private:
friend class ReplicationClosure;
diff --git a/include/string_utils.h b/include/string_utils.h
index 7538c8b8..3d386a68 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -317,4 +317,6 @@ struct StringUtils {
const std::string& replace);
static std::string trim_curly_spaces(const std::string& str);
+
+ static bool ends_with(std::string const &str, std::string const &ending);
};
\ No newline at end of file
diff --git a/src/app_metrics.cpp b/src/app_metrics.cpp
new file mode 100644
index 00000000..0d6dbc3d
--- /dev/null
+++ b/src/app_metrics.cpp
@@ -0,0 +1,114 @@
+#include "app_metrics.h"
+#include "core_api.h"
+
+void AppMetrics::increment_write_metrics(uint64_t route_hash, uint64_t duration) {
+ if(is_doc_import_route(route_hash)) {
+ AppMetrics::get_instance().increment_duration(AppMetrics::IMPORT_LABEL, duration);
+ AppMetrics::get_instance().increment_count(AppMetrics::IMPORT_LABEL, 1);
+ }
+
+ else if(is_doc_write_route(route_hash)) {
+ AppMetrics::get_instance().increment_duration(AppMetrics::DOC_WRITE_LABEL, duration);
+ AppMetrics::get_instance().increment_count(AppMetrics::DOC_WRITE_LABEL, 1);
+ }
+
+ else if(is_doc_del_route(route_hash)) {
+ AppMetrics::get_instance().increment_duration(AppMetrics::DOC_DELETE_LABEL, duration);
+ AppMetrics::get_instance().increment_count(AppMetrics::DOC_DELETE_LABEL, 1);
+ }
+}
+
+void AppMetrics::get(const std::string& rps_key, const std::string& latency_key, nlohmann::json& result) const {
+ std::shared_lock lock(mutex);
+
+ uint64_t total_counts = 0;
+
+ auto SEARCH_RPS_KEY = SEARCH_LABEL + "_" + rps_key;
+ auto SEARCH_LATENCY_KEY = SEARCH_LABEL + "_" + latency_key;
+
+ auto IMPORT_RPS_KEY = IMPORT_LABEL + "_" + rps_key;
+ auto IMPORT_LATENCY_KEY = IMPORT_LABEL + "_" + latency_key;
+
+ auto DOC_WRITE_RPS_KEY = DOC_WRITE_LABEL + "_" + rps_key;
+ auto DOC_WRITE_LATENCY_KEY = DOC_WRITE_LABEL + "_" + latency_key;
+
+ auto DOC_DELETE_RPS_KEY = DOC_DELETE_LABEL + "_" + rps_key;
+ auto DOC_DELETE_LATENCY_KEY = DOC_DELETE_LABEL + "_" + latency_key;
+
+ result[rps_key] = nlohmann::json::object();
+ for(const auto& kv: *counts) {
+ if(kv.first == SEARCH_LABEL) {
+ result[SEARCH_RPS_KEY] = double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000);
+ }
+
+ else if(kv.first == IMPORT_LABEL) {
+ result[IMPORT_RPS_KEY] = double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000);
+ }
+
+ else if(kv.first == DOC_WRITE_LABEL) {
+ result[DOC_WRITE_RPS_KEY] = double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000);
+ }
+
+ else if(kv.first == DOC_DELETE_LABEL) {
+ result[DOC_DELETE_RPS_KEY] = double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000);
+ }
+
+ else {
+ result[rps_key][kv.first] = (double(kv.second) / (METRICS_REFRESH_INTERVAL_MS / 1000));
+ total_counts += kv.second;
+ }
+ }
+
+ result["total_" + rps_key] = double(total_counts) / (METRICS_REFRESH_INTERVAL_MS / 1000);
+
+ result[latency_key] = nlohmann::json::object();
+
+ for(const auto& kv: *durations) {
+ auto counter_it = counts->find(kv.first);
+ if(counter_it != counts->end() && counter_it->second != 0) {
+ if(kv.first == SEARCH_LABEL) {
+ result[SEARCH_LATENCY_KEY] = (double(kv.second) / counter_it->second);
+ }
+
+ else if(kv.first == IMPORT_LABEL) {
+ result[IMPORT_LATENCY_KEY] = (double(kv.second) / counter_it->second);
+ }
+
+ else if(kv.first == DOC_WRITE_LABEL) {
+ result[DOC_WRITE_LATENCY_KEY] = (double(kv.second) / counter_it->second);
+ }
+
+ else if(kv.first == DOC_DELETE_LABEL) {
+ result[DOC_DELETE_LATENCY_KEY] = (double(kv.second) / counter_it->second);
+ }
+
+ else {
+ result[latency_key][kv.first] = (double(kv.second) / counter_it->second);
+ }
+ }
+ }
+
+ std::vector<std::string> keys_to_check = {
+ SEARCH_RPS_KEY, IMPORT_RPS_KEY, DOC_WRITE_RPS_KEY, DOC_DELETE_RPS_KEY,
+ SEARCH_LATENCY_KEY, IMPORT_LATENCY_KEY, DOC_WRITE_LATENCY_KEY, DOC_DELETE_LATENCY_KEY
+ };
+
+ for(auto& key: keys_to_check) {
+ if(!result.contains(key)) {
+ result[key] = 0;
+ }
+ }
+}
+
+void AppMetrics::window_reset() {
+ std::unique_lock lock(mutex);
+
+ delete counts;
+ counts = current_counts;
+ current_counts = new spp::sparse_hash_map<std::string, uint64_t>();
+
+ delete durations;
+ durations = current_durations;
+ current_durations = new spp::sparse_hash_map<std::string, uint64_t>();
+}
+
diff --git a/src/batched_indexer.cpp b/src/batched_indexer.cpp
index 9593e765..c749b418 100644
--- a/src/batched_indexer.cpp
+++ b/src/batched_indexer.cpp
@@ -43,7 +43,7 @@ void BatchedIndexer::enqueue(const std::shared_ptr<http_req>& req, const std::sh
//LOG(INFO) << "req_id: " << req->start_ts << ", chunk_sequence: " << chunk_sequence;
- store->insert(request_chunk_key, req->serialize());
+ store->insert(request_chunk_key, req->to_json());
req->body = "";
{
@@ -110,7 +110,7 @@ void BatchedIndexer::run() {
std::shared_ptr<http_req>& orig_req = orig_req_res.req;
auto _req = orig_req->_req;
orig_req->body = prev_body;
- orig_req->deserialize(iter->value().ToString());
+ orig_req->load_from_json(iter->value().ToString(), _req == nullptr);
orig_req->_req = _req;
// update thread local for reference during a crash
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 0034c71b..18edba3b 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -829,8 +829,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
- AppMetrics::get_instance().increment_search_count(1);
- AppMetrics::get_instance().increment_search_duration(timeMillis);
+ AppMetrics::get_instance().increment_count(AppMetrics::SEARCH_LABEL, 1);
+ AppMetrics::get_instance().increment_duration(AppMetrics::SEARCH_LABEL, timeMillis);
if(!result_op.ok()) {
return Option<bool>(result_op.code(), result_op.error());
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 6d89dc51..730e9649 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1,4 +1,3 @@
-#include <regex>
#include <chrono>
#include <thread>
#include <app_metrics.h>
@@ -1407,3 +1406,21 @@ bool del_synonym(const std::shared_ptr<http_req>& req, const std::shared_ptr<htt
res->set_200(res_json.dump());
return true;
}
+
+bool is_doc_import_route(uint64_t route_hash) {
+ route_path* rpath;
+ bool found = server->get_route(route_hash, &rpath);
+ return found && (rpath->handler == post_import_documents);
+}
+
+bool is_doc_write_route(uint64_t route_hash) {
+ route_path* rpath;
+ bool found = server->get_route(route_hash, &rpath);
+ return found && (rpath->handler == post_add_document || rpath->handler == patch_update_document);
+}
+
+bool is_doc_del_route(uint64_t route_hash) {
+ route_path* rpath;
+ bool found = server->get_route(route_hash, &rpath);
+ return found && (rpath->handler == del_remove_document || rpath->handler == del_remove_documents);
+}
diff --git a/src/http_server.cpp b/src/http_server.cpp
index f97d1f94..c5434b7e 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -230,7 +230,7 @@ h2o_pathconf_t* HttpServer::register_handler(h2o_hostconf_t *hostconf, const cha
uint64_t HttpServer::find_route(const std::vector<std::string> & path_parts, const std::string & http_method,
route_path** found_rpath) {
- for (const auto& index_route : routes) {
+ for (const auto& index_route : route_hash_to_path) {
const route_path & rpath = index_route.second;
if(rpath.path_parts.size() != path_parts.size() || rpath.http_method != http_method) {
@@ -817,35 +817,40 @@ void HttpServer::get(const std::string & path, bool (*handler)(const std::shared
std::vector<std::string> path_parts;
StringUtils::split(path, path_parts, "/");
route_path rpath("GET", path_parts, handler, async_req, async_res);
- routes.emplace_back(rpath.route_hash(), rpath);
+ route_hash_to_path.emplace_back(rpath.route_hash(), rpath);
+ route_hash_to_path_map.emplace(rpath.route_hash(), rpath);
}
void HttpServer::post(const std::string & path, bool (*handler)(const std::shared_ptr<http_req>&, const std::shared_ptr<http_res>&), bool async_req, bool async_res) {
std::vector<std::string> path_parts;
StringUtils::split(path, path_parts, "/");
route_path rpath("POST", path_parts, handler, async_req, async_res);
- routes.emplace_back(rpath.route_hash(), rpath);
+ route_hash_to_path.emplace_back(rpath.route_hash(), rpath);
+ route_hash_to_path_map.emplace(rpath.route_hash(), rpath);
}
void HttpServer::put(const std::string & path, bool (*handler)(const std::shared_ptr<http_req>&, const std::shared_ptr<http_res>&), bool async_req, bool async_res) {
std::vector<std::string> path_parts;
StringUtils::split(path, path_parts, "/");
route_path rpath("PUT", path_parts, handler, async_req, async_res);
- routes.emplace_back(rpath.route_hash(), rpath);
+ route_hash_to_path.emplace_back(rpath.route_hash(), rpath);
+ route_hash_to_path_map.emplace(rpath.route_hash(), rpath);
}
void HttpServer::patch(const std::string & path, bool (*handler)(const std::shared_ptr<http_req>&, const std::shared_ptr<http_res>&), bool async_req, bool async_res) {
std::vector<std::string> path_parts;
StringUtils::split(path, path_parts, "/");
route_path rpath("PATCH", path_parts, handler, async_req, async_res);
- routes.emplace_back(rpath.route_hash(), rpath);
+ route_hash_to_path.emplace_back(rpath.route_hash(), rpath);
+ route_hash_to_path_map.emplace(rpath.route_hash(), rpath);
}
void HttpServer::del(const std::string & path, bool (*handler)(const std::shared_ptr<http_req>&, const std::shared_ptr<http_res>&), bool async_req, bool async_res) {
std::vector<std::string> path_parts;
StringUtils::split(path, path_parts, "/");
route_path rpath("DELETE", path_parts, handler, async_req, async_res);
- routes.emplace_back(rpath.route_hash(), rpath);
+ route_hash_to_path.emplace_back(rpath.route_hash(), rpath);
+ route_hash_to_path_map.emplace(rpath.route_hash(), rpath);
}
void HttpServer::on(const std::string & message, bool (*handler)(void*)) {
@@ -896,14 +901,14 @@ bool HttpServer::is_alive() const {
}
bool HttpServer::get_route(uint64_t hash, route_path** found_rpath) {
- for (auto& hash_route : routes) {
- if(hash_route.first == hash) {
- *found_rpath = &hash_route.second;
- return true;
- }
+ auto route_hash_it = route_hash_to_path_map.find(hash);
+
+ if(route_hash_it == route_hash_to_path_map.end()) {
+ return false;
}
- return false;
+ *found_rpath = &route_hash_it->second;
+ return true;
}
uint64_t HttpServer::node_state() const {
@@ -1028,3 +1033,7 @@ void HttpServer::persist_applying_index() {
int64_t HttpServer::get_num_queued_writes() {
return replication_state->get_num_queued_writes();
}
+
+bool HttpServer::is_leader() const {
+ return replication_state->is_leader();
+}
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index d6391be8..4958796d 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -212,7 +212,7 @@ void ReplicationState::write(const std::shared_ptr<http_req>& request, const std
// NOTE: actual write must be done only on the `on_apply` method to maintain consistency.
butil::IOBufBuilder bufBuilder;
- bufBuilder << request->serialize();
+ bufBuilder << request->to_json();
//LOG(INFO) << "write() pre request ref count " << request.use_count();
@@ -364,7 +364,7 @@ void ReplicationState::on_apply(braft::Iterator& iter) {
if(!iter.done()) {
// indicates log serialized request
- request_generated->deserialize(iter.data().to_string());
+ request_generated->load_from_json(iter.data().to_string(), true);
}
request_generated->log_index = iter.index();
@@ -779,6 +779,16 @@ int64_t ReplicationState::get_num_queued_writes() {
return batched_indexer->get_queued_writes();
}
+bool ReplicationState::is_leader() {
+ std::shared_lock lock(node_mutex);
+
+ if(!node) {
+ return false;
+ }
+
+ return node->is_leader();
+}
+
void OnDemandSnapshotClosure::Run() {
// Auto delete this after Done()
std::unique_ptr<OnDemandSnapshotClosure> self_guard(this);
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index 6a525304..0811b3fc 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -291,6 +291,14 @@ std::string StringUtils::trim_curly_spaces(const std::string& str) {
return right_trimmed;
}
+bool StringUtils::ends_with(const std::string& str, const std::string& ending) {
+ if (str.length() >= ending.length()) {
+ return (0 == str.compare (str.length() - ending.length(), ending.length(), ending));
+ } else {
+ return false;
+ }
+}
+
/*size_t StringUtils::unicode_length(const std::string& bytes) {
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;
return utf8conv.from_bytes(bytes).size();
diff --git a/test/app_metrics_test.cpp b/test/app_metrics_test.cpp
index 61d1710e..440ef222 100644
--- a/test/app_metrics_test.cpp
+++ b/test/app_metrics_test.cpp
@@ -24,10 +24,10 @@ TEST_F(AppMetricsTest, StatefulRemoveDocs) {
metrics.increment_duration("GET /collections", 4);
metrics.increment_duration("GET /operations/vote", 5);
- metrics.increment_search_count(1);
- metrics.increment_search_count(1);
- metrics.increment_search_duration(16);
- metrics.increment_search_duration(12);
+ metrics.increment_count(AppMetrics::SEARCH_LABEL, 1);
+ metrics.increment_count(AppMetrics::SEARCH_LABEL, 1);
+ metrics.increment_duration(AppMetrics::SEARCH_LABEL, 16);
+ metrics.increment_duration(AppMetrics::SEARCH_LABEL, 12);
metrics.window_reset();