Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Configuration to log slow requests
Jason Bosco8b4e07cb81aa
diff --git a/include/config.h b/include/config.hindex 61cd508d..515b8a0c 100644--- a/include/config.h+++ b/include/config.h@@ -37,7 +37,9 @@ private:std::string config_file;int config_file_validity;-public:+ int log_slow_requests_time_ms;++protected:Config() {this->api_address = "0.0.0.0";@@ -47,8 +49,22 @@ public:this->max_memory_ratio = 1.0f;this->snapshot_interval_seconds = 3600;this->catch_up_threshold_percentage = 95;+ this->log_slow_requests_time_ms = -1;+ }++ Config(Config const&) {+}+public:++ static Config & get_instance() {+ static Config instance;+ return instance;+ }++ void operator=(Config const&) = delete;+// settersvoid set_data_dir(const std::string & data_dir) {@@ -92,6 +108,10 @@ public:this->enable_cors = enable_cors;}+ void set_log_slow_requests_time_ms(int log_slow_requests_time_ms) {+ this->log_slow_requests_time_ms = log_slow_requests_time_ms;+ }+// gettersstd::string get_data_dir() const {@@ -163,6 +183,10 @@ public:return this->catch_up_threshold_percentage;}+ int get_log_slow_requests_time_ms() const {+ return this->log_slow_requests_time_ms;+ }+// loadersstd::string get_env(const char *name) {@@ -226,6 +250,10 @@ public:if(!get_env("TYPESENSE_CATCH_UP_THRESHOLD_PERCENTAGE").empty()) {this->catch_up_threshold_percentage = std::stoi(get_env("TYPESENSE_CATCH_UP_THRESHOLD_PERCENTAGE"));}++ if(!get_env("TYPESENSE_LOG_SLOW_REQUESTS_TIME_MS").empty()) {+ this->log_slow_requests_time_ms = std::stoi(get_env("TYPESENSE_LOG_SLOW_REQUESTS_TIME_MS"));+ }}void load_config_file(cmdline::parser & options) {@@ -319,6 +347,10 @@ public:if(reader.Exists("server", "catch-up-threshold-percentage")) {this->catch_up_threshold_percentage = (int) reader.GetInteger("server", "catch-up-threshold-percentage", 95);}++ if(reader.Exists("server", "log-slow-requests-time-ms")) {+ this->log_slow_requests_time_ms = (int) reader.GetInteger("server", "log-slow-requests-time-ms", -1);+ }}void load_config_cmd_args(cmdline::parser & options) {@@ -394,6 +426,10 @@ public:if(options.exist("catch-up-threshold-percentage")) {this->catch_up_threshold_percentage = options.get<int>("catch-up-threshold-percentage");}++ if(options.exist("log-slow-requests-time-ms")) {+ this->log_slow_requests_time_ms = options.exist("log-slow-requests-time-ms");+ }}// validationdiff --git a/include/core_api.h b/include/core_api.hindex 0abdf0bf..f6429302 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -93,6 +93,8 @@ bool post_snapshot(http_req& req, http_res& res);bool post_vote(http_req& req, http_res& res);+bool post_config(http_req& req, http_res& res);+// Misc helpersbool raft_write_send_response(void *data);diff --git a/include/http_data.h b/include/http_data.hindex 12ac1352..6d3d1d28 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -10,6 +10,7 @@#include "string_utils.h"#include "logger.h"#include "app_metrics.h"+#include "config.h"#define H2O_USE_LIBUV 0extern "C" {@@ -163,6 +164,8 @@ struct http_res {};struct http_req {+ static constexpr const char* AUTH_HEADER = "x-typesense-api-key";+h2o_req_t* _req;std::string http_method;std::string path_without_query;@@ -218,6 +221,21 @@ struct http_req {std::string metric_identifier = http_method + " " + path_without_query;AppMetrics::get_instance().increment_duration(metric_identifier, ms_since_start);++ // log slow request if logging is enabled+ Config& config = Config::get_instance();+ std::string query_string = "?";+ for(const auto& kv: params) {+ if(kv.first != AUTH_HEADER) {+ query_string += kv.first + "=" + kv.second + "&";+ }+ }++ std::string full_url_path = metric_identifier + query_string;++ if(config.get_log_slow_requests_time_ms() > 0 && int(ms_since_start) > config.get_log_slow_requests_time_ms()) {+ LOG(INFO) << "SLOW REQUEST: " << "(" + std::to_string(ms_since_start) + " ms) " << full_url_path;+ }}}diff --git a/include/http_server.h b/include/http_server.hindex 3c000bf6..914d58a2 100644--- a/include/http_server.h+++ b/include/http_server.h@@ -164,7 +164,6 @@ public:std::string get_version();- static constexpr const char* AUTH_HEADER = "x-typesense-api-key";static constexpr const char* STOP_SERVER_MESSAGE = "STOP_SERVER";static constexpr const char* STREAM_RESPONSE_MESSAGE = "STREAM_RESPONSE";static constexpr const char* REQUEST_PROCEED_MESSAGE = "REQUEST_PROCEED";diff --git a/src/collection.cpp b/src/collection.cppindex 41329ba7..7eb2c071 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -745,7 +745,8 @@ Option<nlohmann::json> Collection::search(const std::string & query, const std::}if((page * per_page) > limit_hits) {- std::string message = "Only upto " + std::to_string(limit_hits) + " hits can be fetched.";+ std::string message = "Only upto " + std::to_string(limit_hits) + " hits can be fetched. " ++ "Ensure that `page` and `per_page` parameters are within this range.";return Option<nlohmann::json>(422, message);}diff --git a/src/core_api.cpp b/src/core_api.cppindex 04209ef9..9b1f5fec 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1370,6 +1370,35 @@ bool post_vote(http_req& req, http_res& res) {return true;}+bool post_config(http_req &req, http_res &res) {+ nlohmann::json req_json;++ try {+ req_json = nlohmann::json::parse(req.body);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ res.set_400("Bad JSON.");+ return false;+ }++ if(req_json.count("log-slow-requests-time-ms") != 0) {+ if(!req_json["log-slow-requests-time-ms"].is_number_integer()) {+ res.set_400("Configuration `log-slow-requests-time-ms` must be an integer.");+ return false;+ }++ Config::get_instance().set_log_slow_requests_time_ms(req_json["log-slow-requests-time-ms"].get<int>());++ nlohmann::json response;+ response["success"] = true;+ res.set_201(response.dump());+ } else {+ res.set_400("Invalid configuration.");+ }++ return true;+}+bool get_synonyms(http_req &req, http_res &res) {CollectionManager & collectionManager = CollectionManager::get_instance();Collection *collection = collectionManager.get_collection(req.params["collection"]);diff --git a/src/http_server.cpp b/src/http_server.cppindex ced40a0d..226bab15 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -371,12 +371,12 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {// Extract auth key from header. If that does not exist, look for a GET parameter.std::string api_auth_key_sent = "";- ssize_t auth_header_cursor = h2o_find_header_by_str(&req->headers, AUTH_HEADER, strlen(AUTH_HEADER), -1);+ ssize_t auth_header_cursor = h2o_find_header_by_str(&req->headers, http_req::AUTH_HEADER, strlen(http_req::AUTH_HEADER), -1);if(auth_header_cursor != -1) {h2o_iovec_t & slot = req->headers.entries[auth_header_cursor].value;api_auth_key_sent = std::string(slot.base, slot.len);- } else if(query_map.count(AUTH_HEADER) != 0) {- api_auth_key_sent = query_map[AUTH_HEADER];+ } else if(query_map.count(http_req::AUTH_HEADER) != 0) {+ api_auth_key_sent = query_map[http_req::AUTH_HEADER];}route_path *rpath = nullptr;@@ -397,7 +397,7 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {bool authenticated = h2o_handler->http_server->auth_handler(query_map, *rpath, api_auth_key_sent);if(!authenticated) {- std::string message = std::string("{\"message\": \"Forbidden - a valid `") + AUTH_HEADER ++ std::string message = std::string("{\"message\": \"Forbidden - a valid `") + http_req::AUTH_HEADER +"` header must be sent.\"}";return send_response(req, 401, message);}diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex a81ad7c9..b7203ed7 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -62,8 +62,11 @@ void master_server_routes() {server->get("/health", get_health);server->post("/health", post_health);server->get("/sequence", get_log_sequence);+server->post("/operations/snapshot", post_snapshot, false, true);server->post("/operations/vote", post_vote, false, false);++ server->post("/config", post_config, false, false);}int main(int argc, char **argv) {@@ -76,7 +79,7 @@ int main(int argc, char **argv) {je_zone_register();#endif- Config config;+ Config& config = Config::get_instance();cmdline::parser options;init_cmdline_options(options, argc, argv);diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 4e42d0aa..0c8e5569 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -79,6 +79,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<float>("max-memory-ratio", '\0', "Maximum fraction of system memory to be used.", false, 1.0f);options.add<int>("snapshot-interval-seconds", '\0', "Frequency of replication log snapshots.", false, 3600);options.add<int>("catch-up-threshold-percentage", '\0', "The threshold at which a follower is deemed to have caught up with leader.", false, 95);+ options.add<int>("log-slow-requests-time-ms", '\0', "When > 0, requests that take longer than this duration are logged.", false, -1);options.add<std::string>("log-dir", '\0', "Path to the log directory.", false, "");diff --git a/test/collection_test.cpp b/test/collection_test.cppindex c20f1d44..c21c07f3 100644--- a/test/collection_test.cpp+++ b/test/collection_test.cpp@@ -2575,7 +2575,9 @@ TEST_F(CollectionTest, WildcardQueryReturnsResultsBasedOnPerPageParam) {"<mark>", "</mark>", {1}, 20);ASSERT_FALSE(res_op.ok());- ASSERT_STREQ("Only upto 20 hits can be fetched.", res_op.error().c_str());+ ASSERT_STREQ(+ "Only upto 20 hits can be fetched. Ensure that `page` and `per_page` parameters are within this range.",+ res_op.error().c_str());}TEST_F(CollectionTest, RemoveIfFound) {diff --git a/test/config_test.cpp b/test/config_test.cppindex 707eaece..d266f149 100644--- a/test/config_test.cpp+++ b/test/config_test.cpp@@ -15,6 +15,13 @@ std::vector<char*> get_argv(std::vector<std::string> & args) {return argv;}+class ConfigImpl : public Config {+public:+ ConfigImpl(): Config() {++ }+};+TEST(ConfigTest, LoadCmdLineArguments) {cmdline::parser options;@@ -30,7 +37,7 @@ TEST(ConfigTest, LoadCmdLineArguments) {init_cmdline_options(options, argv.size() - 1, argv.data());options.parse(argv.size() - 1, argv.data());- Config config;+ ConfigImpl config;config.load_config_cmd_args(options);ASSERT_EQ("abcd", config.get_api_key());@@ -42,7 +49,7 @@ TEST(ConfigTest, LoadEnvVars) {cmdline::parser options;putenv((char*)"TYPESENSE_DATA_DIR=/tmp/ts");putenv((char*)"TYPESENSE_LISTEN_PORT=9090");- Config config;+ ConfigImpl config;config.load_config_env();ASSERT_EQ("/tmp/ts", config.get_data_dir());@@ -50,14 +57,14 @@ TEST(ConfigTest, LoadEnvVars) {}TEST(ConfigTest, BadConfigurationReturnsError) {- Config config1;+ ConfigImpl config1;config1.set_api_key("abcd");auto validation = config1.is_valid();ASSERT_EQ(false, validation.ok());ASSERT_EQ("Data directory is not specified.", validation.error());- Config config2;+ ConfigImpl config2;config2.set_data_dir("/tmp/ts");validation = config2.is_valid();@@ -76,7 +83,7 @@ TEST(ConfigTest, LoadConfigFile) {init_cmdline_options(options, argv.size() - 1, argv.data());options.parse(argv.size() - 1, argv.data());- Config config;+ ConfigImpl config;config.load_config_file(options);auto validation = config.is_valid();@@ -100,7 +107,7 @@ TEST(ConfigTest, LoadIncompleteConfigFile) {init_cmdline_options(options, argv.size() - 1, argv.data());options.parse(argv.size() - 1, argv.data());- Config config;+ ConfigImpl config;auto validation = config.is_valid();@@ -119,7 +126,7 @@ TEST(ConfigTest, LoadBadConfigFile) {init_cmdline_options(options, argv.size() - 1, argv.data());options.parse(argv.size() - 1, argv.data());- Config config;+ ConfigImpl config;config.load_config_file(options);auto validation = config.is_valid();@@ -148,7 +155,7 @@ TEST(ConfigTest, CmdLineArgsOverrideConfigFileAndEnvVars) {init_cmdline_options(options, argv.size() - 1, argv.data());options.parse(argv.size() - 1, argv.data());- Config config;+ ConfigImpl config;config.load_config_env();config.load_config_file(options);config.load_config_cmd_args(options);@@ -160,4 +167,4 @@ TEST(ConfigTest, CmdLineArgsOverrideConfigFileAndEnvVars) {ASSERT_EQ(true, config.get_enable_cors());ASSERT_EQ("192.168.10.10", config.get_api_address());ASSERT_EQ("abcd", config.get_api_key()); // cli parameter overrides file config-}\ No newline at end of file+}