Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Option to log requests.
Kishore Nallanb3b591d9ddcd
diff --git a/include/app_metrics.h b/include/app_metrics.hindex caffeae1..cc285d09 100644--- a/include/app_metrics.h+++ b/include/app_metrics.h@@ -3,8 +3,10 @@#include "sparsepp.h"#include "json.hpp"#include "logger.h"+#include "config.h"#include <string>#include <shared_mutex>+#include <fstream>class AppMetrics {private:@@ -18,12 +20,20 @@ private:spp::sparse_hash_map<std::string, uint64_t>* current_counts;spp::sparse_hash_map<std::string, uint64_t>* current_durations;+ std::string access_log_path;+ std::ofstream access_log;+AppMetrics() {current_counts = new spp::sparse_hash_map<std::string, uint64_t>();counts = new spp::sparse_hash_map<std::string, uint64_t>();current_durations = new spp::sparse_hash_map<std::string, uint64_t>();durations = new spp::sparse_hash_map<std::string, uint64_t>();++ access_log_path = Config::get_instance().get_access_log_path();+ if(!access_log_path.empty()) {+ access_log.open(access_log_path, std::ofstream::out | std::ofstream::app);+ }}~AppMetrics() {@@ -62,6 +72,10 @@ public:void increment_write_metrics(uint64_t route_hash, uint64_t duration);+ void write_access_log(const uint64_t epoch_millis, const char* remote_ip, const std::string& path);++ void flush_access_log();+void window_reset();void get(const std::string& rps_key, const std::string& latency_key, nlohmann::json &result) const;diff --git a/include/config.h b/include/config.hindex 3ea0bbf4..89bc0638 100644--- a/include/config.h+++ b/include/config.h@@ -50,6 +50,8 @@ private:uint32_t thread_pool_size;+ bool enable_access_logging;+protected:Config() {@@ -66,6 +68,7 @@ protected:this->num_documents_parallel_load = 1000;this->thread_pool_size = 0; // will be set dynamically if not overriddenthis->ssl_refresh_interval_seconds = 8 * 60 * 60;+ this->enable_access_logging = false;}Config(Config const&) {@@ -239,6 +242,18 @@ public:return this->ssl_refresh_interval_seconds;}+ bool get_enable_access_logging() const {+ return this->enable_access_logging;+ }++ std::string get_access_log_path() const {+ if(this->log_dir.empty()) {+ return "";+ }++ return this->log_dir + "/typesense-access.log";+ }+// loadersstd::string get_env(const char *name) {@@ -333,6 +348,8 @@ public:if(!get_env("TYPESENSE_SSL_REFRESH_INTERVAL_SECONDS").empty()) {this->ssl_refresh_interval_seconds = std::stoi(get_env("TYPESENSE_SSL_REFRESH_INTERVAL_SECONDS"));}++ this->enable_access_logging = ("TRUE" == get_env("TYPESENSE_ENABLE_ACCESS_LOGGING"));}void load_config_file(cmdline::parser & options) {@@ -462,6 +479,11 @@ public:if(reader.Exists("server", "ssl-refresh-interval-seconds")) {this->ssl_refresh_interval_seconds = (int) reader.GetInteger("server", "ssl-refresh-interval-seconds", 8 * 60 * 60);}++ if(reader.Exists("server", "enable-access-logging")) {+ auto enable_access_logging_str = reader.Get("server", "enable-cors", "false");+ this->enable_access_logging = (enable_access_logging_str == "true");+ }}void load_config_cmd_args(cmdline::parser & options) {@@ -570,6 +592,10 @@ public:if(options.exist("ssl-refresh-interval-seconds")) {this->ssl_refresh_interval_seconds = options.get<uint32_t>("ssl-refresh-interval-seconds");}++ if(options.exist("enable-access-logging")) {+ this->enable_cors = options.get<bool>("enable-access-logging");+ }}void set_cors_domains(std::string& cors_domains_value) {diff --git a/include/http_data.h b/include/http_data.hindex 97cc5a33..75e91b5e 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -6,6 +6,7 @@#include <vector>#include <future>#include <chrono>+#include <iomanip>#include "json.hpp"#include "string_utils.h"#include "logger.h"@@ -258,27 +259,25 @@ struct http_req {//LOG(INFO) << "~http_req " << this;if(_req != nullptr) {+ Config& config = Config::get_instance();+uint64_t now = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();uint64_t ms_since_start = (now - start_ts) / 1000;std::string metric_identifier = http_method + " " + path_without_query;AppMetrics::get_instance().increment_duration(metric_identifier, ms_since_start);-AppMetrics::get_instance().increment_write_metrics(route_hash, ms_since_start);- // log slow request if logging is enabled- Config& config = Config::get_instance();- std::string query_string = "?";- for(const auto& kv: params) {- if(kv.first != AUTH_HEADER) {- query_string += kv.first + "=" + kv.second + "&";- }- }-- std::string full_url_path = metric_identifier + query_string;-if(config.get_log_slow_requests_time_ms() > 0 && int(ms_since_start) > config.get_log_slow_requests_time_ms()) {+ // log slow request if logging is enabled+ std::string query_string = "?";+ for(const auto& kv: params) {+ if(kv.first != AUTH_HEADER) {+ query_string += kv.first + "=" + kv.second + "&";+ }+ }+ std::string full_url_path = metric_identifier + query_string;LOG(INFO) << "SLOW REQUEST: " << "(" + std::to_string(ms_since_start) + " ms) " << full_url_path;}}diff --git a/include/string_utils.h b/include/string_utils.hindex b444c599..f5e9f414 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -327,4 +327,6 @@ struct StringUtils {static bool ends_with(std::string const &str, std::string const &ending);static bool contains_word(const std::string& haystack, const std::string& needle);++ static char* get_ip_str(const struct sockaddr* sa, char* s, size_t maxlen);};\ No newline at end of filediff --git a/include/typesense_server_utils.h b/include/typesense_server_utils.hindex 6feb4572..e4c61909 100644--- a/include/typesense_server_utils.h+++ b/include/typesense_server_utils.h@@ -20,7 +20,7 @@ bool directory_exists(const std::string& dir_path);void init_cmdline_options(cmdline::parser& options, int argc, char **argv);-int init_logger(Config& config, const std::string& server_version);+int init_root_logger(Config& config, const std::string& server_version);int run_server(const Config& config, const std::string& version,void (*master_server_routes)());diff --git a/src/app_metrics.cpp b/src/app_metrics.cppindex 0d6dbc3d..b4c6c299 100644--- a/src/app_metrics.cpp+++ b/src/app_metrics.cpp@@ -112,3 +112,14 @@ void AppMetrics::window_reset() {current_durations = new spp::sparse_hash_map<std::string, uint64_t>();}+void AppMetrics::write_access_log(const uint64_t epoch_millis, const char* remote_ip, const std::string& path) {+ if(!access_log_path.empty()) {+ access_log << epoch_millis << "\t" << remote_ip << "\t" << path << "\n";+ }+}++void AppMetrics::flush_access_log() {+ if(!access_log_path.empty()) {+ access_log << std::flush;+ }+}diff --git a/src/http_server.cpp b/src/http_server.cppindex bf97130b..cd97131c 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -59,6 +59,7 @@ void HttpServer::on_metrics_refresh_timeout(h2o_timer_t *entry) {h2o_custom_timer_t* custom_timer = reinterpret_cast<h2o_custom_timer_t*>(entry);AppMetrics::get_instance().window_reset();+ AppMetrics::get_instance().flush_access_log();HttpServer *hs = static_cast<HttpServer*>(custom_timer->data);@@ -303,6 +304,23 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {std::string metric_identifier = http_method + " " + path_without_query;AppMetrics::get_instance().increment_count(metric_identifier, 1);+ if(Config::get_instance().get_enable_access_logging()) {+ const size_t IP_MAX_LEN = 64;+ char ip_str[IP_MAX_LEN];+ sockaddr sa;++ if(0 != req->conn->callbacks->get_peername(req->conn, &sa)) {+ StringUtils::get_ip_str(&sa, ip_str, IP_MAX_LEN);+ } else {+ strncpy(ip_str, "NA", 2);+ }++ uint64_t now = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();+ auto epoch_millis = now / 1000;+ AppMetrics::get_instance().write_access_log(epoch_millis, ip_str, metric_identifier);+ }+// Handle CORSif(h2o_handler->http_server->cors_enabled) {h2o_iovec_t response_origin = {(char*)"*", 1};diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 16e96100..ca5e3a3b 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -126,7 +126,7 @@ int main(int argc, char **argv) {exit(1);}- int ret_code = init_logger(config, TYPESENSE_VERSION);+ int ret_code = init_root_logger(config, TYPESENSE_VERSION);if(ret_code != 0) {return ret_code;}diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 221bc2eb..33964dad 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -5,6 +5,8 @@#include <random>#include <openssl/sha.h>#include <map>+#include <netinet/in.h>+#include <arpa/inet.h>#include "logger.h"@@ -313,6 +315,24 @@ bool StringUtils::contains_word(const std::string& haystack, const std::string&return true;}+char* StringUtils::get_ip_str(const struct sockaddr* sa, char* s, size_t maxlen) {+ switch (sa->sa_family) {+ case AF_INET:+ inet_ntop(AF_INET, &(((struct sockaddr_in*) sa)->sin_addr), s, maxlen);+ break;++ case AF_INET6:+ inet_ntop(AF_INET6, &(((struct sockaddr_in6*) sa)->sin6_addr), s, maxlen);+ break;++ default:+ strncpy(s, "Unknown AF", maxlen);+ return NULL;+ }++ return s;+}+/*size_t StringUtils::unicode_length(const std::string& bytes) {std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;return utf8conv.from_bytes(bytes).size();diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 2bfcde92..b214354b 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -97,6 +97,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<std::string>("config", '\0', "Path to the configuration file.", false, "");+ options.add<bool>("enable-access-logging", '\0', "Enable access logging.", false, false);+// DEPRECATEDoptions.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");options.add<uint32_t>("listen-port", 'p', "[DEPRECATED: use `api-port`] Port on which Typesense API service listens.", false, 8108);@@ -104,7 +106,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {"to start as read-only replica.", false, "");}-int init_logger(Config & config, const std::string & server_version) {+int init_root_logger(Config & config, const std::string & server_version) {google::InitGoogleLogging("typesense");std::string log_dir = config.get_log_dir();