Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Option to log requests.

Kishore Nallanb3b591d9ddcd
diff --git a/include/app_metrics.h b/include/app_metrics.h
index caffeae1..cc285d09 100644
--- a/include/app_metrics.h
+++ b/include/app_metrics.h
@@ -3,8 +3,10 @@
#include "sparsepp.h"
#include "json.hpp"
#include "logger.h"
+#include "config.h"
#include <string>
#include <shared_mutex>
+#include <fstream>
class AppMetrics {
private:
@@ -18,12 +20,20 @@ private:
spp::sparse_hash_map<std::string, uint64_t>* current_counts;
spp::sparse_hash_map<std::string, uint64_t>* current_durations;
+ std::string access_log_path;
+ std::ofstream access_log;
+
AppMetrics() {
current_counts = new spp::sparse_hash_map<std::string, uint64_t>();
counts = new spp::sparse_hash_map<std::string, uint64_t>();
current_durations = new spp::sparse_hash_map<std::string, uint64_t>();
durations = new spp::sparse_hash_map<std::string, uint64_t>();
+
+ access_log_path = Config::get_instance().get_access_log_path();
+ if(!access_log_path.empty()) {
+ access_log.open(access_log_path, std::ofstream::out | std::ofstream::app);
+ }
}
~AppMetrics() {
@@ -62,6 +72,10 @@ public:
void increment_write_metrics(uint64_t route_hash, uint64_t duration);
+ void write_access_log(const uint64_t epoch_millis, const char* remote_ip, const std::string& path);
+
+ void flush_access_log();
+
void window_reset();
void get(const std::string& rps_key, const std::string& latency_key, nlohmann::json &result) const;
diff --git a/include/config.h b/include/config.h
index 3ea0bbf4..89bc0638 100644
--- a/include/config.h
+++ b/include/config.h
@@ -50,6 +50,8 @@ private:
uint32_t thread_pool_size;
+ bool enable_access_logging;
+
protected:
Config() {
@@ -66,6 +68,7 @@ protected:
this->num_documents_parallel_load = 1000;
this->thread_pool_size = 0; // will be set dynamically if not overridden
this->ssl_refresh_interval_seconds = 8 * 60 * 60;
+ this->enable_access_logging = false;
}
Config(Config const&) {
@@ -239,6 +242,18 @@ public:
return this->ssl_refresh_interval_seconds;
}
+ bool get_enable_access_logging() const {
+ return this->enable_access_logging;
+ }
+
+ std::string get_access_log_path() const {
+ if(this->log_dir.empty()) {
+ return "";
+ }
+
+ return this->log_dir + "/typesense-access.log";
+ }
+
// loaders
std::string get_env(const char *name) {
@@ -333,6 +348,8 @@ public:
if(!get_env("TYPESENSE_SSL_REFRESH_INTERVAL_SECONDS").empty()) {
this->ssl_refresh_interval_seconds = std::stoi(get_env("TYPESENSE_SSL_REFRESH_INTERVAL_SECONDS"));
}
+
+ this->enable_access_logging = ("TRUE" == get_env("TYPESENSE_ENABLE_ACCESS_LOGGING"));
}
void load_config_file(cmdline::parser & options) {
@@ -462,6 +479,11 @@ public:
if(reader.Exists("server", "ssl-refresh-interval-seconds")) {
this->ssl_refresh_interval_seconds = (int) reader.GetInteger("server", "ssl-refresh-interval-seconds", 8 * 60 * 60);
}
+
+ if(reader.Exists("server", "enable-access-logging")) {
+ auto enable_access_logging_str = reader.Get("server", "enable-cors", "false");
+ this->enable_access_logging = (enable_access_logging_str == "true");
+ }
}
void load_config_cmd_args(cmdline::parser & options) {
@@ -570,6 +592,10 @@ public:
if(options.exist("ssl-refresh-interval-seconds")) {
this->ssl_refresh_interval_seconds = options.get<uint32_t>("ssl-refresh-interval-seconds");
}
+
+ if(options.exist("enable-access-logging")) {
+ this->enable_cors = options.get<bool>("enable-access-logging");
+ }
}
void set_cors_domains(std::string& cors_domains_value) {
diff --git a/include/http_data.h b/include/http_data.h
index 97cc5a33..75e91b5e 100644
--- a/include/http_data.h
+++ b/include/http_data.h
@@ -6,6 +6,7 @@
#include <vector>
#include <future>
#include <chrono>
+#include <iomanip>
#include "json.hpp"
#include "string_utils.h"
#include "logger.h"
@@ -258,27 +259,25 @@ struct http_req {
//LOG(INFO) << "~http_req " << this;
if(_req != nullptr) {
+ Config& config = Config::get_instance();
+
uint64_t now = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
uint64_t ms_since_start = (now - start_ts) / 1000;
std::string metric_identifier = http_method + " " + path_without_query;
AppMetrics::get_instance().increment_duration(metric_identifier, ms_since_start);
-
AppMetrics::get_instance().increment_write_metrics(route_hash, ms_since_start);
- // log slow request if logging is enabled
- Config& config = Config::get_instance();
- std::string query_string = "?";
- for(const auto& kv: params) {
- if(kv.first != AUTH_HEADER) {
- query_string += kv.first + "=" + kv.second + "&";
- }
- }
-
- std::string full_url_path = metric_identifier + query_string;
-
if(config.get_log_slow_requests_time_ms() > 0 && int(ms_since_start) > config.get_log_slow_requests_time_ms()) {
+ // log slow request if logging is enabled
+ std::string query_string = "?";
+ for(const auto& kv: params) {
+ if(kv.first != AUTH_HEADER) {
+ query_string += kv.first + "=" + kv.second + "&";
+ }
+ }
+ std::string full_url_path = metric_identifier + query_string;
LOG(INFO) << "SLOW REQUEST: " << "(" + std::to_string(ms_since_start) + " ms) " << full_url_path;
}
}
diff --git a/include/string_utils.h b/include/string_utils.h
index b444c599..f5e9f414 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -327,4 +327,6 @@ struct StringUtils {
static bool ends_with(std::string const &str, std::string const &ending);
static bool contains_word(const std::string& haystack, const std::string& needle);
+
+ static char* get_ip_str(const struct sockaddr* sa, char* s, size_t maxlen);
};
\ No newline at end of file
diff --git a/include/typesense_server_utils.h b/include/typesense_server_utils.h
index 6feb4572..e4c61909 100644
--- a/include/typesense_server_utils.h
+++ b/include/typesense_server_utils.h
@@ -20,7 +20,7 @@ bool directory_exists(const std::string& dir_path);
void init_cmdline_options(cmdline::parser& options, int argc, char **argv);
-int init_logger(Config& config, const std::string& server_version);
+int init_root_logger(Config& config, const std::string& server_version);
int run_server(const Config& config, const std::string& version,
void (*master_server_routes)());
diff --git a/src/app_metrics.cpp b/src/app_metrics.cpp
index 0d6dbc3d..b4c6c299 100644
--- a/src/app_metrics.cpp
+++ b/src/app_metrics.cpp
@@ -112,3 +112,14 @@ void AppMetrics::window_reset() {
current_durations = new spp::sparse_hash_map<std::string, uint64_t>();
}
+void AppMetrics::write_access_log(const uint64_t epoch_millis, const char* remote_ip, const std::string& path) {
+ if(!access_log_path.empty()) {
+ access_log << epoch_millis << "\t" << remote_ip << "\t" << path << "\n";
+ }
+}
+
+void AppMetrics::flush_access_log() {
+ if(!access_log_path.empty()) {
+ access_log << std::flush;
+ }
+}
diff --git a/src/http_server.cpp b/src/http_server.cpp
index bf97130b..cd97131c 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -59,6 +59,7 @@ void HttpServer::on_metrics_refresh_timeout(h2o_timer_t *entry) {
h2o_custom_timer_t* custom_timer = reinterpret_cast<h2o_custom_timer_t*>(entry);
AppMetrics::get_instance().window_reset();
+ AppMetrics::get_instance().flush_access_log();
HttpServer *hs = static_cast<HttpServer*>(custom_timer->data);
@@ -303,6 +304,23 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {
std::string metric_identifier = http_method + " " + path_without_query;
AppMetrics::get_instance().increment_count(metric_identifier, 1);
+ if(Config::get_instance().get_enable_access_logging()) {
+ const size_t IP_MAX_LEN = 64;
+ char ip_str[IP_MAX_LEN];
+ sockaddr sa;
+
+ if(0 != req->conn->callbacks->get_peername(req->conn, &sa)) {
+ StringUtils::get_ip_str(&sa, ip_str, IP_MAX_LEN);
+ } else {
+ strncpy(ip_str, "NA", 2);
+ }
+
+ uint64_t now = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+ auto epoch_millis = now / 1000;
+ AppMetrics::get_instance().write_access_log(epoch_millis, ip_str, metric_identifier);
+ }
+
// Handle CORS
if(h2o_handler->http_server->cors_enabled) {
h2o_iovec_t response_origin = {(char*)"*", 1};
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index 16e96100..ca5e3a3b 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -126,7 +126,7 @@ int main(int argc, char **argv) {
exit(1);
}
- int ret_code = init_logger(config, TYPESENSE_VERSION);
+ int ret_code = init_root_logger(config, TYPESENSE_VERSION);
if(ret_code != 0) {
return ret_code;
}
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index 221bc2eb..33964dad 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -5,6 +5,8 @@
#include <random>
#include <openssl/sha.h>
#include <map>
+#include <netinet/in.h>
+#include <arpa/inet.h>
#include "logger.h"
@@ -313,6 +315,24 @@ bool StringUtils::contains_word(const std::string& haystack, const std::string&
return true;
}
+char* StringUtils::get_ip_str(const struct sockaddr* sa, char* s, size_t maxlen) {
+ switch (sa->sa_family) {
+ case AF_INET:
+ inet_ntop(AF_INET, &(((struct sockaddr_in*) sa)->sin_addr), s, maxlen);
+ break;
+
+ case AF_INET6:
+ inet_ntop(AF_INET6, &(((struct sockaddr_in6*) sa)->sin6_addr), s, maxlen);
+ break;
+
+ default:
+ strncpy(s, "Unknown AF", maxlen);
+ return NULL;
+ }
+
+ return s;
+}
+
/*size_t StringUtils::unicode_length(const std::string& bytes) {
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;
return utf8conv.from_bytes(bytes).size();
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 2bfcde92..b214354b 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -97,6 +97,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("config", '\0', "Path to the configuration file.", false, "");
+ options.add<bool>("enable-access-logging", '\0', "Enable access logging.", false, false);
+
// DEPRECATED
options.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");
options.add<uint32_t>("listen-port", 'p', "[DEPRECATED: use `api-port`] Port on which Typesense API service listens.", false, 8108);
@@ -104,7 +106,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
"to start as read-only replica.", false, "");
}
-int init_logger(Config & config, const std::string & server_version) {
+int init_root_logger(Config & config, const std::string & server_version) {
google::InitGoogleLogging("typesense");
std::string log_dir = config.get_log_dir();