Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add flag for logging search query at the start of req cycle.

Kishore Nallan4a2892c886f6
diff --git a/include/tsconfig.h b/include/tsconfig.h
index 3c48dea1..fcf36bb0 100644
--- a/include/tsconfig.h
+++ b/include/tsconfig.h
@@ -74,6 +74,8 @@ private:
uint32_t db_compaction_interval;
+ bool enable_search_logging;
+
protected:
Config() {
@@ -103,6 +105,8 @@ protected:
this->analytics_flush_interval = 3600; // in seconds
this->housekeeping_interval = 1800; // in seconds
this->db_compaction_interval = 0; // in seconds, disabled
+
+ this->enable_search_logging = false;
}
Config(Config const&) {
@@ -332,6 +336,10 @@ public:
return this->enable_search_analytics;
}
+ bool get_enable_search_logging() const {
+ return this->enable_search_logging;
+ }
+
int get_disk_used_max_percentage() const {
return this->disk_used_max_percentage;
}
@@ -474,6 +482,7 @@ public:
this->enable_access_logging = ("TRUE" == get_env("TYPESENSE_ENABLE_ACCESS_LOGGING"));
this->enable_search_analytics = ("TRUE" == get_env("TYPESENSE_ENABLE_SEARCH_ANALYTICS"));
+ this->enable_search_logging = ("TRUE" == get_env("TYPESENSE_ENABLE_SEARCH_LOGGING"));
if(!get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE").empty()) {
this->disk_used_max_percentage = std::stoi(get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE"));
@@ -653,6 +662,11 @@ public:
this->enable_search_analytics = (enable_search_analytics_str == "true");
}
+ if(reader.Exists("server", "enable-search-logging")) {
+ auto enable_search_logging_str = reader.Get("server", "enable-search-logging", "false");
+ this->enable_search_logging = (enable_search_logging_str == "true");
+ }
+
if(reader.Exists("server", "disk-used-max-percentage")) {
this->disk_used_max_percentage = (int) reader.GetInteger("server", "disk-used-max-percentage", 100);
}
@@ -833,6 +847,9 @@ public:
this->enable_search_analytics = options.get<bool>("enable-search-analytics");
}
+ if(options.exist("enable-search-logging")) {
+ this->enable_search_logging = options.get<bool>("enable-search-logging");
+ }
}
void set_cors_domains(std::string& cors_domains_value) {
@@ -846,6 +863,10 @@ public:
this->enable_search_analytics = enable_search_analytics;
}
+ void set_enable_search_logging(bool enable_search_logging) {
+ this->enable_search_logging = enable_search_logging;
+ }
+
// validation
Option<bool> is_valid() {
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 794970ea..1be9716a 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -458,17 +458,45 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {
}
}
- const std::string & body = std::string(req->entity.base, req->entity.len);
+ const std::string& body = std::string(req->entity.base, req->entity.len);
std::vector<nlohmann::json> embedded_params_vec;
-
if(RateLimitManager::getInstance()->is_rate_limited({RateLimitedEntityType::api_key, api_auth_key_sent}, {RateLimitedEntityType::ip, client_ip})) {
std::string message = "{ \"message\": \"Rate limit exceeded or blocked\"}";
return send_response(req, 429, message);
}
+ bool is_multi_search_query = (root_resource == "multi_search");
+
+ if(Config::get_instance().get_enable_search_logging()) {
+ std::string query_string = "?";
+ bool is_search_query = (is_multi_search_query ||
+ StringUtils::ends_with(path_without_query, "/documents/search"));
+
+ if(is_search_query) {
+ std::string search_payload;
+
+ if(is_multi_search_query) {
+ search_payload = body;
+ StringUtils::erase_char(search_payload, '\n');
+ } else {
+ // ignore params map of multi_search since it is mutated for every search object in the POST body
+ for(const auto& kv: query_map) {
+ if(kv.first != http_req::AUTH_HEADER) {
+ query_string += kv.first + "=" + kv.second + "&";
+ }
+ }
+ }
+
+ std::string full_url_path = metric_identifier + query_string;
+
+ // NOTE: we log the `body` ONLY for multi-search query
+ LOG(INFO) << "event=search_request" << ", client_ip=" << client_ip << ", endpoint=" << full_url_path
+ << ", body=" << (is_multi_search_query ? search_payload : "");
+ }
+ }
- if(root_resource != "multi_search") {
+ if(!is_multi_search_query) {
// multi_search needs to be handled later because the API key could be part of request body and
// the whole request body might not be available right now.
bool authenticated = h2o_handler->http_server->auth_handler(query_map, embedded_params_vec, body, *rpath,
diff --git a/src/tsconfig.cpp b/src/tsconfig.cpp
index ee7af9e1..66cb0bb3 100644
--- a/src/tsconfig.cpp
+++ b/src/tsconfig.cpp
@@ -16,6 +16,24 @@ Option<bool> Config::update_config(const nlohmann::json& req_json) {
found_config = true;
}
+ if(req_json.count("log-slow-searches-time-ms") != 0) {
+ if(!req_json["log-slow-searches-time-ms"].is_number_integer()) {
+ return Option<bool>(400, "Configuration `log-slow-searches-time-ms` must be an integer.");
+ }
+
+ set_log_slow_searches_time_ms(req_json["log-slow-searches-time-ms"].get<int>());
+ found_config = true;
+ }
+
+ if(req_json.count("enable-search-logging") != 0) {
+ if(!req_json["enable-search-logging"].is_boolean()) {
+ return Option<bool>(400, "Configuration `enable-search-logging` must be a boolean.");
+ }
+
+ set_enable_search_logging(req_json["enable-search-logging"].get<bool>());
+ found_config = true;
+ }
+
if(req_json.count("healthy-read-lag") != 0) {
if(!req_json["healthy-read-lag"].is_number_integer()) {
return Option<bool>(400, "Configuration `healthy-read-lag` must be a positive integer.");
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 9e471aba..2c664d68 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -100,6 +100,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("config", '\0', "Path to the configuration file.", false, "");
options.add<bool>("enable-access-logging", '\0', "Enable access logging.", false, false);
+ options.add<bool>("enable-search-logging", '\0', "Enable search logging.", false, false);
options.add<bool>("enable-search-analytics", '\0', "Enable search analytics.", false, false);
options.add<int>("disk-used-max-percentage", '\0', "Reject writes when used disk space exceeds this percentage. Default: 100 (never reject).", false, 100);
options.add<int>("memory-used-max-percentage", '\0', "Reject writes when memory usage exceeds this percentage. Default: 100 (never reject).", false, 100);