Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add flag for logging search query at the start of req cycle.
Kishore Nallan4a2892c886f6
diff --git a/include/tsconfig.h b/include/tsconfig.hindex 3c48dea1..fcf36bb0 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -74,6 +74,8 @@ private:uint32_t db_compaction_interval;+ bool enable_search_logging;+protected:Config() {@@ -103,6 +105,8 @@ protected:this->analytics_flush_interval = 3600; // in secondsthis->housekeeping_interval = 1800; // in secondsthis->db_compaction_interval = 0; // in seconds, disabled++ this->enable_search_logging = false;}Config(Config const&) {@@ -332,6 +336,10 @@ public:return this->enable_search_analytics;}+ bool get_enable_search_logging() const {+ return this->enable_search_logging;+ }+int get_disk_used_max_percentage() const {return this->disk_used_max_percentage;}@@ -474,6 +482,7 @@ public:this->enable_access_logging = ("TRUE" == get_env("TYPESENSE_ENABLE_ACCESS_LOGGING"));this->enable_search_analytics = ("TRUE" == get_env("TYPESENSE_ENABLE_SEARCH_ANALYTICS"));+ this->enable_search_logging = ("TRUE" == get_env("TYPESENSE_ENABLE_SEARCH_LOGGING"));if(!get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE").empty()) {this->disk_used_max_percentage = std::stoi(get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE"));@@ -653,6 +662,11 @@ public:this->enable_search_analytics = (enable_search_analytics_str == "true");}+ if(reader.Exists("server", "enable-search-logging")) {+ auto enable_search_logging_str = reader.Get("server", "enable-search-logging", "false");+ this->enable_search_logging = (enable_search_logging_str == "true");+ }+if(reader.Exists("server", "disk-used-max-percentage")) {this->disk_used_max_percentage = (int) reader.GetInteger("server", "disk-used-max-percentage", 100);}@@ -833,6 +847,9 @@ public:this->enable_search_analytics = options.get<bool>("enable-search-analytics");}+ if(options.exist("enable-search-logging")) {+ this->enable_search_logging = options.get<bool>("enable-search-logging");+ }}void set_cors_domains(std::string& cors_domains_value) {@@ -846,6 +863,10 @@ public:this->enable_search_analytics = enable_search_analytics;}+ void set_enable_search_logging(bool enable_search_logging) {+ this->enable_search_logging = enable_search_logging;+ }+// validationOption<bool> is_valid() {diff --git a/src/http_server.cpp b/src/http_server.cppindex 794970ea..1be9716a 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -458,17 +458,45 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {}}- const std::string & body = std::string(req->entity.base, req->entity.len);+ const std::string& body = std::string(req->entity.base, req->entity.len);std::vector<nlohmann::json> embedded_params_vec;-if(RateLimitManager::getInstance()->is_rate_limited({RateLimitedEntityType::api_key, api_auth_key_sent}, {RateLimitedEntityType::ip, client_ip})) {std::string message = "{ \"message\": \"Rate limit exceeded or blocked\"}";return send_response(req, 429, message);}+ bool is_multi_search_query = (root_resource == "multi_search");++ if(Config::get_instance().get_enable_search_logging()) {+ std::string query_string = "?";+ bool is_search_query = (is_multi_search_query ||+ StringUtils::ends_with(path_without_query, "/documents/search"));++ if(is_search_query) {+ std::string search_payload;++ if(is_multi_search_query) {+ search_payload = body;+ StringUtils::erase_char(search_payload, '\n');+ } else {+ // ignore params map of multi_search since it is mutated for every search object in the POST body+ for(const auto& kv: query_map) {+ if(kv.first != http_req::AUTH_HEADER) {+ query_string += kv.first + "=" + kv.second + "&";+ }+ }+ }++ std::string full_url_path = metric_identifier + query_string;++ // NOTE: we log the `body` ONLY for multi-search query+ LOG(INFO) << "event=search_request" << ", client_ip=" << client_ip << ", endpoint=" << full_url_path+ << ", body=" << (is_multi_search_query ? search_payload : "");+ }+ }- if(root_resource != "multi_search") {+ if(!is_multi_search_query) {// multi_search needs to be handled later because the API key could be part of request body and// the whole request body might not be available right now.bool authenticated = h2o_handler->http_server->auth_handler(query_map, embedded_params_vec, body, *rpath,diff --git a/src/tsconfig.cpp b/src/tsconfig.cppindex ee7af9e1..66cb0bb3 100644--- a/src/tsconfig.cpp+++ b/src/tsconfig.cpp@@ -16,6 +16,24 @@ Option<bool> Config::update_config(const nlohmann::json& req_json) {found_config = true;}+ if(req_json.count("log-slow-searches-time-ms") != 0) {+ if(!req_json["log-slow-searches-time-ms"].is_number_integer()) {+ return Option<bool>(400, "Configuration `log-slow-searches-time-ms` must be an integer.");+ }++ set_log_slow_searches_time_ms(req_json["log-slow-searches-time-ms"].get<int>());+ found_config = true;+ }++ if(req_json.count("enable-search-logging") != 0) {+ if(!req_json["enable-search-logging"].is_boolean()) {+ return Option<bool>(400, "Configuration `enable-search-logging` must be a boolean.");+ }++ set_enable_search_logging(req_json["enable-search-logging"].get<bool>());+ found_config = true;+ }+if(req_json.count("healthy-read-lag") != 0) {if(!req_json["healthy-read-lag"].is_number_integer()) {return Option<bool>(400, "Configuration `healthy-read-lag` must be a positive integer.");diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 9e471aba..2c664d68 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -100,6 +100,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<std::string>("config", '\0', "Path to the configuration file.", false, "");options.add<bool>("enable-access-logging", '\0', "Enable access logging.", false, false);+ options.add<bool>("enable-search-logging", '\0', "Enable search logging.", false, false);options.add<bool>("enable-search-analytics", '\0', "Enable search analytics.", false, false);options.add<int>("disk-used-max-percentage", '\0', "Reject writes when used disk space exceeds this percentage. Default: 100 (never reject).", false, 100);options.add<int>("memory-used-max-percentage", '\0', "Reject writes when memory usage exceeds this percentage. Default: 100 (never reject).", false, 100);