Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Enable slow searches logging by default (30s cutoff)
Kishore Nallan1a66a25e2f3c
diff --git a/include/collection.h b/include/collection.hindex a7391cb5..859967b5 100644--- a/include/collection.h+++ b/include/collection.h@@ -409,7 +409,8 @@ public:const size_t filter_curated_hits_option = 2,const bool prioritize_token_position = false,const std::string& vector_query_str = "",- const bool enable_highlight_v1 = true) const;+ const bool enable_highlight_v1 = true,+ const uint64_t search_time_start_us = 0) const;Option<bool> get_filter_ids(const std::string & simple_filter_query,std::vector<std::pair<size_t, uint32_t*>>& index_ids);diff --git a/include/config.h b/include/config.hindex 305c7f81..59095649 100644--- a/include/config.h+++ b/include/config.h@@ -59,6 +59,8 @@ private:std::atomic<bool> skip_writes;+ std::atomic<int> log_slow_searches_time_ms;+protected:Config() {@@ -80,6 +82,7 @@ protected:this->disk_used_max_percentage = 100;this->memory_used_max_percentage = 100;this->skip_writes = false;+ this->log_slow_searches_time_ms = 30 * 1000;}Config(Config const&) {@@ -142,6 +145,10 @@ public:this->log_slow_requests_time_ms = log_slow_requests_time_ms;}+ void set_log_slow_searches_time_ms(int log_slow_searches_time_ms) {+ this->log_slow_searches_time_ms = log_slow_searches_time_ms;+ }+void set_healthy_read_lag(size_t healthy_read_lag) {this->healthy_read_lag = healthy_read_lag;}@@ -245,6 +252,10 @@ public:return this->log_slow_requests_time_ms;}+ int get_log_slow_searches_time_ms() const {+ return this->log_slow_searches_time_ms;+ }+size_t get_num_collections_parallel_load() const {return this->num_collections_parallel_load;}@@ -364,6 +375,10 @@ public:this->log_slow_requests_time_ms = std::stoi(get_env("TYPESENSE_LOG_SLOW_REQUESTS_TIME_MS"));}+ if(!get_env("TYPESENSE_LOG_SLOW_SEARCHES_TIME_MS").empty()) {+ this->log_slow_searches_time_ms = std::stoi(get_env("TYPESENSE_LOG_SLOW_SEARCHES_TIME_MS"));+ }+if(!get_env("TYPESENSE_NUM_COLLECTIONS_PARALLEL_LOAD").empty()) {this->num_collections_parallel_load = std::stoi(get_env("TYPESENSE_NUM_COLLECTIONS_PARALLEL_LOAD"));}@@ -513,6 +528,10 @@ public:this->log_slow_requests_time_ms = (int) reader.GetInteger("server", "log-slow-requests-time-ms", -1);}+ if(reader.Exists("server", "log-slow-searches-time-ms")) {+ this->log_slow_searches_time_ms = (int) reader.GetInteger("server", "log-slow-searches-time-ms", 30*1000);+ }+if(reader.Exists("server", "num-collections-parallel-load")) {this->num_collections_parallel_load = (int) reader.GetInteger("server", "num-collections-parallel-load", 0);}@@ -643,6 +662,10 @@ public:this->log_slow_requests_time_ms = options.get<int>("log-slow-requests-time-ms");}+ if(options.exist("log-slow-searches-time-ms")) {+ this->log_slow_searches_time_ms = options.get<int>("log-slow-searches-time-ms");+ }+if(options.exist("num-collections-parallel-load")) {this->num_collections_parallel_load = options.get<uint32_t>("num-collections-parallel-load");}diff --git a/include/http_data.h b/include/http_data.hindex 9c4228d2..f51ed2d1 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -261,8 +261,8 @@ struct http_req {chunk_len(0), body(body), body_index(0), data(nullptr), ready(false),log_index(0), is_diposed(false), client_ip(client_ip) {- start_ts = std::chrono::duration_cast<std::chrono::microseconds>(- std::chrono::system_clock::now().time_since_epoch()).count();+ const auto& tv = _req->processed_at.at;+ start_ts = (tv.tv_sec * 1000 * 1000) + tv.tv_usec;if(_req != nullptr) {is_http_v1 = (_req->version < 0x200);@@ -279,21 +279,40 @@ struct http_req {std::chrono::system_clock::now().time_since_epoch()).count();uint64_t ms_since_start = (now - start_ts) / 1000;- std::string metric_identifier = http_method + " " + path_without_query;+ const std::string metric_identifier = http_method + " " + path_without_query;AppMetrics::get_instance().increment_duration(metric_identifier, ms_since_start);AppMetrics::get_instance().increment_write_metrics(route_hash, ms_since_start);- if(config.get_log_slow_requests_time_ms() >= 0 && int(ms_since_start) >= config.get_log_slow_requests_time_ms()) {+ bool log_slow_searches = config.get_log_slow_searches_time_ms() >= 0 &&+ int(ms_since_start) >= config.get_log_slow_searches_time_ms() &&+ (path_without_query == "/multi_search" ||+ StringUtils::ends_with(path_without_query, "/documents/search"));++ bool log_slow_requests = config.get_log_slow_requests_time_ms() >= 0 &&+ int(ms_since_start) >= config.get_log_slow_requests_time_ms();++ if(log_slow_searches || log_slow_requests) {// log slow request if logging is enabledstd::string query_string = "?";- for(const auto& kv: params) {- if(kv.first != AUTH_HEADER) {- query_string += kv.first + "=" + kv.second + "&";+ bool is_multi_search_query = (path_without_query == "/multi_search");++ if(is_multi_search_query) {+ StringUtils::erase_char(body, '\n');+ } else {+ // ignore params map of multi_search since it is mutated for every search object in the POST body+ for(const auto& kv: params) {+ if(kv.first != AUTH_HEADER) {+ query_string += kv.first + "=" + kv.second + "&";+ }}}+std::string full_url_path = metric_identifier + query_string;- LOG(INFO) << "SLOW REQUEST: " << "(" + std::to_string(ms_since_start) + " ms) "- << client_ip << " " << full_url_path;++ // NOTE: we log the `body` ONLY for multi-search query+ LOG(INFO) << "event=slow_request, time=" << ms_since_start << " ms"+ << ", client_ip=" << client_ip << ", endpoint=" << full_url_path+ << ", body=" << (is_multi_search_query ? body : "");}}diff --git a/include/or_iterator.h b/include/or_iterator.hindex d98b94a9..67fd5ddf 100644--- a/include/or_iterator.h+++ b/include/or_iterator.h@@ -68,9 +68,8 @@ bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_statewhile(its.size() == it_size && its[0].valid()) {num_processed++;- if (num_processed % 65536 == 0 &&- std::chrono::duration_cast<std::chrono::milliseconds>(- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {search_cutoff = true;break;}@@ -100,9 +99,8 @@ bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_statewhile(its.size() == it_size && !at_end2(its)) {num_processed++;- if (num_processed % 65536 == 0 &&- std::chrono::duration_cast<std::chrono::milliseconds>(- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {search_cutoff = true;break;}@@ -138,9 +136,8 @@ bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_statewhile(its.size() == it_size && !at_end(its)) {num_processed++;- if (num_processed % 65536 == 0 &&- std::chrono::duration_cast<std::chrono::milliseconds>(- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {search_cutoff = true;break;}diff --git a/include/posting_list.h b/include/posting_list.hindex 95a57cbc..dea742f7 100644--- a/include/posting_list.h+++ b/include/posting_list.h@@ -211,9 +211,8 @@ bool posting_list_t::block_intersect(std::vector<posting_list_t::iterator_t>& itcase 1:while(its[0].valid()) {num_processed++;- if (num_processed % 65536 == 0 &&- std::chrono::duration_cast<std::chrono::milliseconds>(- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {search_cutoff = true;break;}@@ -228,9 +227,8 @@ bool posting_list_t::block_intersect(std::vector<posting_list_t::iterator_t>& itcase 2:while(!at_end2(its)) {num_processed++;- if (num_processed % 65536 == 0 &&- std::chrono::duration_cast<std::chrono::milliseconds>(- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {search_cutoff = true;break;}@@ -249,9 +247,8 @@ bool posting_list_t::block_intersect(std::vector<posting_list_t::iterator_t>& itdefault:while(!at_end(its)) {num_processed++;- if (num_processed % 65536 == 0 &&- std::chrono::duration_cast<std::chrono::milliseconds>(- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {search_cutoff = true;break;}diff --git a/include/string_utils.h b/include/string_utils.hindex 3472b582..01d3dccd 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -315,6 +315,8 @@ struct StringUtils {static void replace_all(std::string& subject, const std::string& search,const std::string& replace);+ static void erase_char(std::string& str, const char c);+static std::string trim_curly_spaces(const std::string& str);static bool ends_with(std::string const &str, std::string const &ending);diff --git a/include/thread_local_vars.h b/include/thread_local_vars.hindex d4dd725f..0ec8ed1f 100644--- a/include/thread_local_vars.h+++ b/include/thread_local_vars.h@@ -4,6 +4,6 @@ extern thread_local int64_t write_log_index;// These are used for circuit breaking search requests// NOTE: if you fork off main search thread, care must be taken to initialize these from parent thread values-extern thread_local std::chrono::high_resolution_clock::time_point search_begin;-extern thread_local int64_t search_stop_ms;+extern thread_local uint64_t search_begin_us;+extern thread_local uint64_t search_stop_us;extern thread_local bool search_cutoff;\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex ca608293..edb6e388 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -867,13 +867,16 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,const size_t filter_curated_hits_option,const bool prioritize_token_position,const std::string& vector_query_str,- const bool enable_highlight_v1) const {+ const bool enable_highlight_v1,+ const uint64_t search_time_start_us) const {std::shared_lock lock(mutex);// setup thread local vars- search_stop_ms = search_stop_millis;- search_begin = std::chrono::high_resolution_clock::now();+ search_stop_us = search_stop_millis * 1000;+ search_begin_us = (search_time_start_us != 0) ? search_time_start_us :+ std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();search_cutoff = false;if(raw_query != "*" && raw_search_fields.empty()) {diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 69e1a425..81098b6c 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -767,7 +767,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& resize_t filter_curated_hits_option = 2;std::string highlight_fields;bool exhaustive_search = false;- size_t search_cutoff_ms = 3600000;+ size_t search_cutoff_ms = 30 * 1000;enable_t split_join_tokens = fallback;size_t max_candidates = 0;std::vector<enable_t> infixes;diff --git a/src/http_server.cpp b/src/http_server.cppindex 6dcf8ed2..ec155639 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -474,8 +474,6 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {}}--std::shared_ptr<http_req> request = std::make_shared<http_req>(req, rpath->http_method, path_without_query,route_hash, query_map, embedded_params_vec,api_auth_key_sent, body, client_ip);diff --git a/src/index.cpp b/src/index.cppindex 93b7375d..39c80189 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -23,17 +23,17 @@#include <timsort.hpp>#include "logger.h"-#define RETURN_CIRCUIT_BREAKER if(std::chrono::duration_cast<std::chrono::milliseconds>(\- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) { \- search_cutoff = true; \- return ;\- }+#define RETURN_CIRCUIT_BREAKER if((std::chrono::duration_cast<std::chrono::microseconds>( \+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \+ search_cutoff = true; \+ return ;\+ }-#define BREAK_CIRCUIT_BREAKER if(std::chrono::duration_cast<std::chrono::milliseconds>(\- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) { \- search_cutoff = true; \- break;\- }+#define BREAK_CIRCUIT_BREAKER if((std::chrono::duration_cast<std::chrono::microseconds>( \+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \+ search_cutoff = true; \+ break;\+ }spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;@@ -2328,8 +2328,8 @@ void Index::search_infix(const std::string& query, const std::string& field_nameauto search_tree = search_index.at(field_name);- const auto parent_search_begin = search_begin;- const auto parent_search_stop_ms = search_stop_ms;+ const auto parent_search_begin = search_begin_us;+ const auto parent_search_stop_ms = search_stop_us;auto parent_search_cutoff = search_cutoff;for(auto infix_set: infix_sets) {@@ -2337,7 +2337,7 @@ void Index::search_infix(const std::string& query, const std::string& field_name&num_processed, &m_process, &cv_process,&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff]() {- search_begin = parent_search_begin;+ search_begin_us = parent_search_begin;search_cutoff = parent_search_cutoff;auto op_search_stop_ms = parent_search_stop_ms/2;@@ -2362,8 +2362,8 @@ void Index::search_infix(const std::string& query, const std::string& field_name// check for search cutoff but only once every 2^10 docs to reduce overheadif(((num_iterated + 1) % (1 << 12)) == 0) {- if (std::chrono::duration_cast<std::chrono::milliseconds>(- std::chrono::high_resolution_clock::now() - search_begin).count() > op_search_stop_ms) {+ if ((std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().+ time_since_epoch()).count() - search_begin_us) > op_search_stop_ms) {search_cutoff = true;break;}@@ -4335,8 +4335,8 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,size_t num_queued = 0;size_t filter_index = 0;- const auto parent_search_begin = search_begin;- const auto parent_search_stop_ms = search_stop_ms;+ const auto parent_search_begin = search_begin_us;+ const auto parent_search_stop_ms = search_stop_us;auto parent_search_cutoff = search_cutoff;for(size_t thread_id = 0; thread_id < num_threads && filter_index < filter_ids_length; thread_id++) {@@ -4361,8 +4361,8 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,batch_result_ids, batch_res_len,&num_processed, &m_process, &cv_process]() {- search_begin = parent_search_begin;- search_stop_ms = parent_search_stop_ms;+ search_begin_us = parent_search_begin;+ search_stop_us = parent_search_stop_ms;search_cutoff = parent_search_cutoff;size_t filter_index = 0;diff --git a/src/string_utils.cpp b/src/string_utils.cppindex a4374b65..e7893004 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -217,6 +217,10 @@ void StringUtils::replace_all(std::string& subject, const std::string& search, c}}+void StringUtils::erase_char(std::string& str, const char c) {+ str.erase(std::remove(str.begin(), str.end(), c), str.cend());+}+std::string StringUtils::trim_curly_spaces(const std::string& str) {std::string left_trimmed;int i = 0;diff --git a/src/thread_local_vars.cpp b/src/thread_local_vars.cppindex 4cb3562a..258d8b5f 100644--- a/src/thread_local_vars.cpp+++ b/src/thread_local_vars.cpp@@ -2,6 +2,6 @@#include "thread_local_vars.h"thread_local int64_t write_log_index = 0;-thread_local std::chrono::high_resolution_clock::time_point search_begin;-thread_local int64_t search_stop_ms;+thread_local uint64_t search_begin_us;+thread_local uint64_t search_stop_us;thread_local bool search_cutoff = false;diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 4ad8d321..64b945fa 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -105,6 +105,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<int>("memory-used-max-percentage", '\0', "Reject writes when memory usage exceeds this percentage. Default: 100 (never reject).", false, 100);options.add<bool>("skip-writes", '\0', "Skip all writes except config changes. Default: false.", false, false);+ options.add<int>("log-slow-searches-time-ms", '\0', "When >= 0, searches that take longer than this duration are logged.", false, 30*1000);+// DEPRECATEDoptions.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");options.add<uint32_t>("listen-port", 'p', "[DEPRECATED: use `api-port`] Port on which Typesense API service listens.", false, 8108);