Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Enable slow searches logging by default (30s cutoff)

Kishore Nallan1a66a25e2f3c
diff --git a/include/collection.h b/include/collection.h
index a7391cb5..859967b5 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -409,7 +409,8 @@ public:
const size_t filter_curated_hits_option = 2,
const bool prioritize_token_position = false,
const std::string& vector_query_str = "",
- const bool enable_highlight_v1 = true) const;
+ const bool enable_highlight_v1 = true,
+ const uint64_t search_time_start_us = 0) const;
Option<bool> get_filter_ids(const std::string & simple_filter_query,
std::vector<std::pair<size_t, uint32_t*>>& index_ids);
diff --git a/include/config.h b/include/config.h
index 305c7f81..59095649 100644
--- a/include/config.h
+++ b/include/config.h
@@ -59,6 +59,8 @@ private:
std::atomic<bool> skip_writes;
+ std::atomic<int> log_slow_searches_time_ms;
+
protected:
Config() {
@@ -80,6 +82,7 @@ protected:
this->disk_used_max_percentage = 100;
this->memory_used_max_percentage = 100;
this->skip_writes = false;
+ this->log_slow_searches_time_ms = 30 * 1000;
}
Config(Config const&) {
@@ -142,6 +145,10 @@ public:
this->log_slow_requests_time_ms = log_slow_requests_time_ms;
}
+ void set_log_slow_searches_time_ms(int log_slow_searches_time_ms) {
+ this->log_slow_searches_time_ms = log_slow_searches_time_ms;
+ }
+
void set_healthy_read_lag(size_t healthy_read_lag) {
this->healthy_read_lag = healthy_read_lag;
}
@@ -245,6 +252,10 @@ public:
return this->log_slow_requests_time_ms;
}
+ int get_log_slow_searches_time_ms() const {
+ return this->log_slow_searches_time_ms;
+ }
+
size_t get_num_collections_parallel_load() const {
return this->num_collections_parallel_load;
}
@@ -364,6 +375,10 @@ public:
this->log_slow_requests_time_ms = std::stoi(get_env("TYPESENSE_LOG_SLOW_REQUESTS_TIME_MS"));
}
+ if(!get_env("TYPESENSE_LOG_SLOW_SEARCHES_TIME_MS").empty()) {
+ this->log_slow_searches_time_ms = std::stoi(get_env("TYPESENSE_LOG_SLOW_SEARCHES_TIME_MS"));
+ }
+
if(!get_env("TYPESENSE_NUM_COLLECTIONS_PARALLEL_LOAD").empty()) {
this->num_collections_parallel_load = std::stoi(get_env("TYPESENSE_NUM_COLLECTIONS_PARALLEL_LOAD"));
}
@@ -513,6 +528,10 @@ public:
this->log_slow_requests_time_ms = (int) reader.GetInteger("server", "log-slow-requests-time-ms", -1);
}
+ if(reader.Exists("server", "log-slow-searches-time-ms")) {
+ this->log_slow_searches_time_ms = (int) reader.GetInteger("server", "log-slow-searches-time-ms", 30*1000);
+ }
+
if(reader.Exists("server", "num-collections-parallel-load")) {
this->num_collections_parallel_load = (int) reader.GetInteger("server", "num-collections-parallel-load", 0);
}
@@ -643,6 +662,10 @@ public:
this->log_slow_requests_time_ms = options.get<int>("log-slow-requests-time-ms");
}
+ if(options.exist("log-slow-searches-time-ms")) {
+ this->log_slow_searches_time_ms = options.get<int>("log-slow-searches-time-ms");
+ }
+
if(options.exist("num-collections-parallel-load")) {
this->num_collections_parallel_load = options.get<uint32_t>("num-collections-parallel-load");
}
diff --git a/include/http_data.h b/include/http_data.h
index 9c4228d2..f51ed2d1 100644
--- a/include/http_data.h
+++ b/include/http_data.h
@@ -261,8 +261,8 @@ struct http_req {
chunk_len(0), body(body), body_index(0), data(nullptr), ready(false),
log_index(0), is_diposed(false), client_ip(client_ip) {
- start_ts = std::chrono::duration_cast<std::chrono::microseconds>(
- std::chrono::system_clock::now().time_since_epoch()).count();
+ const auto& tv = _req->processed_at.at;
+ start_ts = (tv.tv_sec * 1000 * 1000) + tv.tv_usec;
if(_req != nullptr) {
is_http_v1 = (_req->version < 0x200);
@@ -279,21 +279,40 @@ struct http_req {
std::chrono::system_clock::now().time_since_epoch()).count();
uint64_t ms_since_start = (now - start_ts) / 1000;
- std::string metric_identifier = http_method + " " + path_without_query;
+ const std::string metric_identifier = http_method + " " + path_without_query;
AppMetrics::get_instance().increment_duration(metric_identifier, ms_since_start);
AppMetrics::get_instance().increment_write_metrics(route_hash, ms_since_start);
- if(config.get_log_slow_requests_time_ms() >= 0 && int(ms_since_start) >= config.get_log_slow_requests_time_ms()) {
+ bool log_slow_searches = config.get_log_slow_searches_time_ms() >= 0 &&
+ int(ms_since_start) >= config.get_log_slow_searches_time_ms() &&
+ (path_without_query == "/multi_search" ||
+ StringUtils::ends_with(path_without_query, "/documents/search"));
+
+ bool log_slow_requests = config.get_log_slow_requests_time_ms() >= 0 &&
+ int(ms_since_start) >= config.get_log_slow_requests_time_ms();
+
+ if(log_slow_searches || log_slow_requests) {
// log slow request if logging is enabled
std::string query_string = "?";
- for(const auto& kv: params) {
- if(kv.first != AUTH_HEADER) {
- query_string += kv.first + "=" + kv.second + "&";
+ bool is_multi_search_query = (path_without_query == "/multi_search");
+
+ if(is_multi_search_query) {
+ StringUtils::erase_char(body, '\n');
+ } else {
+ // ignore params map of multi_search since it is mutated for every search object in the POST body
+ for(const auto& kv: params) {
+ if(kv.first != AUTH_HEADER) {
+ query_string += kv.first + "=" + kv.second + "&";
+ }
}
}
+
std::string full_url_path = metric_identifier + query_string;
- LOG(INFO) << "SLOW REQUEST: " << "(" + std::to_string(ms_since_start) + " ms) "
- << client_ip << " " << full_url_path;
+
+ // NOTE: we log the `body` ONLY for multi-search query
+ LOG(INFO) << "event=slow_request, time=" << ms_since_start << " ms"
+ << ", client_ip=" << client_ip << ", endpoint=" << full_url_path
+ << ", body=" << (is_multi_search_query ? body : "");
}
}
diff --git a/include/or_iterator.h b/include/or_iterator.h
index d98b94a9..67fd5ddf 100644
--- a/include/or_iterator.h
+++ b/include/or_iterator.h
@@ -68,9 +68,8 @@ bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_state
while(its.size() == it_size && its[0].valid()) {
num_processed++;
- if (num_processed % 65536 == 0 &&
- std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {
+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {
search_cutoff = true;
break;
}
@@ -100,9 +99,8 @@ bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_state
while(its.size() == it_size && !at_end2(its)) {
num_processed++;
- if (num_processed % 65536 == 0 &&
- std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {
+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {
search_cutoff = true;
break;
}
@@ -138,9 +136,8 @@ bool or_iterator_t::intersect(std::vector<or_iterator_t>& its, result_iter_state
while(its.size() == it_size && !at_end(its)) {
num_processed++;
- if (num_processed % 65536 == 0 &&
- std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {
+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {
search_cutoff = true;
break;
}
diff --git a/include/posting_list.h b/include/posting_list.h
index 95a57cbc..dea742f7 100644
--- a/include/posting_list.h
+++ b/include/posting_list.h
@@ -211,9 +211,8 @@ bool posting_list_t::block_intersect(std::vector<posting_list_t::iterator_t>& it
case 1:
while(its[0].valid()) {
num_processed++;
- if (num_processed % 65536 == 0 &&
- std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {
+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {
search_cutoff = true;
break;
}
@@ -228,9 +227,8 @@ bool posting_list_t::block_intersect(std::vector<posting_list_t::iterator_t>& it
case 2:
while(!at_end2(its)) {
num_processed++;
- if (num_processed % 65536 == 0 &&
- std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {
+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {
search_cutoff = true;
break;
}
@@ -249,9 +247,8 @@ bool posting_list_t::block_intersect(std::vector<posting_list_t::iterator_t>& it
default:
while(!at_end(its)) {
num_processed++;
- if (num_processed % 65536 == 0 &&
- std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) {
+ if (num_processed % 65536 == 0 && (std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) {
search_cutoff = true;
break;
}
diff --git a/include/string_utils.h b/include/string_utils.h
index 3472b582..01d3dccd 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -315,6 +315,8 @@ struct StringUtils {
static void replace_all(std::string& subject, const std::string& search,
const std::string& replace);
+ static void erase_char(std::string& str, const char c);
+
static std::string trim_curly_spaces(const std::string& str);
static bool ends_with(std::string const &str, std::string const &ending);
diff --git a/include/thread_local_vars.h b/include/thread_local_vars.h
index d4dd725f..0ec8ed1f 100644
--- a/include/thread_local_vars.h
+++ b/include/thread_local_vars.h
@@ -4,6 +4,6 @@ extern thread_local int64_t write_log_index;
// These are used for circuit breaking search requests
// NOTE: if you fork off main search thread, care must be taken to initialize these from parent thread values
-extern thread_local std::chrono::high_resolution_clock::time_point search_begin;
-extern thread_local int64_t search_stop_ms;
+extern thread_local uint64_t search_begin_us;
+extern thread_local uint64_t search_stop_us;
extern thread_local bool search_cutoff;
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index ca608293..edb6e388 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -867,13 +867,16 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query,
const size_t filter_curated_hits_option,
const bool prioritize_token_position,
const std::string& vector_query_str,
- const bool enable_highlight_v1) const {
+ const bool enable_highlight_v1,
+ const uint64_t search_time_start_us) const {
std::shared_lock lock(mutex);
// setup thread local vars
- search_stop_ms = search_stop_millis;
- search_begin = std::chrono::high_resolution_clock::now();
+ search_stop_us = search_stop_millis * 1000;
+ search_begin_us = (search_time_start_us != 0) ? search_time_start_us :
+ std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
search_cutoff = false;
if(raw_query != "*" && raw_search_fields.empty()) {
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 69e1a425..81098b6c 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -767,7 +767,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
size_t filter_curated_hits_option = 2;
std::string highlight_fields;
bool exhaustive_search = false;
- size_t search_cutoff_ms = 3600000;
+ size_t search_cutoff_ms = 30 * 1000;
enable_t split_join_tokens = fallback;
size_t max_candidates = 0;
std::vector<enable_t> infixes;
diff --git a/src/http_server.cpp b/src/http_server.cpp
index 6dcf8ed2..ec155639 100644
--- a/src/http_server.cpp
+++ b/src/http_server.cpp
@@ -474,8 +474,6 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {
}
}
-
-
std::shared_ptr<http_req> request = std::make_shared<http_req>(req, rpath->http_method, path_without_query,
route_hash, query_map, embedded_params_vec,
api_auth_key_sent, body, client_ip);
diff --git a/src/index.cpp b/src/index.cpp
index 93b7375d..39c80189 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -23,17 +23,17 @@
#include <timsort.hpp>
#include "logger.h"
-#define RETURN_CIRCUIT_BREAKER if(std::chrono::duration_cast<std::chrono::milliseconds>(\
- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) { \
- search_cutoff = true; \
- return ;\
- }
+#define RETURN_CIRCUIT_BREAKER if((std::chrono::duration_cast<std::chrono::microseconds>( \
+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \
+ search_cutoff = true; \
+ return ;\
+ }
-#define BREAK_CIRCUIT_BREAKER if(std::chrono::duration_cast<std::chrono::milliseconds>(\
- std::chrono::high_resolution_clock::now() - search_begin).count() > search_stop_ms) { \
- search_cutoff = true; \
- break;\
- }
+#define BREAK_CIRCUIT_BREAKER if((std::chrono::duration_cast<std::chrono::microseconds>( \
+ std::chrono::system_clock::now().time_since_epoch()).count() - search_begin_us) > search_stop_us) { \
+ search_cutoff = true; \
+ break;\
+ }
spp::sparse_hash_map<uint32_t, int64_t> Index::text_match_sentinel_value;
spp::sparse_hash_map<uint32_t, int64_t> Index::seq_id_sentinel_value;
@@ -2328,8 +2328,8 @@ void Index::search_infix(const std::string& query, const std::string& field_name
auto search_tree = search_index.at(field_name);
- const auto parent_search_begin = search_begin;
- const auto parent_search_stop_ms = search_stop_ms;
+ const auto parent_search_begin = search_begin_us;
+ const auto parent_search_stop_ms = search_stop_us;
auto parent_search_cutoff = search_cutoff;
for(auto infix_set: infix_sets) {
@@ -2337,7 +2337,7 @@ void Index::search_infix(const std::string& query, const std::string& field_name
&num_processed, &m_process, &cv_process,
&parent_search_begin, &parent_search_stop_ms, &parent_search_cutoff]() {
- search_begin = parent_search_begin;
+ search_begin_us = parent_search_begin;
search_cutoff = parent_search_cutoff;
auto op_search_stop_ms = parent_search_stop_ms/2;
@@ -2362,8 +2362,8 @@ void Index::search_infix(const std::string& query, const std::string& field_name
// check for search cutoff but only once every 2^10 docs to reduce overhead
if(((num_iterated + 1) % (1 << 12)) == 0) {
- if (std::chrono::duration_cast<std::chrono::milliseconds>(
- std::chrono::high_resolution_clock::now() - search_begin).count() > op_search_stop_ms) {
+ if ((std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().
+ time_since_epoch()).count() - search_begin_us) > op_search_stop_ms) {
search_cutoff = true;
break;
}
@@ -4335,8 +4335,8 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
size_t num_queued = 0;
size_t filter_index = 0;
- const auto parent_search_begin = search_begin;
- const auto parent_search_stop_ms = search_stop_ms;
+ const auto parent_search_begin = search_begin_us;
+ const auto parent_search_stop_ms = search_stop_us;
auto parent_search_cutoff = search_cutoff;
for(size_t thread_id = 0; thread_id < num_threads && filter_index < filter_ids_length; thread_id++) {
@@ -4361,8 +4361,8 @@ void Index::search_wildcard(filter_node_t const* const& filter_tree_root,
batch_result_ids, batch_res_len,
&num_processed, &m_process, &cv_process]() {
- search_begin = parent_search_begin;
- search_stop_ms = parent_search_stop_ms;
+ search_begin_us = parent_search_begin;
+ search_stop_us = parent_search_stop_ms;
search_cutoff = parent_search_cutoff;
size_t filter_index = 0;
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index a4374b65..e7893004 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -217,6 +217,10 @@ void StringUtils::replace_all(std::string& subject, const std::string& search, c
}
}
+void StringUtils::erase_char(std::string& str, const char c) {
+ str.erase(std::remove(str.begin(), str.end(), c), str.cend());
+}
+
std::string StringUtils::trim_curly_spaces(const std::string& str) {
std::string left_trimmed;
int i = 0;
diff --git a/src/thread_local_vars.cpp b/src/thread_local_vars.cpp
index 4cb3562a..258d8b5f 100644
--- a/src/thread_local_vars.cpp
+++ b/src/thread_local_vars.cpp
@@ -2,6 +2,6 @@
#include "thread_local_vars.h"
thread_local int64_t write_log_index = 0;
-thread_local std::chrono::high_resolution_clock::time_point search_begin;
-thread_local int64_t search_stop_ms;
+thread_local uint64_t search_begin_us;
+thread_local uint64_t search_stop_us;
thread_local bool search_cutoff = false;
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 4ad8d321..64b945fa 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -105,6 +105,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<int>("memory-used-max-percentage", '\0', "Reject writes when memory usage exceeds this percentage. Default: 100 (never reject).", false, 100);
options.add<bool>("skip-writes", '\0', "Skip all writes except config changes. Default: false.", false, false);
+ options.add<int>("log-slow-searches-time-ms", '\0', "When >= 0, searches that take longer than this duration are logged.", false, 30*1000);
+
// DEPRECATED
options.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");
options.add<uint32_t>("listen-port", 'p', "[DEPRECATED: use `api-port`] Port on which Typesense API service listens.", false, 8108);