Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Evented query aggregation skeleton.
Kishore Nallanb7f29aeed129
diff --git a/include/collection.h b/include/collection.hindex caaf897f..c2312659 100644--- a/include/collection.h+++ b/include/collection.h@@ -553,6 +553,8 @@ public:std::vector<uint32_t>& query_by_weights,std::vector<search_field_t>& weighted_search_fields,std::vector<std::string>& reordered_search_fields) const;++ Option<bool> truncate_after_top_k(const std::string& field_name, size_t k);};template<class T>diff --git a/include/core_api.h b/include/core_api.hindex a549c4a1..bc8d297c 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -141,6 +141,14 @@ bool del_throttle(const std::shared_ptr<http_req>& req, const std::shared_ptr<htbool del_exceed(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+// Events++bool post_create_event(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);++bool post_create_event_sink(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);++bool del_event_sink(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+// Misc helpersvoid get_collections_for_auth(std::map<std::string, std::string>& req_params, const std::string& body,diff --git a/include/event_manager.h b/include/event_manager.hnew file mode 100644index 00000000..e012ee8d--- /dev/null+++ b/include/event_manager.h@@ -0,0 +1,30 @@+#pragma once++#include "json.hpp"+#include "option.h"++class EventManager {+private:+ EventManager() = default;++ ~EventManager() = default;++ static constexpr char* EVENT_TYPE = "type";+ static constexpr char* EVENT_DATA = "data";++public:+ static EventManager& get_instance() {+ static EventManager instance;+ return instance;+ }++ EventManager(EventManager const&) = delete;+ void operator=(EventManager const&) = delete;++ Option<uint32_t> create_sink(const nlohmann::json& sink_config, bool write_to_disk = true);++ Option<bool> remove_sink(const std::string& name);++ bool add_event(const nlohmann::json& event);++};\ No newline at end of filediff --git a/include/http_data.h b/include/http_data.hindex 893c1138..c060adc7 100644--- a/include/http_data.h+++ b/include/http_data.h@@ -207,6 +207,7 @@ public:struct http_req {static constexpr const char* AUTH_HEADER = "x-typesense-api-key";+ static constexpr const char* USER_HEADER = "x-typesense-user-id";static constexpr const char* AGENT_HEADER = "user-agent";h2o_req_t* _req;diff --git a/include/index.h b/include/index.hindex 138c774d..b9aa57f6 100644--- a/include/index.h+++ b/include/index.h@@ -956,6 +956,9 @@ public:std::map<size_t, std::map<size_t, uint32_t>>& included_ids_map,std::vector<uint32_t>& included_ids_vec,std::unordered_set<uint32_t>& excluded_group_ids) const;++ Option<bool> seq_ids_outside_top_k(const std::string& field_name, size_t k,+ std::vector<uint32_t>& outside_seq_ids);};template<class T>diff --git a/include/num_tree.h b/include/num_tree.hindex 5406a109..2170a30e 100644--- a/include/num_tree.h+++ b/include/num_tree.h@@ -48,6 +48,8 @@ public:size_t size();+ void seq_ids_outside_top_k(size_t k, std::vector<uint32_t>& seq_ids);+void contains(const NUM_COMPARATOR& comparator, const int64_t& value,const uint32_t& context_ids_length,uint32_t* const& context_ids,diff --git a/include/popular_queries.h b/include/popular_queries.hnew file mode 100644index 00000000..95493f27--- /dev/null+++ b/include/popular_queries.h@@ -0,0 +1,51 @@+#pragma once++#include <string>+#include <vector>+#include <tsl/htrie_map.h>+#include <json.hpp>+#include <atomic>+#include <shared_mutex>++class PopularQueries {+public:+ struct QWithTimestamp {+ std::string query;+ uint64_t timestamp;++ QWithTimestamp(const std::string& query, uint64_t timestamp) : query(query), timestamp(timestamp) {}+ };++ static const size_t QUERY_FINALIZATION_INTERVAL_MICROS = 4 * 1000 * 1000;++private:++ size_t k;+ const size_t max_size;++ // counts aggregated within the current node+ tsl::htrie_map<char, uint32_t> local_counts;+ std::shared_mutex lmutex;++ std::unordered_map<std::string, std::vector<QWithTimestamp>> user_prefix_queries;+ std::shared_mutex umutex;++public:++ PopularQueries(size_t k);++ void add(const std::string& value, const bool live_query, const std::string& user_id,+ uint64_t now_ts_us = 0);++ void compact_user_queries(uint64_t now_ts_us);++ void serialize_as_docs(std::string& docs);++ void reset_local_counts();++ size_t get_k();++ std::unordered_map<std::string, std::vector<QWithTimestamp>> get_user_prefix_queries();++ tsl::htrie_map<char, uint32_t> get_local_counts();+};diff --git a/include/query_suggestions.h b/include/query_suggestions.hnew file mode 100644index 00000000..e42b4c4f--- /dev/null+++ b/include/query_suggestions.h@@ -0,0 +1,69 @@+#pragma once+#include "popular_queries.h"+#include "option.h"+#include "raft_server.h"+#include <vector>+#include <string>+#include <unordered_map>+#include <shared_mutex>++class QuerySuggestions {+private:+ mutable std::mutex mutex;+ std::condition_variable cv;++ std::atomic<bool> quit = false;++ const size_t QUERY_COMPACTION_INTERVAL_S = 60;++ struct suggestion_config_t {+ std::string name;+ std::string suggestion_collection;+ std::vector<std::string> query_collections;+ size_t max_suggestions;+ };++ // config name => config+ std::unordered_map<std::string, suggestion_config_t> suggestion_configs;++ // query collection => suggestion collections+ std::unordered_map<std::string, std::vector<std::string>> query_collection_mapping;++ // suggestion collection => popular queries+ std::unordered_map<std::string, PopularQueries*> popular_queries;++ Store* store = nullptr;++ QuerySuggestions() {}++ ~QuerySuggestions();++public:++ static constexpr const char* EVENT_SINK_CONFIG_PREFIX = "$ES";++ static constexpr const char* SINK_TYPE = "query_suggestions";++ static QuerySuggestions& get_instance() {+ static QuerySuggestions instance;+ return instance;+ }++ QuerySuggestions(QuerySuggestions const&) = delete;+ void operator=(QuerySuggestions const&) = delete;++ void init(Store* store);++ void run(ReplicationState* raft_server);++ Option<nlohmann::json> create_index(const nlohmann::json& payload, bool write_to_disk = true);++ Option<bool> remove_suggestion_index(const std::string& name);++ void add_suggestion(const std::string& query_collection,+ std::string& query, const bool live_query, const std::string& user_id);++ void stop();++ void dispose();+};diff --git a/include/raft_server.h b/include/raft_server.hindex 8280338b..0a9c8368 100644--- a/include/raft_server.h+++ b/include/raft_server.h@@ -237,6 +237,8 @@ public:nlohmann::json get_status();+ std::string get_leader_url() const;+private:friend class ReplicationClosure;diff --git a/include/tsconfig.h b/include/tsconfig.hindex 50f5e094..fe7b8f0c 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -66,6 +66,10 @@ private:std::atomic<bool> reset_peers_on_error;+ bool enable_search_analytics;++ uint32_t analytics_flush_interval;+protected:Config() {@@ -90,6 +94,9 @@ protected:this->skip_writes = false;this->log_slow_searches_time_ms = 30 * 1000;this->reset_peers_on_error = false;++ this->enable_search_analytics = false;+ this->analytics_flush_interval = 3600; // in seconds}Config(Config const&) {@@ -291,6 +298,10 @@ public:return this->cache_num_entries;}+ size_t get_analytics_flush_interval() const {+ return this->analytics_flush_interval;+ }+size_t get_thread_pool_size() const {return this->thread_pool_size;}@@ -303,6 +314,10 @@ public:return this->enable_access_logging;}+ bool get_enable_search_analytics() const {+ return this->enable_search_analytics;+ }+int get_disk_used_max_percentage() const {return this->disk_used_max_percentage;}@@ -418,6 +433,10 @@ public:this->cache_num_entries = std::stoi(get_env("TYPESENSE_CACHE_NUM_ENTRIES"));}+ if(!get_env("TYPESENSE_ANALYTICS_FLUSH_INTERVAL").empty()) {+ this->analytics_flush_interval = std::stoi(get_env("TYPESENSE_ANALYTICS_FLUSH_INTERVAL"));+ }+if(!get_env("TYPESENSE_THREAD_POOL_SIZE").empty()) {this->thread_pool_size = std::stoi(get_env("TYPESENSE_THREAD_POOL_SIZE"));}@@ -431,6 +450,7 @@ public:}this->enable_access_logging = ("TRUE" == get_env("TYPESENSE_ENABLE_ACCESS_LOGGING"));+ this->enable_search_analytics = ("TRUE" == get_env("TYPESENSE_ENABLE_SEARCH_ANALYTICS"));if(!get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE").empty()) {this->disk_used_max_percentage = std::stoi(get_env("TYPESENSE_DISK_USED_MAX_PERCENTAGE"));@@ -576,6 +596,10 @@ public:this->cache_num_entries = (int) reader.GetInteger("server", "cache-num-entries", 1000);}+ if(reader.Exists("server", "analytics-flush-interval")) {+ this->analytics_flush_interval = (int) reader.GetInteger("server", "analytics-flush-interval", 3600);+ }+if(reader.Exists("server", "thread-pool-size")) {this->thread_pool_size = (int) reader.GetInteger("server", "thread-pool-size", 0);}@@ -589,6 +613,11 @@ public:this->enable_access_logging = (enable_access_logging_str == "true");}+ if(reader.Exists("server", "enable-search-analytics")) {+ auto enable_search_analytics_str = reader.Get("server", "enable-search-analytics", "false");+ this->enable_search_analytics = (enable_search_analytics_str == "true");+ }+if(reader.Exists("server", "disk-used-max-percentage")) {this->disk_used_max_percentage = (int) reader.GetInteger("server", "disk-used-max-percentage", 100);}@@ -728,6 +757,10 @@ public:this->cache_num_entries = options.get<uint32_t>("cache-num-entries");}+ if(options.exist("analytics-flush-interval")) {+ this->analytics_flush_interval = options.get<uint32_t>("analytics-flush-interval");+ }+if(options.exist("thread-pool-size")) {this->thread_pool_size = options.get<uint32_t>("thread-pool-size");}@@ -755,6 +788,11 @@ public:if(options.exist("reset-peers-on-error")) {this->reset_peers_on_error = options.get<bool>("reset-peers-on-error");}++ if(options.exist("enable-search-analytics")) {+ this->enable_search_analytics = options.get<bool>("enable-search-analytics");+ }+}void set_cors_domains(std::string& cors_domains_value) {diff --git a/src/collection.cpp b/src/collection.cppindex b9c5a012..1c3e51b1 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -4769,4 +4769,22 @@ void Collection::process_remove_field_for_embedding_fields(const field& the_fiel}}-}\ No newline at end of file+}++Option<bool> Collection::truncate_after_top_k(const string &field_name, size_t k) {+ std::vector<uint32_t> seq_ids;+ auto op = index->seq_ids_outside_top_k(field_name, k, seq_ids);++ if(!op.ok()) {+ return op;+ }++ for(auto seq_id: seq_ids) {+ auto remove_op = remove_if_found(seq_id);+ if(!remove_op.ok()) {+ LOG(ERROR) << "Error while truncating top k: " << remove_op.error();+ }+ }++ return Option<bool>(true);+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 4fe4702e..8d45cdc8 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -2,6 +2,8 @@#include <vector>#include <json.hpp>#include <app_metrics.h>+#include <query_suggestions.h>+#include <event_manager.h>#include "collection_manager.h"#include "batched_indexer.h"#include "logger.h"@@ -1060,6 +1062,15 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& renlohmann::json result = result_op.get();+ if(Config::get_instance().get_enable_search_analytics()) {+ if(result.count("found") != 0 && result["found"].get<size_t>() != 0) {+ std::string processed_query = raw_query;+ Tokenizer::normalize_ascii(processed_query);+ QuerySuggestions::get_instance().add_suggestion(collection->get_name(), processed_query,+ true, req_params["x-typesense-user-id"]);+ }+ }+if(exclude_fields.count("search_time_ms") == 0) {result["search_time_ms"] = timeMillis;}@@ -1287,6 +1298,17 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectioncollection->add_synonym(collection_synonym);}+ // restore query suggestions configs+ std::vector<std::string> sink_config_jsons;+ cm.store->scan_fill(QuerySuggestions::EVENT_SINK_CONFIG_PREFIX,+ std::string(QuerySuggestions::EVENT_SINK_CONFIG_PREFIX) + "`",+ sink_config_jsons);++ for(const auto& sink_config_json: sink_config_jsons) {+ nlohmann::json sink_config = nlohmann::json::parse(sink_config_json);+ EventManager::get_instance().create_sink(sink_config, false);+ }+// Fetch records from the store and re-create memory indexconst std::string seq_id_prefix = collection->get_seq_id_collection_prefix();std::string upper_bound_key = collection->get_seq_id_collection_prefix() + "`"; // cannot inline thisdiff --git a/src/core_api.cpp b/src/core_api.cppindex b32176d7..3ace3fa8 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -13,6 +13,7 @@#include "core_api_utils.h"#include "lru/lru.hpp"#include "ratelimit_manager.h"+#include "event_manager.h"using namespace std::chrono_literals;@@ -2032,4 +2033,59 @@ Option<std::pair<std::string,std::string>> get_api_key_and_ip(const std::string&}return Option<std::pair<std::string,std::string>>(std::make_pair(api_key, ip));+}++bool post_create_event(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ nlohmann::json req_json;++ try {+ req_json = nlohmann::json::parse(req->body);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ res->set_400("Bad JSON.");+ return false;+ }++ bool success = EventManager::get_instance().add_event(req_json);+ if(success) {+ res->set_201(R"({"ok": true)");+ return true;+ }++ res->set_400(R"({"ok": false)");+ return false;+}++bool post_create_event_sink(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ // connects an event to a sink, which for now, is another collection+ nlohmann::json req_json;++ try {+ req_json = nlohmann::json::parse(req->body);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ res->set_400("Bad JSON.");+ return false;+ }++ auto op = EventManager::get_instance().create_sink(req_json);++ if(!op.ok()) {+ res->set(op.code(), op.error());+ return false;+ }++ res->set_201(req_json.dump());+ return true;+}++bool del_event_sink(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ auto op = EventManager::get_instance().remove_sink(req->params["name"]);+ if(!op.ok()) {+ res->set(op.code(), op.error());+ return false;+ }++ res->set_200(R"({"ok": true)");+ return true;}\ No newline at end of filediff --git a/src/event_manager.cpp b/src/event_manager.cppnew file mode 100644index 00000000..49aa8185--- /dev/null+++ b/src/event_manager.cpp@@ -0,0 +1,84 @@+#include <query_suggestions.h>+#include "event_manager.h"++bool EventManager::add_event(const nlohmann::json& event) {+ /*+ Sample event payload:++ {+ "type": "search",+ "data": {+ "q": "Nike shoes",+ "collections": ["products"]+ }+ }+ */++ if(!event.contains("type")) {+ return false;+ }++ const auto& event_type_val = event[EVENT_TYPE];++ if(event_type_val.is_string()) {+ const std::string& event_type = event_type_val.get<std::string>();+ if(event_type == "search") {+ if(!event.contains("data")) {+ return false;+ }++ const auto& event_data_val = event[EVENT_DATA];++ if(!event_data_val.is_object()) {+ return false;+ }++ const auto& event_data_query_it = event_data_val["q"];++ if(!event_data_query_it.is_string() || !event_data_val["collections"].is_array()) {+ return false;+ }++ for(const auto& coll: event_data_val["collections"]) {+ if(!coll.is_string()) {+ return false;+ }++ std::string query = event_data_query_it.get<std::string>();+ QuerySuggestions::get_instance().add_suggestion(coll.get<std::string>(), query, false, "");+ }+ }+ }++ return true;+}++Option<uint32_t> EventManager::create_sink(const nlohmann::json& sink_config, bool write_to_disk) {+ if(!sink_config.contains("name") || !sink_config["name"].is_string()) {+ return Option<uint32_t>(400, "Request payload contains invalid name.");+ }++ if(!sink_config.contains("type") || !sink_config["type"].is_string()) {+ return Option<uint32_t>(400, "Request payload contains invalid type.");+ }++ if(!sink_config.contains("source") || !sink_config["source"].is_object()) {+ return Option<uint32_t>(400, "Request payload contains invalid source.");+ }++ if(!sink_config.contains("destination") || !sink_config["destination"].is_object()) {+ return Option<uint32_t>(400, "Request payload contains invalid destination.");+ }++ if(sink_config.contains("type") && sink_config["type"] == QuerySuggestions::SINK_TYPE) {+ QuerySuggestions::get_instance().create_index(sink_config, write_to_disk);+ } else {+ return Option<uint32_t>(400, ("Missing or invalid event sink type."));+ }++ return Option<uint32_t>(200);+}++Option<bool> EventManager::remove_sink(const std::string& name) {+ return QuerySuggestions::get_instance().remove_suggestion_index(name);+}diff --git a/src/http_server.cpp b/src/http_server.cppindex 7756b481..855fe2c2 100644--- a/src/http_server.cpp+++ b/src/http_server.cpp@@ -407,6 +407,17 @@ int HttpServer::catch_all_handler(h2o_handler_t *_h2o_handler, h2o_req_t *req) {api_auth_key_sent = query_map[http_req::AUTH_HEADER];}+ // extract user id from header, if not already present as GET param+ ssize_t user_header_cursor = h2o_find_header_by_str(&req->headers, http_req::USER_HEADER, strlen(http_req::USER_HEADER), -1);++ if(user_header_cursor != -1) {+ h2o_iovec_t & slot = req->headers.entries[user_header_cursor].value;+ std::string user_id_sent = std::string(slot.base, slot.len);+ query_map[http_req::USER_HEADER] = user_id_sent;+ } else if(query_map.count(http_req::USER_HEADER) == 0) {+ query_map[http_req::USER_HEADER] = client_ip;+ }+route_path *rpath = nullptr;uint64_t route_hash = h2o_handler->http_server->find_route(path_parts, http_method, &rpath);diff --git a/src/index.cpp b/src/index.cppindex 66a1470d..f70a4f55 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -6275,6 +6275,18 @@ size_t Index::num_seq_ids() const {return seq_ids->num_ids();}+Option<bool> Index::seq_ids_outside_top_k(const std::string& field_name, size_t k,+ std::vector<uint32_t>& outside_seq_ids) {+ auto field_it = numerical_index.find(field_name);++ if(field_it == sort_index.end()) {+ return Option<bool>(400, "Field not found in numerical index.");+ }++ field_it->second->seq_ids_outside_top_k(k, outside_seq_ids);+ return Option<bool>(true);+}+void Index::resolve_space_as_typos(std::vector<std::string>& qtokens, const string& field_name,std::vector<std::vector<std::string>>& resolved_queries) const {diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex 6dad2175..915b5a91 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -68,6 +68,11 @@ void master_server_routes() {server->put("/presets/:name", put_upsert_preset);server->del("/presets/:name", del_preset);+ // events+ server->post("/events", post_create_event);+ server->post("/events/sinks", post_create_event_sink);+ server->del("/events/sinks/:name", del_event_sink);+// metaserver->get("/metrics.json", get_metrics_json);server->get("/stats.json", get_stats_json);diff --git a/src/num_tree.cpp b/src/num_tree.cppindex 1bcdbc9f..c59cb008 100644--- a/src/num_tree.cpp+++ b/src/num_tree.cpp@@ -318,6 +318,30 @@ void num_tree_t::contains(const NUM_COMPARATOR& comparator, const int64_t& valueresult_ids = out;}+void num_tree_t::seq_ids_outside_top_k(size_t k, std::vector<uint32_t> &seq_ids) {+ size_t ids_skipped = 0;++ for (auto iter = int64map.rbegin(); iter != int64map.rend(); ++iter) {+ auto num_ids = ids_t::num_ids(iter->second);+ if(ids_skipped > k) {+ ids_t::uncompress(iter->second, seq_ids);+ } else if((ids_skipped + num_ids) > k) {+ // this element hits the limit, so we pick partial IDs to satisfy k+ std::vector<uint32_t> ids;+ ids_t::uncompress(iter->second, ids);+ for(size_t i = 0; i < ids.size(); i++) {+ auto seq_id = ids[i];+ if(ids_skipped + i >= k) {+ seq_ids.push_back(seq_id);+ }+ }+ }++ ids_skipped += num_ids;+ }+}++size_t num_tree_t::size() {return int64map.size();}diff --git a/src/popular_queries.cpp b/src/popular_queries.cppnew file mode 100644index 00000000..ba46ff72--- /dev/null+++ b/src/popular_queries.cpp@@ -0,0 +1,118 @@+#include "popular_queries.h"+#include "logger.h"+#include <algorithm>+#include <mutex>+#include "string_utils.h"++PopularQueries::PopularQueries(size_t k) : k(k), max_size(k * 2) {++}++void PopularQueries::add(const std::string& key, const bool live_query, const std::string& user_id, uint64_t now_ts_us) {+ if(live_query) {+ // live query must be aggregated first to their final form as they could be prefix queries+ if(now_ts_us == 0) {+ now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();+ }++ if(!umutex.try_lock()) {+ // instead of locking we just skip incrementing keys during consolidation time+ return ;+ }++ auto& queries = user_prefix_queries[user_id];+ if(queries.size() < 100) {+ queries.emplace_back(key, now_ts_us);+ }++ umutex.unlock();++ } else {+ if(!lmutex.try_lock()) {+ // instead of locking we just skip incrementing keys during consolidation time+ return ;+ }++ auto it = local_counts.find(key);++ if(it != local_counts.end()) {+ it.value()++;+ } else if(local_counts.size() < max_size) {+ // skip count when map has become too large (to prevent abuse)+ local_counts.emplace(key, 1);+ }++ lmutex.unlock();+ }+}++void PopularQueries::serialize_as_docs(std::string& docs) {+ std::shared_lock lk(lmutex);++ std::string key_buffer;+ for(auto it = local_counts.begin(); it != local_counts.end(); ++it) {+ it.key(key_buffer);+ nlohmann::json doc;+ doc["id"] = std::to_string(StringUtils::hash_wy(key_buffer.c_str(), key_buffer.size()));+ doc["q"] = key_buffer;+ doc["$operations"]["increment"]["count"] = it.value();+ docs += doc.dump() + "\n";+ }++ if(!docs.empty()) {+ docs.pop_back();+ }+}++void PopularQueries::reset_local_counts() {+ std::unique_lock lk(lmutex);+ local_counts.clear();+}++size_t PopularQueries::get_k() {+ return k;+}++void PopularQueries::compact_user_queries(uint64_t now_ts_us) {+ std::unique_lock lk(umutex);+ std::vector<std::string> keys_to_delete;++ for(auto& kv: user_prefix_queries) {+ auto& queries = kv.second;+ int64_t last_consolidated_index = -1;+ for(uint32_t i = 0; i < queries.size(); i++) {+ if(now_ts_us - queries[i].timestamp < QUERY_FINALIZATION_INTERVAL_MICROS) {+ break;+ }++ uint64_t diff_micros = (i == queries.size()-1) ? (now_ts_us - queries[i].timestamp) :+ (queries[i + 1].timestamp - queries[i].timestamp);++ if(diff_micros > QUERY_FINALIZATION_INTERVAL_MICROS) {+ add(queries[i].query, false, "");+ last_consolidated_index = i;+ }+ }++ queries.erase(queries.begin(), queries.begin() + last_consolidated_index+1);++ if(queries.empty()) {+ keys_to_delete.push_back(kv.first);+ }+ }++ for(auto& key: keys_to_delete) {+ user_prefix_queries.erase(key);+ }+}++std::unordered_map<std::string, std::vector<PopularQueries::QWithTimestamp>> PopularQueries::get_user_prefix_queries() {+ std::unique_lock lk(umutex);+ return user_prefix_queries;+}++tsl::htrie_map<char, uint32_t> PopularQueries::get_local_counts() {+ std::unique_lock lk(lmutex);+ return local_counts;+}diff --git a/src/query_suggestions.cpp b/src/query_suggestions.cppnew file mode 100644index 00000000..a8bafc00--- /dev/null+++ b/src/query_suggestions.cpp@@ -0,0 +1,243 @@+#include <mutex>+#include <thread>+#include "query_suggestions.h"+#include "tokenizer.h"+#include "http_client.h"+#include "collection_manager.h"++Option<nlohmann::json> QuerySuggestions::create_index(const nlohmann::json& payload, bool write_to_disk) {+ /*+ Sample payload:++ {+ "name": "top_queries",+ "type": "query_suggestions",+ "max_suggestions": 1000,+ "source": {+ "collections": ["brands", "products"]+ },+ "destination": {+ "collection": "top_queries"+ }+ }+ */++ // structural payload validation is done upstream, e.g. presence of source and destination is validated+ // specific validations will be done here++ const std::string& suggestion_config_name = payload["name"].get<std::string>();++ size_t max_suggestions = 1000;++ if(payload.contains("max_suggestions") && payload["max_suggestions"].is_number_integer()) {+ max_suggestions = payload["max_suggestions"].get<size_t>();+ }++ if(suggestion_configs.find(suggestion_config_name) != suggestion_configs.end()) {+ return Option<nlohmann::json>(400, "There's already another configuration with the name `" ++ suggestion_config_name + "`.");+ }++ if(!payload["source"].contains("collections") || !payload["source"]["collections"].is_array()) {+ return Option<nlohmann::json>(400, "Must contain a valid list of source collections.");+ }++ if(!payload["destination"].contains("collection") || !payload["destination"]["collection"].is_string()) {+ return Option<nlohmann::json>(400, "Must contain a valid destination collection.");+ }++ const std::string& suggestion_collection = payload["destination"]["collection"].get<std::string>();+ suggestion_config_t suggestion_config;+ suggestion_config.name = suggestion_config_name;+ suggestion_config.suggestion_collection = suggestion_collection;+ suggestion_config.max_suggestions = max_suggestions;++ for(const auto& coll: payload["source"]["collections"]) {+ if(!coll.is_string()) {+ return Option<nlohmann::json>(400, "Must contain a valid list of source collection names.");+ }++ const std::string& src_collection = coll.get<std::string>();+ suggestion_config.query_collections.push_back(src_collection);+ }++ std::unique_lock lock(mutex);++ suggestion_configs.emplace(suggestion_config_name, suggestion_config);++ for(const auto& query_coll: suggestion_config.query_collections) {+ query_collection_mapping[query_coll].push_back(suggestion_collection);+ }++ PopularQueries* popularQueries = new PopularQueries(max_suggestions);+ popular_queries.emplace(suggestion_collection, popularQueries);++ if(write_to_disk) {+ auto suggestion_key = std::string(EVENT_SINK_CONFIG_PREFIX) + "_" + suggestion_config_name;+ bool inserted = store->insert(suggestion_key, payload.dump());+ if(!inserted) {+ return Option<nlohmann::json>(500, "Error while storing the suggestion config to disk.");+ }+ }++ return Option<nlohmann::json>(payload);+}++QuerySuggestions::~QuerySuggestions() {+ std::unique_lock lock(mutex);++ for(auto& kv: popular_queries) {+ delete kv.second;+ }+}++Option<bool> QuerySuggestions::remove_suggestion_index(const std::string &name) {+ std::unique_lock lock(mutex);++ auto suggestion_configs_it = suggestion_configs.find(name);++ if(suggestion_configs_it == suggestion_configs.end()) {+ return Option<bool>(404, "Sink not found.");+ }++ const auto& suggestion_collection = suggestion_configs_it->second.suggestion_collection;++ for(const auto& query_collection: suggestion_configs_it->second.query_collections) {+ query_collection_mapping.erase(query_collection);+ }++ if(popular_queries.count(suggestion_collection) != 0) {+ delete popular_queries[suggestion_collection];+ popular_queries.erase(suggestion_collection);+ }++ suggestion_configs.erase(name);++ auto suggestion_key = std::string(EVENT_SINK_CONFIG_PREFIX) + "_" + name;+ bool erased = store->remove(suggestion_key);+ if(!erased) {+ return Option<bool>(500, "Error while removing the sink config from disk.");+ }++ return Option<bool>(true);+}++void QuerySuggestions::add_suggestion(const std::string &query_collection, std::string &query,+ const bool live_query, const std::string& user_id) {+ // look up suggestion collections for the query collection+ std::unique_lock lock(mutex);+ const auto& suggestion_collections_it = query_collection_mapping.find(query_collection);+ if(suggestion_collections_it != query_collection_mapping.end()) {+ for(const auto& suggestion_collection: suggestion_collections_it->second) {+ const auto& popular_queries_it = popular_queries.find(suggestion_collection);+ if(popular_queries_it != popular_queries.end()) {+ Tokenizer::normalize_ascii(query);+ popular_queries_it->second->add(query, live_query, user_id);+ }+ }+ }+}++void QuerySuggestions::run(ReplicationState* raft_server) {+ uint64_t prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ while(!quit) {+ std::unique_lock lk(mutex);+ cv.wait_for(lk, std::chrono::seconds(QUERY_COMPACTION_INTERVAL_S), [&] { return quit.load(); });++ //LOG(INFO) << "QuerySuggestions::run";++ if(quit) {+ lk.unlock();+ break;+ }++ for(const auto& suggestion_config: suggestion_configs) {+ const std::string& sink_name = suggestion_config.first;+ const std::string& suggestion_coll = suggestion_config.second.suggestion_collection;++ auto popular_queries_it = popular_queries.find(suggestion_coll);+ if(popular_queries_it == popular_queries.end()) {+ continue;+ }++ // need to prepare the counts as JSON docs for import into the suggestion collection+ // {"id": "432432", "q": "foo", "$operations": {"increment": {"count": 100}}}++ PopularQueries* popularQueries = popular_queries_it->second;++ // aggregate prefix queries to their final form+ auto now = std::chrono::system_clock::now().time_since_epoch();+ auto now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(now).count();+ popularQueries->compact_user_queries(now_ts_us);++ auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(now).count();++ if(now_ts_seconds - prev_persistence_s < Config::get_instance().get_analytics_flush_interval()) {+ // we will persist aggregation every hour+ continue;+ }++ prev_persistence_s = now_ts_seconds;++ std::string import_payload;+ popularQueries->serialize_as_docs(import_payload);++ if(import_payload.empty()) {+ continue;+ }++ // send http request+ std::string leader_url = raft_server->get_leader_url();+ if(!leader_url.empty()) {+ const std::string& resource_url = leader_url + "collections/" + suggestion_coll ++ "/documents/import?action=emplace";+ std::string res;+ std::map<std::string, std::string> res_headers;+ long status_code = HttpClient::post_response(resource_url, import_payload,+ res, res_headers, 10*1000);++ if(status_code != 200) {+ LOG(ERROR) << "Error while sending query suggestions events to leader. "+ << "Status code: " << status_code << ", response: " << res;+ } else {+ LOG(INFO) << "Sent query suggestions to leader for aggregation.";+ popularQueries->reset_local_counts();++ if(raft_server->is_leader()) {+ // try to run top-K compaction of suggestion collection+ auto coll = CollectionManager::get_instance().get_collection(suggestion_coll);+ if (coll == nullptr) {+ LOG(ERROR) << "No collection found for suggestions aggregation: " + suggestion_coll;+ continue;+ }++ coll->truncate_after_top_k("count", popularQueries->get_k());+ }+ }+ }+ }++ lk.unlock();+ }++ dispose();+}++void QuerySuggestions::stop() {+ quit = true;+ cv.notify_all();+}++void QuerySuggestions::dispose() {+ for(auto& kv: popular_queries) {+ delete kv.second;+ }++ popular_queries.clear();+}++void QuerySuggestions::init(Store* store) {+ this->store = store;+}diff --git a/src/raft_server.cpp b/src/raft_server.cppindex a8432bee..14c7323f 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -955,6 +955,21 @@ bool ReplicationState::get_ext_snapshot_succeeded() {return ext_snapshot_succeeded;}+std::string ReplicationState::get_leader_url() const {+ std::shared_lock lock(node_mutex);++ if(node->leader_id().is_empty()) {+ LOG(ERROR) << "Could not get leader status, as node does not have a leader!";+ return "";+ }++ const std::string & leader_addr = node->leader_id().to_string();+ lock.unlock();++ const std::string protocol = api_uses_ssl ? "https" : "http";+ return get_node_url_path(leader_addr, "/", protocol);+}+void TimedSnapshotClosure::Run() {// Auto delete this after Done()std::unique_ptr<TimedSnapshotClosure> self_guard(this);diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex c362f60c..73a15e2b 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -13,6 +13,7 @@#include <arpa/inet.h>#include <sys/socket.h>#include <ifaddrs.h>+#include <query_suggestions.h>#include "core_api.h"#include "ratelimit_manager.h"@@ -94,6 +95,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<std::string>("config", '\0', "Path to the configuration file.", false, "");options.add<bool>("enable-access-logging", '\0', "Enable access logging.", false, false);+ options.add<bool>("enable-search-analytics", '\0', "Enable search analytics.", false, false);options.add<int>("disk-used-max-percentage", '\0', "Reject writes when used disk space exceeds this percentage. Default: 100 (never reject).", false, 100);options.add<int>("memory-used-max-percentage", '\0', "Reject writes when memory usage exceeds this percentage. Default: 100 (never reject).", false, 100);options.add<bool>("skip-writes", '\0', "Skip all writes except config changes. Default: false.", false, false);@@ -101,6 +103,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<int>("log-slow-searches-time-ms", '\0', "When >= 0, searches that take longer than this duration are logged.", false, 30*1000);options.add<int>("cache-num-entries", '\0', "Number of entries to cache.", false, 1000);+ options.add<uint32_t>("analytics-flush-interval", '\0', "Frequency of persisting analytics data to disk (in seconds).", false, 3600);// DEPRECATEDoptions.add<std::string>("listen-address", 'h', "[DEPRECATED: use `api-address`] Address to which Typesense API service binds.", false, "0.0.0.0");@@ -394,6 +397,8 @@ int run_server(const Config & config, const std::string & version, void (*masterHttpClient & httpClient = HttpClient::get_instance();httpClient.init(config.get_api_key());+ QuerySuggestions::get_instance().init(&store);+server = new HttpServer(version,config.get_api_address(),@@ -450,6 +455,10 @@ int run_server(const Config & config, const std::string & version, void (*masterbatch_indexer->run();});+ std::thread event_sink_thread([&replication_state]() {+ QuerySuggestions::get_instance().run(&replication_state);+ });+std::string path_to_nodes = config.get_nodes();start_raft_server(replication_state, state_dir, path_to_nodes,config.get_peering_address(),@@ -466,6 +475,12 @@ int run_server(const Config & config, const std::string & version, void (*masterLOG(INFO) << "Waiting for batch indexing thread to be done...";batch_indexing_thread.join();+ LOG(INFO) << "Shutting down event sink thread...";+ QuerySuggestions::get_instance().stop();++ LOG(INFO) << "Waiting for event sink thread to be done...";+ event_sink_thread.join();+LOG(INFO) << "Shutting down server_thread_pool";server_thread_pool.shutdown();diff --git a/test/popular_queries_test.cpp b/test/popular_queries_test.cppnew file mode 100644index 00000000..5877243d--- /dev/null+++ b/test/popular_queries_test.cpp@@ -0,0 +1,87 @@+#include <gtest/gtest.h>+#include "popular_queries.h"+#include "logger.h"++class PopularQueriesTest : public ::testing::Test {+protected:+ virtual void SetUp() {++ }++ virtual void TearDown() {++ }+};++TEST_F(PopularQueriesTest, PrefixQueryCompaction) {+ PopularQueries pq(10);++ auto now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ // compaction when no queries have been entered+ pq.compact_user_queries(now_ts_us);+ auto queries = pq.get_user_prefix_queries();+ ASSERT_TRUE(queries.empty());++ // compaction after user has typed first prefix but before compaction interval has happened+ pq.add("f", true, "0", now_ts_us+1);+ pq.compact_user_queries(now_ts_us+2);+ queries = pq.get_user_prefix_queries();+ ASSERT_EQ(1, queries.size());+ ASSERT_EQ(1, queries.count("0"));+ ASSERT_EQ(1, queries["0"].size());+ ASSERT_EQ("f", queries["0"][0].query);+ ASSERT_EQ(now_ts_us+1, queries["0"][0].timestamp);+ ASSERT_EQ(0, pq.get_local_counts().size());++ // compaction interval has happened+ pq.compact_user_queries(now_ts_us+PopularQueries::QUERY_FINALIZATION_INTERVAL_MICROS+100);+ queries = pq.get_user_prefix_queries();+ ASSERT_EQ(0, queries.size());+ auto local_counts = pq.get_local_counts();+ ASSERT_EQ(1, local_counts.size());+ ASSERT_EQ(1, local_counts.count("f"));+ ASSERT_EQ(1, local_counts["f"]);++ // 3 letter search+ pq.reset_local_counts();+ pq.add("f", true, "0", now_ts_us+1);+ pq.add("fo", true, "0", now_ts_us+2);+ pq.add("foo", true, "0", now_ts_us+3);+ pq.compact_user_queries(now_ts_us+PopularQueries::QUERY_FINALIZATION_INTERVAL_MICROS+100);+ queries = pq.get_user_prefix_queries();+ ASSERT_EQ(0, queries.size());+ local_counts = pq.get_local_counts();+ ASSERT_EQ(1, local_counts.size());+ ASSERT_EQ(1, local_counts.count("foo"));+ ASSERT_EQ(1, local_counts["foo"]);++ // 3 letter search + start of next search+ pq.reset_local_counts();+ pq.add("f", true, "0", now_ts_us+1);+ pq.add("fo", true, "0", now_ts_us+2);+ pq.add("foo", true, "0", now_ts_us+3);+ pq.add("b", true, "0", now_ts_us+3+PopularQueries::QUERY_FINALIZATION_INTERVAL_MICROS+100);+ pq.compact_user_queries(now_ts_us+3+PopularQueries::QUERY_FINALIZATION_INTERVAL_MICROS+100+1);+ queries = pq.get_user_prefix_queries();+ ASSERT_EQ(1, queries.size());+ ASSERT_EQ(1, queries["0"].size());+ ASSERT_EQ("b", queries["0"][0].query);+ local_counts = pq.get_local_counts();+ ASSERT_EQ(1, local_counts.size());+ ASSERT_EQ(1, local_counts.count("foo"));+ ASSERT_EQ(1, local_counts["foo"]);++ // continue with that query+ auto prev_ts = now_ts_us+3+PopularQueries::QUERY_FINALIZATION_INTERVAL_MICROS+100+1;+ pq.add("ba", true, "0", prev_ts+1);+ pq.add("bar", true, "0", prev_ts+2);+ pq.compact_user_queries(prev_ts+2+PopularQueries::QUERY_FINALIZATION_INTERVAL_MICROS+1);+ queries = pq.get_user_prefix_queries();+ ASSERT_EQ(0, queries.size());+ local_counts = pq.get_local_counts();+ ASSERT_EQ(2, local_counts.size());+ ASSERT_EQ(1, local_counts.count("bar"));+ ASSERT_EQ(1, local_counts["bar"]);+}