Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
noresults queries analytics
krunal81ae4f80697b
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 41abe683..2aaad8f2 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -6,6 +6,7 @@#include <string>#include <unordered_map>#include <shared_mutex>+#include "noresults_queries.h"struct ClickEvent {std::string query;@@ -93,6 +94,9 @@ private:// suggestion collection => popular queriesstd::unordered_map<std::string, PopularQueries*> popular_queries;+ // suggestion collection => noresults queries+ std::unordered_map<std::string, NoresultsQueries*> noresults_queries;+//query collection => click eventsstd::unordered_map<std::string, std::vector<ClickEvent>> query_collection_click_events;@@ -103,17 +107,18 @@ private:~AnalyticsManager();- Option<bool> remove_popular_queries_index(const std::string& name);+ Option<bool> remove_queries_index(const std::string& name);- Option<bool> create_popular_queries_index(nlohmann::json &payload,+ Option<bool> create_queries_index(nlohmann::json &payload,bool upsert,bool write_to_disk);public:static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";- static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";static constexpr const char* CLICK_EVENT = "$CE";+ static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";+ static constexpr const char* NORESULTS_QUERIES_TYPE = "noresults_queries";static AnalyticsManager& get_instance() {static AnalyticsManager instance;@@ -155,5 +160,12 @@ public:Option<bool> write_click_event_to_store(nlohmann::json& click_event_json);+ void add_noresults_query(const std::string& query_collection,+ const std::string& query, bool live_query, const std::string& user_id);++ void persist_noresults_queries(ReplicationState *raft_server, uint64_t prev_persistence_s);++ std::unordered_map<std::string, NoresultsQueries*> get_noresults_queries();+void resetRateLimit();};diff --git a/include/noresults_queries.h b/include/noresults_queries.hnew file mode 100644index 00000000..d0ed902d--- /dev/null+++ b/include/noresults_queries.h@@ -0,0 +1,51 @@+#pragma once++#include <string>+#include <vector>+#include <tsl/htrie_map.h>+#include <json.hpp>+#include <atomic>+#include <shared_mutex>++class NoresultsQueries {+public:+ struct QWithTimestamp {+ std::string query;+ uint64_t timestamp;++ QWithTimestamp(const std::string& query, uint64_t timestamp) : query(query), timestamp(timestamp) {}+ };++ static const size_t QUERY_FINALIZATION_INTERVAL_MICROS = 4 * 1000 * 1000;++private:++ size_t k;+ const size_t max_size;++ // counts aggregated within the current node+ tsl::htrie_map<char, uint32_t> local_counts;+ std::shared_mutex lmutex;++ std::unordered_map<std::string, std::vector<QWithTimestamp>> user_prefix_queries;+ std::shared_mutex umutex;++public:++ NoresultsQueries(size_t k);++ void add(const std::string& value, const bool live_query, const std::string& user_id,+ uint64_t now_ts_us = 0);++ void compact_user_queries(uint64_t now_ts_us);++ void serialize_as_docs(std::string& docs);++ void reset_local_counts();++ size_t get_k();++ std::unordered_map<std::string, std::vector<QWithTimestamp>> get_user_prefix_queries();++ tsl::htrie_map<char, uint32_t> get_local_counts();+};diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex 5cc95bf0..804e7c79 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -41,14 +41,14 @@ Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert,return Option<bool>(400, "Bad or missing params.");}- if(payload["type"] == POPULAR_QUERIES_TYPE) {- return create_popular_queries_index(payload, upsert, write_to_disk);+ if(payload["type"] == POPULAR_QUERIES_TYPE || payload["type"] == NORESULTS_QUERIES_TYPE) {+ return create_queries_index(payload, upsert, write_to_disk);}return Option<bool>(400, "Invalid type.");}-Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &payload, bool upsert, bool write_to_disk) {+Option<bool> AnalyticsManager::create_queries_index(nlohmann::json &payload, bool upsert, bool write_to_disk) {// params and name are validated upstreamconst std::string& suggestion_config_name = payload["name"].get<std::string>();bool already_exists = suggestion_configs.find(suggestion_config_name) != suggestion_configs.end();@@ -88,8 +88,14 @@ Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &paylsuggestion_config.suggestion_collection = suggestion_collection;suggestion_config.limit = limit;- if(!upsert && popular_queries.count(suggestion_collection) != 0) {- return Option<bool>(400, "There's already another configuration for this destination collection.");+ if(payload["type"] == POPULAR_QUERIES_TYPE) {+ if (!upsert && popular_queries.count(suggestion_collection) != 0) {+ return Option<bool>(400, "There's already another configuration for this destination collection.");+ }+ } else if(payload["type"] == NORESULTS_QUERIES_TYPE) {+ if (!upsert && noresults_queries.count(suggestion_collection) != 0) {+ return Option<bool>(400, "There's already another configuration for this destination collection.");+ }}for(const auto& coll: params["source"]["collections"]) {@@ -105,7 +111,7 @@ Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &paylif(already_exists) {// remove the previous configuration with same name (upsert)- Option<bool> remove_op = remove_popular_queries_index(suggestion_config_name);+ Option<bool> remove_op = remove_queries_index(suggestion_config_name);if(!remove_op.ok()) {return Option<bool>(500, "Error erasing the existing configuration.");;}@@ -117,8 +123,13 @@ Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &paylquery_collection_mapping[query_coll].push_back(suggestion_collection);}- PopularQueries* popularQueries = new PopularQueries(limit);- popular_queries.emplace(suggestion_collection, popularQueries);+ if(payload["type"] == POPULAR_QUERIES_TYPE) {+ PopularQueries *popularQueries = new PopularQueries(limit);+ popular_queries.emplace(suggestion_collection, popularQueries);+ } else if(payload["type"] == NORESULTS_QUERIES_TYPE) {+ NoresultsQueries *noresultsQueries = new NoresultsQueries(limit);+ noresults_queries.emplace(suggestion_collection, noresultsQueries);+ }if(write_to_disk) {auto suggestion_key = std::string(ANALYTICS_RULE_PREFIX) + "_" + suggestion_config_name;@@ -137,6 +148,10 @@ AnalyticsManager::~AnalyticsManager() {for(auto& kv: popular_queries) {delete kv.second;}++ for(auto& kv: noresults_queries) {+ delete kv.second;+ }}Option<nlohmann::json> AnalyticsManager::list_rules() {@@ -154,7 +169,7 @@ Option<nlohmann::json> AnalyticsManager::list_rules() {return Option<nlohmann::json>(rules);}-Option<nlohmann::json> AnalyticsManager::get_rule(const string& name) {+Option<nlohmann::json> AnalyticsManager::get_rule(const std::string& name) {nlohmann::json rule;std::unique_lock lock(mutex);@@ -167,18 +182,18 @@ Option<nlohmann::json> AnalyticsManager::get_rule(const string& name) {return Option<nlohmann::json>(rule);}-Option<bool> AnalyticsManager::remove_rule(const string &name) {+Option<bool> AnalyticsManager::remove_rule(const std::string &name) {std::unique_lock lock(mutex);auto suggestion_configs_it = suggestion_configs.find(name);if(suggestion_configs_it != suggestion_configs.end()) {- return remove_popular_queries_index(name);+ return remove_queries_index(name);}return Option<bool>(404, "Rule not found.");}-Option<bool> AnalyticsManager::remove_popular_queries_index(const std::string &name) {+Option<bool> AnalyticsManager::remove_queries_index(const std::string &name) {// lock is held by callerauto suggestion_configs_it = suggestion_configs.find(name);@@ -197,6 +212,11 @@ Option<bool> AnalyticsManager::remove_popular_queries_index(const std::string &npopular_queries.erase(suggestion_collection);}+ if(noresults_queries.count(suggestion_collection) != 0) {+ delete noresults_queries[suggestion_collection];+ noresults_queries.erase(suggestion_collection);+ }+suggestion_configs.erase(name);auto suggestion_key = std::string(ANALYTICS_RULE_PREFIX) + "_" + name;@@ -263,6 +283,21 @@ Option<bool> AnalyticsManager::add_click_event(const std::string &query_collectireturn Option<bool>(true);}+void AnalyticsManager::add_noresults_query(const std::string &query_collection, const std::string &query,+ bool live_query, const std::string &user_id) {+ // look up suggestion collections for the query collection+ std::unique_lock lock(mutex);+ const auto& suggestion_collections_it = query_collection_mapping.find(query_collection);+ if(suggestion_collections_it != query_collection_mapping.end()) {+ for(const auto& suggestion_collection: suggestion_collections_it->second) {+ const auto& noresults_queries_it = noresults_queries.find(suggestion_collection);+ if(noresults_queries_it != noresults_queries.end()) {+ noresults_queries_it->second->add(query, live_query, user_id);+ }+ }+ }+}+void AnalyticsManager::run(ReplicationState* raft_server) {uint64_t prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();@@ -289,6 +324,7 @@ void AnalyticsManager::run(ReplicationState* raft_server) {persist_suggestions(raft_server, prev_persistence_s);persist_click_events(raft_server, prev_persistence_s);+ persist_noresults_queries(raft_server, prev_persistence_s);prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();@@ -404,6 +440,71 @@ void AnalyticsManager::persist_click_events(ReplicationState *raft_server, uint6}}+void AnalyticsManager::persist_noresults_queries(ReplicationState *raft_server, uint64_t prev_persistence_s) {+ // lock is held by caller+ for(const auto& suggestion_config: suggestion_configs) {+ const std::string& sink_name = suggestion_config.first;+ const std::string& suggestion_coll = suggestion_config.second.suggestion_collection;++ auto noresults_queries_it = noresults_queries.find(suggestion_coll);+ if(noresults_queries_it == noresults_queries.end()) {+ continue;+ }++ // need to prepare the counts as JSON docs for import into the suggestion collection+ // {"id": "432432", "q": "foo", "$operations": {"increment": {"count": 100}}}++ NoresultsQueries* noresultsQueries = noresults_queries_it->second;++ // aggregate prefix queries to their final form+ auto now = std::chrono::system_clock::now().time_since_epoch();+ auto now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(now).count();+ noresultsQueries->compact_user_queries(now_ts_us);++ std::string import_payload;+ noresultsQueries->serialize_as_docs(import_payload);++ if(import_payload.empty()) {+ continue;+ }++ // send http request+ std::string leader_url = raft_server->get_leader_url();+ if(!leader_url.empty()) {+ const std::string& base_url = leader_url + "collections/" + suggestion_coll;+ std::string res;++ const std::string& update_url = base_url + "/documents/import?action=emplace";+ std::map<std::string, std::string> res_headers;+ long status_code = HttpClient::post_response(update_url, import_payload,+ res, res_headers, {}, 10*1000, true);++ if(status_code != 200) {+ LOG(ERROR) << "Error while sending query suggestions events to leader. "+ << "Status code: " << status_code << ", response: " << res;+ } else {+ LOG(INFO) << "Query aggregation for collection: " + suggestion_coll;+ noresultsQueries->reset_local_counts();++ if(raft_server->is_leader()) {+ // try to run top-K compaction of suggestion collection+ const std::string top_k_param = "count:" + std::to_string(noresultsQueries->get_k());+ const std::string& truncate_topk_url = base_url + "/documents?top_k_by=" + top_k_param;+ res.clear();+ res_headers.clear();+ status_code = HttpClient::delete_response(truncate_topk_url, res, res_headers, 10*1000, true);+ if(status_code != 200) {+ LOG(ERROR) << "Error while running top K for query suggestions collection. "+ << "Status code: " << status_code << ", response: " << res;+ } else {+ LOG(INFO) << "Top K aggregation for collection: " + suggestion_coll;+ }+ }+ }+ }+ }+}+void AnalyticsManager::stop() {quit = true;cv.notify_all();@@ -417,6 +518,12 @@ void AnalyticsManager::dispose() {}popular_queries.clear();++ for(auto& kv: noresults_queries) {+ delete kv.second;+ }++ noresults_queries.clear();}void AnalyticsManager::init(Store* store, Store* analytics_store) {@@ -429,6 +536,11 @@ std::unordered_map<std::string, PopularQueries*> AnalyticsManager::get_popular_qreturn popular_queries;}+std::unordered_map<std::string, NoresultsQueries*> AnalyticsManager::get_noresults_queries() {+ std::unique_lock lk(mutex);+ return noresults_queries;+}+nlohmann::json AnalyticsManager::get_click_events() {std::unique_lock lk(mutex);std::vector<std::string> click_event_jsons;diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex cc2657aa..32001590 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1534,6 +1534,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restd::string analytics_query = Tokenizer::normalize_ascii_no_spaces(raw_query);AnalyticsManager::get_instance().add_suggestion(orig_coll_name, analytics_query,true, req_params["x-typesense-user-id"]);+ } else if(result.contains("found") == 0 && result["found"].get<size_t>() == 0) {+ std::string analytics_query = Tokenizer::normalize_ascii_no_spaces(raw_query);+ AnalyticsManager::get_instance().add_noresults_query(orig_coll_name, analytics_query,+ true, req_params["x-typesense-user-id"]);}}diff --git a/src/noresults_queries.cpp b/src/noresults_queries.cppnew file mode 100644index 00000000..e337c17c--- /dev/null+++ b/src/noresults_queries.cpp@@ -0,0 +1,117 @@+#include "noresults_queries.h"+#include "logger.h"+#include <algorithm>+#include <mutex>+#include "string_utils.h"++NoresultsQueries::NoresultsQueries(size_t k) : k(k), max_size(k * 2) {+}++void NoresultsQueries::add(const std::string& key, const bool live_query, const std::string& user_id, uint64_t now_ts_us) {+ if(live_query) {+ // live query must be aggregated first to their final form as they could be prefix queries+ if(now_ts_us == 0) {+ now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();+ }++ if(!umutex.try_lock()) {+ // instead of locking we just skip incrementing keys during consolidation time+ return ;+ }++ auto& queries = user_prefix_queries[user_id];+ if(queries.size() < 100) {+ queries.emplace_back(key, now_ts_us);+ }++ umutex.unlock();++ } else {+ if(!lmutex.try_lock()) {+ // instead of locking we just skip incrementing keys during consolidation time+ return ;+ }++ auto it = local_counts.find(key);++ if(it != local_counts.end()) {+ it.value()++;+ } else if(local_counts.size() < max_size) {+ // skip count when map has become too large (to prevent abuse)+ local_counts.emplace(key, 1);+ }++ lmutex.unlock();+ }+}++void NoresultsQueries::serialize_as_docs(std::string& docs) {+ std::shared_lock lk(lmutex);++ std::string key_buffer;+ for(auto it = local_counts.begin(); it != local_counts.end(); ++it) {+ it.key(key_buffer);+ nlohmann::json doc;+ doc["id"] = std::to_string(StringUtils::hash_wy(key_buffer.c_str(), key_buffer.size()));+ doc["q"] = key_buffer;+ doc["$operations"]["increment"]["count"] = it.value();+ docs += doc.dump() + "\n";+ }++ if(!docs.empty()) {+ docs.pop_back();+ }+}++void NoresultsQueries::reset_local_counts() {+ std::unique_lock lk(lmutex);+ local_counts.clear();+}++size_t NoresultsQueries::get_k() {+ return k;+}++void NoresultsQueries::compact_user_queries(uint64_t now_ts_us) {+ std::unique_lock lk(umutex);+ std::vector<std::string> keys_to_delete;++ for(auto& kv: user_prefix_queries) {+ auto& queries = kv.second;+ int64_t last_consolidated_index = -1;+ for(uint32_t i = 0; i < queries.size(); i++) {+ if(now_ts_us - queries[i].timestamp < QUERY_FINALIZATION_INTERVAL_MICROS) {+ break;+ }++ uint64_t diff_micros = (i == queries.size()-1) ? (now_ts_us - queries[i].timestamp) :+ (queries[i + 1].timestamp - queries[i].timestamp);++ if(diff_micros > QUERY_FINALIZATION_INTERVAL_MICROS) {+ add(queries[i].query, false, "");+ last_consolidated_index = i;+ }+ }++ queries.erase(queries.begin(), queries.begin() + last_consolidated_index+1);++ if(queries.empty()) {+ keys_to_delete.push_back(kv.first);+ }+ }++ for(auto& key: keys_to_delete) {+ user_prefix_queries.erase(key);+ }+}++std::unordered_map<std::string, std::vector<NoresultsQueries::QWithTimestamp>> NoresultsQueries::get_user_prefix_queries() {+ std::unique_lock lk(umutex);+ return user_prefix_queries;+}++tsl::htrie_map<char, uint32_t> NoresultsQueries::get_local_counts() {+ std::unique_lock lk(lmutex);+ return local_counts;+}diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex 051ad579..26b4a20e 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -383,4 +383,61 @@ TEST_F(AnalyticsManagerTest, ClickEventsRateLimitTest) {//as rate limit is 5, adding one more event above that should trigger rate limitASSERT_FALSE(post_create_event(req, res));ASSERT_EQ("{\"message\": \"click event rate limit reached.\"}", res->body);+}++TEST_F(AnalyticsManagerTest, NoresultsQueries) {+ nlohmann::json titles_schema = R"({+ "name": "titles",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection* titles_coll = collectionManager.create_collection(titles_schema).get();++ nlohmann::json doc;+ doc["title"] = "Cool trousers";+ ASSERT_TRUE(titles_coll->add(doc.dump()).ok());++ nlohmann::json suggestions_schema = R"({+ "name": "top_queries",+ "fields": [+ {"name": "q", "type": "string" },+ {"name": "count", "type": "int32" }+ ]+ })"_json;++ Collection* suggestions_coll = collectionManager.create_collection(suggestions_schema).get();++ nlohmann::json analytics_rule = R"({+ "name": "search_queries",+ "type": "noresults_queries",+ "params": {+ "limit": 100,+ "source": {+ "collections": ["titles"]+ },+ "destination": {+ "collection": "top_queries"+ }+ }+ })"_json;++ auto create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_TRUE(create_op.ok());++ std::string q = "foobar";+ analyticsManager.add_noresults_query("titles", q, true, "1");++ auto noresults_queries = analyticsManager.get_noresults_queries();+ auto userQueries = noresults_queries["top_queries"]->get_user_prefix_queries()["1"];++ ASSERT_EQ(1, userQueries.size());+ ASSERT_EQ("foobar", userQueries[0].query);++ //try deleting noresults_queries rule+ ASSERT_TRUE(analyticsManager.remove_rule("search_queries").ok());++ noresults_queries = analyticsManager.get_noresults_queries();+ ASSERT_EQ(0, noresults_queries.size());}\ No newline at end of file