Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

noresults queries analytics

krunal81ae4f80697b
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index 41abe683..2aaad8f2 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -6,6 +6,7 @@
#include <string>
#include <unordered_map>
#include <shared_mutex>
+#include "noresults_queries.h"
struct ClickEvent {
std::string query;
@@ -93,6 +94,9 @@ private:
// suggestion collection => popular queries
std::unordered_map<std::string, PopularQueries*> popular_queries;
+ // suggestion collection => noresults queries
+ std::unordered_map<std::string, NoresultsQueries*> noresults_queries;
+
//query collection => click events
std::unordered_map<std::string, std::vector<ClickEvent>> query_collection_click_events;
@@ -103,17 +107,18 @@ private:
~AnalyticsManager();
- Option<bool> remove_popular_queries_index(const std::string& name);
+ Option<bool> remove_queries_index(const std::string& name);
- Option<bool> create_popular_queries_index(nlohmann::json &payload,
+ Option<bool> create_queries_index(nlohmann::json &payload,
bool upsert,
bool write_to_disk);
public:
static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";
- static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";
static constexpr const char* CLICK_EVENT = "$CE";
+ static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";
+ static constexpr const char* NORESULTS_QUERIES_TYPE = "noresults_queries";
static AnalyticsManager& get_instance() {
static AnalyticsManager instance;
@@ -155,5 +160,12 @@ public:
Option<bool> write_click_event_to_store(nlohmann::json& click_event_json);
+ void add_noresults_query(const std::string& query_collection,
+ const std::string& query, bool live_query, const std::string& user_id);
+
+ void persist_noresults_queries(ReplicationState *raft_server, uint64_t prev_persistence_s);
+
+ std::unordered_map<std::string, NoresultsQueries*> get_noresults_queries();
+
void resetRateLimit();
};
diff --git a/include/noresults_queries.h b/include/noresults_queries.h
new file mode 100644
index 00000000..d0ed902d
--- /dev/null
+++ b/include/noresults_queries.h
@@ -0,0 +1,51 @@
+#pragma once
+
+#include <string>
+#include <vector>
+#include <tsl/htrie_map.h>
+#include <json.hpp>
+#include <atomic>
+#include <shared_mutex>
+
+class NoresultsQueries {
+public:
+ struct QWithTimestamp {
+ std::string query;
+ uint64_t timestamp;
+
+ QWithTimestamp(const std::string& query, uint64_t timestamp) : query(query), timestamp(timestamp) {}
+ };
+
+ static const size_t QUERY_FINALIZATION_INTERVAL_MICROS = 4 * 1000 * 1000;
+
+private:
+
+ size_t k;
+ const size_t max_size;
+
+ // counts aggregated within the current node
+ tsl::htrie_map<char, uint32_t> local_counts;
+ std::shared_mutex lmutex;
+
+ std::unordered_map<std::string, std::vector<QWithTimestamp>> user_prefix_queries;
+ std::shared_mutex umutex;
+
+public:
+
+ NoresultsQueries(size_t k);
+
+ void add(const std::string& value, const bool live_query, const std::string& user_id,
+ uint64_t now_ts_us = 0);
+
+ void compact_user_queries(uint64_t now_ts_us);
+
+ void serialize_as_docs(std::string& docs);
+
+ void reset_local_counts();
+
+ size_t get_k();
+
+ std::unordered_map<std::string, std::vector<QWithTimestamp>> get_user_prefix_queries();
+
+ tsl::htrie_map<char, uint32_t> get_local_counts();
+};
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index 5cc95bf0..804e7c79 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -41,14 +41,14 @@ Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert,
return Option<bool>(400, "Bad or missing params.");
}
- if(payload["type"] == POPULAR_QUERIES_TYPE) {
- return create_popular_queries_index(payload, upsert, write_to_disk);
+ if(payload["type"] == POPULAR_QUERIES_TYPE || payload["type"] == NORESULTS_QUERIES_TYPE) {
+ return create_queries_index(payload, upsert, write_to_disk);
}
return Option<bool>(400, "Invalid type.");
}
-Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &payload, bool upsert, bool write_to_disk) {
+Option<bool> AnalyticsManager::create_queries_index(nlohmann::json &payload, bool upsert, bool write_to_disk) {
// params and name are validated upstream
const std::string& suggestion_config_name = payload["name"].get<std::string>();
bool already_exists = suggestion_configs.find(suggestion_config_name) != suggestion_configs.end();
@@ -88,8 +88,14 @@ Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &payl
suggestion_config.suggestion_collection = suggestion_collection;
suggestion_config.limit = limit;
- if(!upsert && popular_queries.count(suggestion_collection) != 0) {
- return Option<bool>(400, "There's already another configuration for this destination collection.");
+ if(payload["type"] == POPULAR_QUERIES_TYPE) {
+ if (!upsert && popular_queries.count(suggestion_collection) != 0) {
+ return Option<bool>(400, "There's already another configuration for this destination collection.");
+ }
+ } else if(payload["type"] == NORESULTS_QUERIES_TYPE) {
+ if (!upsert && noresults_queries.count(suggestion_collection) != 0) {
+ return Option<bool>(400, "There's already another configuration for this destination collection.");
+ }
}
for(const auto& coll: params["source"]["collections"]) {
@@ -105,7 +111,7 @@ Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &payl
if(already_exists) {
// remove the previous configuration with same name (upsert)
- Option<bool> remove_op = remove_popular_queries_index(suggestion_config_name);
+ Option<bool> remove_op = remove_queries_index(suggestion_config_name);
if(!remove_op.ok()) {
return Option<bool>(500, "Error erasing the existing configuration.");;
}
@@ -117,8 +123,13 @@ Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &payl
query_collection_mapping[query_coll].push_back(suggestion_collection);
}
- PopularQueries* popularQueries = new PopularQueries(limit);
- popular_queries.emplace(suggestion_collection, popularQueries);
+ if(payload["type"] == POPULAR_QUERIES_TYPE) {
+ PopularQueries *popularQueries = new PopularQueries(limit);
+ popular_queries.emplace(suggestion_collection, popularQueries);
+ } else if(payload["type"] == NORESULTS_QUERIES_TYPE) {
+ NoresultsQueries *noresultsQueries = new NoresultsQueries(limit);
+ noresults_queries.emplace(suggestion_collection, noresultsQueries);
+ }
if(write_to_disk) {
auto suggestion_key = std::string(ANALYTICS_RULE_PREFIX) + "_" + suggestion_config_name;
@@ -137,6 +148,10 @@ AnalyticsManager::~AnalyticsManager() {
for(auto& kv: popular_queries) {
delete kv.second;
}
+
+ for(auto& kv: noresults_queries) {
+ delete kv.second;
+ }
}
Option<nlohmann::json> AnalyticsManager::list_rules() {
@@ -154,7 +169,7 @@ Option<nlohmann::json> AnalyticsManager::list_rules() {
return Option<nlohmann::json>(rules);
}
-Option<nlohmann::json> AnalyticsManager::get_rule(const string& name) {
+Option<nlohmann::json> AnalyticsManager::get_rule(const std::string& name) {
nlohmann::json rule;
std::unique_lock lock(mutex);
@@ -167,18 +182,18 @@ Option<nlohmann::json> AnalyticsManager::get_rule(const string& name) {
return Option<nlohmann::json>(rule);
}
-Option<bool> AnalyticsManager::remove_rule(const string &name) {
+Option<bool> AnalyticsManager::remove_rule(const std::string &name) {
std::unique_lock lock(mutex);
auto suggestion_configs_it = suggestion_configs.find(name);
if(suggestion_configs_it != suggestion_configs.end()) {
- return remove_popular_queries_index(name);
+ return remove_queries_index(name);
}
return Option<bool>(404, "Rule not found.");
}
-Option<bool> AnalyticsManager::remove_popular_queries_index(const std::string &name) {
+Option<bool> AnalyticsManager::remove_queries_index(const std::string &name) {
// lock is held by caller
auto suggestion_configs_it = suggestion_configs.find(name);
@@ -197,6 +212,11 @@ Option<bool> AnalyticsManager::remove_popular_queries_index(const std::string &n
popular_queries.erase(suggestion_collection);
}
+ if(noresults_queries.count(suggestion_collection) != 0) {
+ delete noresults_queries[suggestion_collection];
+ noresults_queries.erase(suggestion_collection);
+ }
+
suggestion_configs.erase(name);
auto suggestion_key = std::string(ANALYTICS_RULE_PREFIX) + "_" + name;
@@ -263,6 +283,21 @@ Option<bool> AnalyticsManager::add_click_event(const std::string &query_collecti
return Option<bool>(true);
}
+void AnalyticsManager::add_noresults_query(const std::string &query_collection, const std::string &query,
+ bool live_query, const std::string &user_id) {
+ // look up suggestion collections for the query collection
+ std::unique_lock lock(mutex);
+ const auto& suggestion_collections_it = query_collection_mapping.find(query_collection);
+ if(suggestion_collections_it != query_collection_mapping.end()) {
+ for(const auto& suggestion_collection: suggestion_collections_it->second) {
+ const auto& noresults_queries_it = noresults_queries.find(suggestion_collection);
+ if(noresults_queries_it != noresults_queries.end()) {
+ noresults_queries_it->second->add(query, live_query, user_id);
+ }
+ }
+ }
+}
+
void AnalyticsManager::run(ReplicationState* raft_server) {
uint64_t prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
@@ -289,6 +324,7 @@ void AnalyticsManager::run(ReplicationState* raft_server) {
persist_suggestions(raft_server, prev_persistence_s);
persist_click_events(raft_server, prev_persistence_s);
+ persist_noresults_queries(raft_server, prev_persistence_s);
prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
@@ -404,6 +440,71 @@ void AnalyticsManager::persist_click_events(ReplicationState *raft_server, uint6
}
}
+void AnalyticsManager::persist_noresults_queries(ReplicationState *raft_server, uint64_t prev_persistence_s) {
+ // lock is held by caller
+ for(const auto& suggestion_config: suggestion_configs) {
+ const std::string& sink_name = suggestion_config.first;
+ const std::string& suggestion_coll = suggestion_config.second.suggestion_collection;
+
+ auto noresults_queries_it = noresults_queries.find(suggestion_coll);
+ if(noresults_queries_it == noresults_queries.end()) {
+ continue;
+ }
+
+ // need to prepare the counts as JSON docs for import into the suggestion collection
+ // {"id": "432432", "q": "foo", "$operations": {"increment": {"count": 100}}}
+
+ NoresultsQueries* noresultsQueries = noresults_queries_it->second;
+
+ // aggregate prefix queries to their final form
+ auto now = std::chrono::system_clock::now().time_since_epoch();
+ auto now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(now).count();
+ noresultsQueries->compact_user_queries(now_ts_us);
+
+ std::string import_payload;
+ noresultsQueries->serialize_as_docs(import_payload);
+
+ if(import_payload.empty()) {
+ continue;
+ }
+
+ // send http request
+ std::string leader_url = raft_server->get_leader_url();
+ if(!leader_url.empty()) {
+ const std::string& base_url = leader_url + "collections/" + suggestion_coll;
+ std::string res;
+
+ const std::string& update_url = base_url + "/documents/import?action=emplace";
+ std::map<std::string, std::string> res_headers;
+ long status_code = HttpClient::post_response(update_url, import_payload,
+ res, res_headers, {}, 10*1000, true);
+
+ if(status_code != 200) {
+ LOG(ERROR) << "Error while sending query suggestions events to leader. "
+ << "Status code: " << status_code << ", response: " << res;
+ } else {
+ LOG(INFO) << "Query aggregation for collection: " + suggestion_coll;
+ noresultsQueries->reset_local_counts();
+
+ if(raft_server->is_leader()) {
+ // try to run top-K compaction of suggestion collection
+ const std::string top_k_param = "count:" + std::to_string(noresultsQueries->get_k());
+ const std::string& truncate_topk_url = base_url + "/documents?top_k_by=" + top_k_param;
+ res.clear();
+ res_headers.clear();
+ status_code = HttpClient::delete_response(truncate_topk_url, res, res_headers, 10*1000, true);
+ if(status_code != 200) {
+ LOG(ERROR) << "Error while running top K for query suggestions collection. "
+ << "Status code: " << status_code << ", response: " << res;
+ } else {
+ LOG(INFO) << "Top K aggregation for collection: " + suggestion_coll;
+ }
+ }
+ }
+ }
+ }
+}
+
void AnalyticsManager::stop() {
quit = true;
cv.notify_all();
@@ -417,6 +518,12 @@ void AnalyticsManager::dispose() {
}
popular_queries.clear();
+
+ for(auto& kv: noresults_queries) {
+ delete kv.second;
+ }
+
+ noresults_queries.clear();
}
void AnalyticsManager::init(Store* store, Store* analytics_store) {
@@ -429,6 +536,11 @@ std::unordered_map<std::string, PopularQueries*> AnalyticsManager::get_popular_q
return popular_queries;
}
+std::unordered_map<std::string, NoresultsQueries*> AnalyticsManager::get_noresults_queries() {
+ std::unique_lock lk(mutex);
+ return noresults_queries;
+}
+
nlohmann::json AnalyticsManager::get_click_events() {
std::unique_lock lk(mutex);
std::vector<std::string> click_event_jsons;
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index cc2657aa..32001590 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1534,6 +1534,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
std::string analytics_query = Tokenizer::normalize_ascii_no_spaces(raw_query);
AnalyticsManager::get_instance().add_suggestion(orig_coll_name, analytics_query,
true, req_params["x-typesense-user-id"]);
+ } else if(result.contains("found") == 0 && result["found"].get<size_t>() == 0) {
+ std::string analytics_query = Tokenizer::normalize_ascii_no_spaces(raw_query);
+ AnalyticsManager::get_instance().add_noresults_query(orig_coll_name, analytics_query,
+ true, req_params["x-typesense-user-id"]);
}
}
diff --git a/src/noresults_queries.cpp b/src/noresults_queries.cpp
new file mode 100644
index 00000000..e337c17c
--- /dev/null
+++ b/src/noresults_queries.cpp
@@ -0,0 +1,117 @@
+#include "noresults_queries.h"
+#include "logger.h"
+#include <algorithm>
+#include <mutex>
+#include "string_utils.h"
+
+NoresultsQueries::NoresultsQueries(size_t k) : k(k), max_size(k * 2) {
+}
+
+void NoresultsQueries::add(const std::string& key, const bool live_query, const std::string& user_id, uint64_t now_ts_us) {
+ if(live_query) {
+ // live query must be aggregated first to their final form as they could be prefix queries
+ if(now_ts_us == 0) {
+ now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+ }
+
+ if(!umutex.try_lock()) {
+ // instead of locking we just skip incrementing keys during consolidation time
+ return ;
+ }
+
+ auto& queries = user_prefix_queries[user_id];
+ if(queries.size() < 100) {
+ queries.emplace_back(key, now_ts_us);
+ }
+
+ umutex.unlock();
+
+ } else {
+ if(!lmutex.try_lock()) {
+ // instead of locking we just skip incrementing keys during consolidation time
+ return ;
+ }
+
+ auto it = local_counts.find(key);
+
+ if(it != local_counts.end()) {
+ it.value()++;
+ } else if(local_counts.size() < max_size) {
+ // skip count when map has become too large (to prevent abuse)
+ local_counts.emplace(key, 1);
+ }
+
+ lmutex.unlock();
+ }
+}
+
+void NoresultsQueries::serialize_as_docs(std::string& docs) {
+ std::shared_lock lk(lmutex);
+
+ std::string key_buffer;
+ for(auto it = local_counts.begin(); it != local_counts.end(); ++it) {
+ it.key(key_buffer);
+ nlohmann::json doc;
+ doc["id"] = std::to_string(StringUtils::hash_wy(key_buffer.c_str(), key_buffer.size()));
+ doc["q"] = key_buffer;
+ doc["$operations"]["increment"]["count"] = it.value();
+ docs += doc.dump() + "\n";
+ }
+
+ if(!docs.empty()) {
+ docs.pop_back();
+ }
+}
+
+void NoresultsQueries::reset_local_counts() {
+ std::unique_lock lk(lmutex);
+ local_counts.clear();
+}
+
+size_t NoresultsQueries::get_k() {
+ return k;
+}
+
+void NoresultsQueries::compact_user_queries(uint64_t now_ts_us) {
+ std::unique_lock lk(umutex);
+ std::vector<std::string> keys_to_delete;
+
+ for(auto& kv: user_prefix_queries) {
+ auto& queries = kv.second;
+ int64_t last_consolidated_index = -1;
+ for(uint32_t i = 0; i < queries.size(); i++) {
+ if(now_ts_us - queries[i].timestamp < QUERY_FINALIZATION_INTERVAL_MICROS) {
+ break;
+ }
+
+ uint64_t diff_micros = (i == queries.size()-1) ? (now_ts_us - queries[i].timestamp) :
+ (queries[i + 1].timestamp - queries[i].timestamp);
+
+ if(diff_micros > QUERY_FINALIZATION_INTERVAL_MICROS) {
+ add(queries[i].query, false, "");
+ last_consolidated_index = i;
+ }
+ }
+
+ queries.erase(queries.begin(), queries.begin() + last_consolidated_index+1);
+
+ if(queries.empty()) {
+ keys_to_delete.push_back(kv.first);
+ }
+ }
+
+ for(auto& key: keys_to_delete) {
+ user_prefix_queries.erase(key);
+ }
+}
+
+std::unordered_map<std::string, std::vector<NoresultsQueries::QWithTimestamp>> NoresultsQueries::get_user_prefix_queries() {
+ std::unique_lock lk(umutex);
+ return user_prefix_queries;
+}
+
+tsl::htrie_map<char, uint32_t> NoresultsQueries::get_local_counts() {
+ std::unique_lock lk(lmutex);
+ return local_counts;
+}
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index 051ad579..26b4a20e 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -383,4 +383,61 @@ TEST_F(AnalyticsManagerTest, ClickEventsRateLimitTest) {
//as rate limit is 5, adding one more event above that should trigger rate limit
ASSERT_FALSE(post_create_event(req, res));
ASSERT_EQ("{\"message\": \"click event rate limit reached.\"}", res->body);
+}
+
+TEST_F(AnalyticsManagerTest, NoresultsQueries) {
+ nlohmann::json titles_schema = R"({
+ "name": "titles",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* titles_coll = collectionManager.create_collection(titles_schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "Cool trousers";
+ ASSERT_TRUE(titles_coll->add(doc.dump()).ok());
+
+ nlohmann::json suggestions_schema = R"({
+ "name": "top_queries",
+ "fields": [
+ {"name": "q", "type": "string" },
+ {"name": "count", "type": "int32" }
+ ]
+ })"_json;
+
+ Collection* suggestions_coll = collectionManager.create_collection(suggestions_schema).get();
+
+ nlohmann::json analytics_rule = R"({
+ "name": "search_queries",
+ "type": "noresults_queries",
+ "params": {
+ "limit": 100,
+ "source": {
+ "collections": ["titles"]
+ },
+ "destination": {
+ "collection": "top_queries"
+ }
+ }
+ })"_json;
+
+ auto create_op = analyticsManager.create_rule(analytics_rule, false, true);
+ ASSERT_TRUE(create_op.ok());
+
+ std::string q = "foobar";
+ analyticsManager.add_noresults_query("titles", q, true, "1");
+
+ auto noresults_queries = analyticsManager.get_noresults_queries();
+ auto userQueries = noresults_queries["top_queries"]->get_user_prefix_queries()["1"];
+
+ ASSERT_EQ(1, userQueries.size());
+ ASSERT_EQ("foobar", userQueries[0].query);
+
+ //try deleting noresults_queries rule
+ ASSERT_TRUE(analyticsManager.remove_rule("search_queries").ok());
+
+ noresults_queries = analyticsManager.get_noresults_queries();
+ ASSERT_EQ(0, noresults_queries.size());
}
\ No newline at end of file