Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

add flag to enable/disable auto-aggregation of search queries (#1865)

Krunal Gandhi631563d1c621
diff --git a/include/query_analytics.h b/include/query_analytics.h
index 390eadf0..c4f2c624 100644
--- a/include/query_analytics.h
+++ b/include/query_analytics.h
@@ -25,7 +25,7 @@ private:
const size_t max_query_length = 1024;
bool expand_query = false;
-
+ bool auto_aggregation_enabled;
// counts aggregated within the current node
tsl::htrie_map<char, uint32_t> local_counts;
std::shared_mutex lmutex;
@@ -35,7 +35,7 @@ private:
public:
- QueryAnalytics(size_t k);
+ QueryAnalytics(size_t k, bool enable_auto_aggregation = true);
void add(const std::string& value, const std::string& expanded_key,
const bool live_query, const std::string& user_id, uint64_t now_ts_us = 0);
@@ -53,4 +53,6 @@ public:
tsl::htrie_map<char, uint32_t> get_local_counts();
void set_expand_query(bool expand_query);
+
+ bool is_auto_aggregation_enabled() const;
};
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index f7dd64e9..39be9f6e 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -49,6 +49,7 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert
size_t limit = 1000;
bool expand_query = false;
+ bool enable_auto_aggregation = true;
if(params.contains("limit") && params["limit"].is_number_integer()) {
limit = params["limit"].get<size_t>();
@@ -88,6 +89,16 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert
}
}
+ if((payload["type"] == POPULAR_QUERIES_TYPE || payload["type"] == NOHITS_QUERIES_TYPE)
+ && (params.contains("enable_auto_aggregation"))) {
+
+ if(!params["enable_auto_aggregation"].is_boolean()) {
+ return Option<bool>(400, "enable_auto_aggregation should be boolean.");
+ }
+
+ enable_auto_aggregation = params["enable_auto_aggregation"];
+ }
+
bool valid_events_found = params["source"].contains("events") && !params["source"]["events"].empty()
&& params["source"]["events"].is_array()
&& params["source"]["events"][0].is_object();
@@ -162,11 +173,11 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert
bool log_to_store = payload["type"] == LOG_TYPE ? true : false;
if(payload["type"] == POPULAR_QUERIES_TYPE) {
- QueryAnalytics* popularQueries = new QueryAnalytics(limit);
+ QueryAnalytics* popularQueries = new QueryAnalytics(limit, enable_auto_aggregation);
popularQueries->set_expand_query(suggestion_config.expand_query);
popular_queries.emplace(suggestion_collection, popularQueries);
} else if(payload["type"] == NOHITS_QUERIES_TYPE) {
- QueryAnalytics* noresultsQueries = new QueryAnalytics(limit);
+ QueryAnalytics* noresultsQueries = new QueryAnalytics(limit, enable_auto_aggregation);
nohits_queries.emplace(suggestion_collection, noresultsQueries);
}
@@ -360,7 +371,7 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection,
if(suggestion_collections_it != query_collection_mapping.end()) {
for(const auto& suggestion_collection: suggestion_collections_it->second) {
const auto& popular_queries_it = popular_queries.find(suggestion_collection);
- if(popular_queries_it != popular_queries.end()) {
+ if(popular_queries_it != popular_queries.end() && popular_queries_it->second->is_auto_aggregation_enabled()) {
popular_queries_it->second->add(query, expanded_query, live_query, user_id);
}
}
@@ -485,7 +496,7 @@ void AnalyticsManager::add_nohits_query(const std::string &query_collection, con
if(suggestion_collections_it != query_collection_mapping.end()) {
for(const auto& suggestion_collection: suggestion_collections_it->second) {
const auto& noresults_queries_it = nohits_queries.find(suggestion_collection);
- if(noresults_queries_it != nohits_queries.end()) {
+ if(noresults_queries_it != nohits_queries.end() && noresults_queries_it->second->is_auto_aggregation_enabled()) {
noresults_queries_it->second->add(query, query, live_query, user_id);
}
}
diff --git a/src/event_manager.cpp b/src/event_manager.cpp
index 6760df68..056069d1 100644
--- a/src/event_manager.cpp
+++ b/src/event_manager.cpp
@@ -23,32 +23,45 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::str
if(event_type_val.is_string()) {
const std::string& event_type = event_type_val.get<std::string>();
if(event_type == AnalyticsManager::CLICK_EVENT || event_type == AnalyticsManager::CONVERSION_EVENT
- || event_type == AnalyticsManager::VISIT_EVENT || event_type == AnalyticsManager::CUSTOM_EVENT
- || event_type == AnalyticsManager::SEARCH_EVENT) {
+ || event_type == AnalyticsManager::VISIT_EVENT || event_type == AnalyticsManager::CUSTOM_EVENT
+ || event_type == AnalyticsManager::SEARCH_EVENT) {
- if (!event.contains(EVENT_DATA)) {
+ if(!event.contains(EVENT_DATA)) {
return Option<bool>(404, "key `data` not found.");
}
- const auto &event_data_val = event[EVENT_DATA];
+ const auto& event_data_val = event[EVENT_DATA];
- if (!event.contains(EVENT_NAME)) {
+ if(!event.contains(EVENT_NAME)) {
return Option<bool>(404, "key `name` not found.");
}
- const auto &event_name = event[EVENT_NAME];
- if (!event_data_val.is_object()) {
+ const auto& event_name = event[EVENT_NAME];
+ if(!event_data_val.is_object()) {
return Option<bool>(500, "event_data_val is not object.");
}
- if(event_type != AnalyticsManager::CUSTOM_EVENT) {
- if ((event_type == AnalyticsManager::SEARCH_EVENT) && (!event_data_val.contains("user_id") || !event_data_val.contains("q"))) {
+ if(event_type == AnalyticsManager::SEARCH_EVENT) {
+ if(!event_data_val.contains("user_id") || !event_data_val["user_id"].is_string()) {
return Option<bool>(500,
- "search event json data fields should contain `user_id` and 'q'.");
+ "search event json data fields should contain `user_id` as string value.");
}
- if (!event_data_val.contains("doc_id") || !event_data_val["doc_id"].is_string()) {
+ if(!event_data_val.contains("q") || !event_data_val["q"].is_string()) {
+ return Option<bool>(500,
+ "search event json data fields should contain `q` as string value.");
+ }
+ } else {
+ if(!event_data_val.contains("doc_id") || !event_data_val["doc_id"].is_string()) {
return Option<bool>(500, "event should have 'doc_id' as string value.");
}
+
+ if(event_data_val.contains("user_id") && !event_data_val["user_id"].is_string()) {
+ return Option<bool>(500, "'user_id' should be a string value.");
+ }
+
+ if(event_data_val.contains("q") && !event_data_val["q"].is_string()) {
+ return Option<bool>(500, "'q' should be a string value.");
+ }
}
auto op = AnalyticsManager::get_instance().add_event(client_ip, event_type, event_name, event_data_val);
diff --git a/src/query_analytics.cpp b/src/query_analytics.cpp
index ac373943..409d1c41 100644
--- a/src/query_analytics.cpp
+++ b/src/query_analytics.cpp
@@ -4,7 +4,8 @@
#include <mutex>
#include "string_utils.h"
-QueryAnalytics::QueryAnalytics(size_t k) : k(k), max_size(k * 2) {
+QueryAnalytics::QueryAnalytics(size_t k, bool enable_auto_aggregation)
+ : k(k), max_size(k * 2), auto_aggregation_enabled(enable_auto_aggregation) {
}
@@ -123,3 +124,7 @@ tsl::htrie_map<char, uint32_t> QueryAnalytics::get_local_counts() {
void QueryAnalytics::set_expand_query(bool expand_query) {
this->expand_query = expand_query;
}
+
+bool QueryAnalytics::is_auto_aggregation_enabled() const {
+ return auto_aggregation_enabled;
+}
\ No newline at end of file
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index 30526542..ff652fa4 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -357,6 +357,34 @@ TEST_F(AnalyticsManagerTest, EventsValidation) {
ASSERT_FALSE(post_create_event(req, res));
ASSERT_EQ("{\"message\": \"event should have 'doc_id' as string value.\"}", res->body);
+ event3 = R"({
+ "type": "conversion",
+ "name": "AP",
+ "data": {
+ "q": "technology",
+ "doc_id": "21",
+ "user_id": 12
+ }
+ })"_json;
+
+ req->body = event3.dump();
+ ASSERT_FALSE(post_create_event(req, res));
+ ASSERT_EQ("{\"message\": \"'user_id' should be a string value.\"}", res->body);
+
+ event3 = R"({
+ "type": "conversion",
+ "name": "AP",
+ "data": {
+ "q": 1245,
+ "doc_id": "21",
+ "user_id": "13"
+ }
+ })"_json;
+
+ req->body = event3.dump();
+ ASSERT_FALSE(post_create_event(req, res));
+ ASSERT_EQ("{\"message\": \"'q' should be a string value.\"}", res->body);
+
//event name should be unique
analytics_rule = R"({
"name": "product_click_events2",
@@ -538,14 +566,23 @@ TEST_F(AnalyticsManagerTest, EventsValidation) {
"type": "search",
"name": "NH1",
"data": {
- "doc_id": "21",
- "user_id": "11",
- "expanded_query": "tech"
+ "user_id": "11"
+ }
+ })"_json;
+ req->body = event9.dump();
+ ASSERT_FALSE(post_create_event(req, res));
+ ASSERT_EQ("{\"message\": \"search event json data fields should contain `q` as string value.\"}", res->body);
+
+ event9 = R"({
+ "type": "search",
+ "name": "NH1",
+ "data": {
+ "q": "11"
}
})"_json;
req->body = event9.dump();
ASSERT_FALSE(post_create_event(req, res));
- ASSERT_EQ("{\"message\": \"search event json data fields should contain `user_id` and 'q'.\"}", res->body);
+ ASSERT_EQ("{\"message\": \"search event json data fields should contain `user_id` as string value.\"}", res->body);
//correct params
event9 = R"({
@@ -553,9 +590,7 @@ TEST_F(AnalyticsManagerTest, EventsValidation) {
"name": "NH1",
"data": {
"q": "tech",
- "doc_id": "21",
- "user_id": "11",
- "expanded_query": "technology"
+ "user_id": "11"
}
})"_json;
req->body = event9.dump();
@@ -1942,8 +1977,6 @@ TEST_F(AnalyticsManagerTest, AddSuggestionByEvent) {
nlohmann::json event_data;
event_data["q"] = "coo";
event_data["user_id"] = "1";
- event_data["expanded_query"] = "cool";
- event_data["live_query"] = true;
analyticsManager.add_event("127.0.0.1", "search", "coll_search", event_data);
@@ -1955,9 +1988,6 @@ TEST_F(AnalyticsManagerTest, AddSuggestionByEvent) {
// add another query which is more popular
event_data["q"] = "buzzfoo";
- event_data["expanded_query"] = "buzzfoo";
- event_data["live_query"] = true;
- event_data["user_id"] = "1";
analyticsManager.add_event("127.0.0.1", "search", "coll_search", event_data);
event_data["user_id"] = "2";
@@ -1995,9 +2025,6 @@ TEST_F(AnalyticsManagerTest, AddSuggestionByEvent) {
ASSERT_TRUE(create_op.ok());
event_data["q"] = "foobar";
- event_data["expanded_query"] = "foobar";
- event_data["live_query"] = true;
- event_data["user_id"] = "1";
analyticsManager.add_event("127.0.0.1", "search", "nohits_search", event_data);
auto noresults_queries = analyticsManager.get_nohits_queries();
@@ -2036,4 +2063,109 @@ TEST_F(AnalyticsManagerTest, AddSuggestionByEvent) {
create_op = analyticsManager.create_rule(analytics_rule, false, true);
ASSERT_FALSE(create_op.ok());
ASSERT_EQ("Events must contain a unique name.", create_op.error());
+}
+
+TEST_F(AnalyticsManagerTest, EventsOnlySearchTest) {
+ nlohmann::json titles_schema = R"({
+ "name": "titles",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* titles_coll = collectionManager.create_collection(titles_schema).get();
+
+ nlohmann::json doc;
+ doc["title"] = "Cool trousers";
+ ASSERT_TRUE(titles_coll->add(doc.dump()).ok());
+
+ // create a collection to store suggestions
+ nlohmann::json suggestions_schema = R"({
+ "name": "top_queries",
+ "fields": [
+ {"name": "q", "type": "string" },
+ {"name": "count", "type": "int32" }
+ ]
+ })"_json;
+
+ Collection* suggestions_coll = collectionManager.create_collection(suggestions_schema).get();
+
+ //enable_auto_aggregation flag enables query aggregation via events only
+ nlohmann::json analytics_rule = R"({
+ "name": "top_search_queries",
+ "type": "popular_queries",
+ "params": {
+ "limit": 100,
+ "enable_auto_aggregation": false,
+ "source": {
+ "collections": ["titles"],
+ "events": [{"type": "search", "name": "coll_search"}]
+ },
+ "destination": {
+ "collection": "top_queries"
+ }
+ }
+ })"_json;
+
+ auto create_op = analyticsManager.create_rule(analytics_rule, false, true);
+ ASSERT_TRUE(create_op.ok());
+
+ std::string q = "coo";
+ analyticsManager.add_suggestion("titles", q, "cool", true, "1");
+
+ auto popularQueries = analyticsManager.get_popular_queries();
+ auto userQueries = popularQueries["top_queries"]->get_user_prefix_queries();
+ ASSERT_EQ(0, userQueries.size());
+
+ //try sending via events api
+ nlohmann::json event_data;
+ event_data["q"] = "coo";
+ event_data["user_id"] = "1";
+
+ analyticsManager.add_event("127.0.0.1", "search", "coll_search", event_data);
+
+ popularQueries = analyticsManager.get_popular_queries();
+ auto localCounts = popularQueries["top_queries"]->get_local_counts();
+ ASSERT_EQ(1, localCounts.size());
+ ASSERT_EQ(1, localCounts.count("coo"));
+ ASSERT_EQ(1, localCounts["coo"]);
+
+ //try with nohits analytic rule
+ analytics_rule = R"({
+ "name": "noresults_queries",
+ "type": "nohits_queries",
+ "params": {
+ "limit": 100,
+ "enable_auto_aggregation": false,
+ "source": {
+ "collections": ["titles"],
+ "events": [{"type": "search", "name": "nohits_search"}]
+ },
+ "destination": {
+ "collection": "top_queries"
+ }
+ }
+ })"_json;
+
+ create_op = analyticsManager.create_rule(analytics_rule, false, true);
+ ASSERT_TRUE(create_op.ok());
+
+ q = "foobar";
+ analyticsManager.add_nohits_query("titles", q, true, "1");
+
+ auto noresults_queries = analyticsManager.get_nohits_queries();
+ userQueries = noresults_queries["top_queries"]->get_user_prefix_queries();
+
+ ASSERT_EQ(0, userQueries.size());
+
+ //send events for same
+ event_data["q"] = "foobar";
+ analyticsManager.add_event("127.0.0.1", "search", "nohits_search", event_data);
+
+ noresults_queries = analyticsManager.get_nohits_queries();
+ localCounts = noresults_queries["top_queries"]->get_local_counts();
+
+ ASSERT_EQ(1, localCounts.size());
+ ASSERT_EQ(1, localCounts.count("foobar"));
+ ASSERT_EQ(1, localCounts["foobar"]);
}
\ No newline at end of file