Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
add flag to enable/disable auto-aggregation of search queries (#1865)
Krunal Gandhi631563d1c621
diff --git a/include/query_analytics.h b/include/query_analytics.hindex 390eadf0..c4f2c624 100644--- a/include/query_analytics.h+++ b/include/query_analytics.h@@ -25,7 +25,7 @@ private:const size_t max_query_length = 1024;bool expand_query = false;-+ bool auto_aggregation_enabled;// counts aggregated within the current nodetsl::htrie_map<char, uint32_t> local_counts;std::shared_mutex lmutex;@@ -35,7 +35,7 @@ private:public:- QueryAnalytics(size_t k);+ QueryAnalytics(size_t k, bool enable_auto_aggregation = true);void add(const std::string& value, const std::string& expanded_key,const bool live_query, const std::string& user_id, uint64_t now_ts_us = 0);@@ -53,4 +53,6 @@ public:tsl::htrie_map<char, uint32_t> get_local_counts();void set_expand_query(bool expand_query);++ bool is_auto_aggregation_enabled() const;};diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex f7dd64e9..39be9f6e 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -49,6 +49,7 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsertsize_t limit = 1000;bool expand_query = false;+ bool enable_auto_aggregation = true;if(params.contains("limit") && params["limit"].is_number_integer()) {limit = params["limit"].get<size_t>();@@ -88,6 +89,16 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert}}+ if((payload["type"] == POPULAR_QUERIES_TYPE || payload["type"] == NOHITS_QUERIES_TYPE)+ && (params.contains("enable_auto_aggregation"))) {++ if(!params["enable_auto_aggregation"].is_boolean()) {+ return Option<bool>(400, "enable_auto_aggregation should be boolean.");+ }++ enable_auto_aggregation = params["enable_auto_aggregation"];+ }+bool valid_events_found = params["source"].contains("events") && !params["source"]["events"].empty()&& params["source"]["events"].is_array()&& params["source"]["events"][0].is_object();@@ -162,11 +173,11 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsertbool log_to_store = payload["type"] == LOG_TYPE ? true : false;if(payload["type"] == POPULAR_QUERIES_TYPE) {- QueryAnalytics* popularQueries = new QueryAnalytics(limit);+ QueryAnalytics* popularQueries = new QueryAnalytics(limit, enable_auto_aggregation);popularQueries->set_expand_query(suggestion_config.expand_query);popular_queries.emplace(suggestion_collection, popularQueries);} else if(payload["type"] == NOHITS_QUERIES_TYPE) {- QueryAnalytics* noresultsQueries = new QueryAnalytics(limit);+ QueryAnalytics* noresultsQueries = new QueryAnalytics(limit, enable_auto_aggregation);nohits_queries.emplace(suggestion_collection, noresultsQueries);}@@ -360,7 +371,7 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection,if(suggestion_collections_it != query_collection_mapping.end()) {for(const auto& suggestion_collection: suggestion_collections_it->second) {const auto& popular_queries_it = popular_queries.find(suggestion_collection);- if(popular_queries_it != popular_queries.end()) {+ if(popular_queries_it != popular_queries.end() && popular_queries_it->second->is_auto_aggregation_enabled()) {popular_queries_it->second->add(query, expanded_query, live_query, user_id);}}@@ -485,7 +496,7 @@ void AnalyticsManager::add_nohits_query(const std::string &query_collection, conif(suggestion_collections_it != query_collection_mapping.end()) {for(const auto& suggestion_collection: suggestion_collections_it->second) {const auto& noresults_queries_it = nohits_queries.find(suggestion_collection);- if(noresults_queries_it != nohits_queries.end()) {+ if(noresults_queries_it != nohits_queries.end() && noresults_queries_it->second->is_auto_aggregation_enabled()) {noresults_queries_it->second->add(query, query, live_query, user_id);}}diff --git a/src/event_manager.cpp b/src/event_manager.cppindex 6760df68..056069d1 100644--- a/src/event_manager.cpp+++ b/src/event_manager.cpp@@ -23,32 +23,45 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::strif(event_type_val.is_string()) {const std::string& event_type = event_type_val.get<std::string>();if(event_type == AnalyticsManager::CLICK_EVENT || event_type == AnalyticsManager::CONVERSION_EVENT- || event_type == AnalyticsManager::VISIT_EVENT || event_type == AnalyticsManager::CUSTOM_EVENT- || event_type == AnalyticsManager::SEARCH_EVENT) {+ || event_type == AnalyticsManager::VISIT_EVENT || event_type == AnalyticsManager::CUSTOM_EVENT+ || event_type == AnalyticsManager::SEARCH_EVENT) {- if (!event.contains(EVENT_DATA)) {+ if(!event.contains(EVENT_DATA)) {return Option<bool>(404, "key `data` not found.");}- const auto &event_data_val = event[EVENT_DATA];+ const auto& event_data_val = event[EVENT_DATA];- if (!event.contains(EVENT_NAME)) {+ if(!event.contains(EVENT_NAME)) {return Option<bool>(404, "key `name` not found.");}- const auto &event_name = event[EVENT_NAME];- if (!event_data_val.is_object()) {+ const auto& event_name = event[EVENT_NAME];+ if(!event_data_val.is_object()) {return Option<bool>(500, "event_data_val is not object.");}- if(event_type != AnalyticsManager::CUSTOM_EVENT) {- if ((event_type == AnalyticsManager::SEARCH_EVENT) && (!event_data_val.contains("user_id") || !event_data_val.contains("q"))) {+ if(event_type == AnalyticsManager::SEARCH_EVENT) {+ if(!event_data_val.contains("user_id") || !event_data_val["user_id"].is_string()) {return Option<bool>(500,- "search event json data fields should contain `user_id` and 'q'.");+ "search event json data fields should contain `user_id` as string value.");}- if (!event_data_val.contains("doc_id") || !event_data_val["doc_id"].is_string()) {+ if(!event_data_val.contains("q") || !event_data_val["q"].is_string()) {+ return Option<bool>(500,+ "search event json data fields should contain `q` as string value.");+ }+ } else {+ if(!event_data_val.contains("doc_id") || !event_data_val["doc_id"].is_string()) {return Option<bool>(500, "event should have 'doc_id' as string value.");}++ if(event_data_val.contains("user_id") && !event_data_val["user_id"].is_string()) {+ return Option<bool>(500, "'user_id' should be a string value.");+ }++ if(event_data_val.contains("q") && !event_data_val["q"].is_string()) {+ return Option<bool>(500, "'q' should be a string value.");+ }}auto op = AnalyticsManager::get_instance().add_event(client_ip, event_type, event_name, event_data_val);diff --git a/src/query_analytics.cpp b/src/query_analytics.cppindex ac373943..409d1c41 100644--- a/src/query_analytics.cpp+++ b/src/query_analytics.cpp@@ -4,7 +4,8 @@#include <mutex>#include "string_utils.h"-QueryAnalytics::QueryAnalytics(size_t k) : k(k), max_size(k * 2) {+QueryAnalytics::QueryAnalytics(size_t k, bool enable_auto_aggregation)+ : k(k), max_size(k * 2), auto_aggregation_enabled(enable_auto_aggregation) {}@@ -123,3 +124,7 @@ tsl::htrie_map<char, uint32_t> QueryAnalytics::get_local_counts() {void QueryAnalytics::set_expand_query(bool expand_query) {this->expand_query = expand_query;}++bool QueryAnalytics::is_auto_aggregation_enabled() const {+ return auto_aggregation_enabled;+}\ No newline at end of filediff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex 30526542..ff652fa4 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -357,6 +357,34 @@ TEST_F(AnalyticsManagerTest, EventsValidation) {ASSERT_FALSE(post_create_event(req, res));ASSERT_EQ("{\"message\": \"event should have 'doc_id' as string value.\"}", res->body);+ event3 = R"({+ "type": "conversion",+ "name": "AP",+ "data": {+ "q": "technology",+ "doc_id": "21",+ "user_id": 12+ }+ })"_json;++ req->body = event3.dump();+ ASSERT_FALSE(post_create_event(req, res));+ ASSERT_EQ("{\"message\": \"'user_id' should be a string value.\"}", res->body);++ event3 = R"({+ "type": "conversion",+ "name": "AP",+ "data": {+ "q": 1245,+ "doc_id": "21",+ "user_id": "13"+ }+ })"_json;++ req->body = event3.dump();+ ASSERT_FALSE(post_create_event(req, res));+ ASSERT_EQ("{\"message\": \"'q' should be a string value.\"}", res->body);+//event name should be uniqueanalytics_rule = R"({"name": "product_click_events2",@@ -538,14 +566,23 @@ TEST_F(AnalyticsManagerTest, EventsValidation) {"type": "search","name": "NH1","data": {- "doc_id": "21",- "user_id": "11",- "expanded_query": "tech"+ "user_id": "11"+ }+ })"_json;+ req->body = event9.dump();+ ASSERT_FALSE(post_create_event(req, res));+ ASSERT_EQ("{\"message\": \"search event json data fields should contain `q` as string value.\"}", res->body);++ event9 = R"({+ "type": "search",+ "name": "NH1",+ "data": {+ "q": "11"}})"_json;req->body = event9.dump();ASSERT_FALSE(post_create_event(req, res));- ASSERT_EQ("{\"message\": \"search event json data fields should contain `user_id` and 'q'.\"}", res->body);+ ASSERT_EQ("{\"message\": \"search event json data fields should contain `user_id` as string value.\"}", res->body);//correct paramsevent9 = R"({@@ -553,9 +590,7 @@ TEST_F(AnalyticsManagerTest, EventsValidation) {"name": "NH1","data": {"q": "tech",- "doc_id": "21",- "user_id": "11",- "expanded_query": "technology"+ "user_id": "11"}})"_json;req->body = event9.dump();@@ -1942,8 +1977,6 @@ TEST_F(AnalyticsManagerTest, AddSuggestionByEvent) {nlohmann::json event_data;event_data["q"] = "coo";event_data["user_id"] = "1";- event_data["expanded_query"] = "cool";- event_data["live_query"] = true;analyticsManager.add_event("127.0.0.1", "search", "coll_search", event_data);@@ -1955,9 +1988,6 @@ TEST_F(AnalyticsManagerTest, AddSuggestionByEvent) {// add another query which is more popularevent_data["q"] = "buzzfoo";- event_data["expanded_query"] = "buzzfoo";- event_data["live_query"] = true;- event_data["user_id"] = "1";analyticsManager.add_event("127.0.0.1", "search", "coll_search", event_data);event_data["user_id"] = "2";@@ -1995,9 +2025,6 @@ TEST_F(AnalyticsManagerTest, AddSuggestionByEvent) {ASSERT_TRUE(create_op.ok());event_data["q"] = "foobar";- event_data["expanded_query"] = "foobar";- event_data["live_query"] = true;- event_data["user_id"] = "1";analyticsManager.add_event("127.0.0.1", "search", "nohits_search", event_data);auto noresults_queries = analyticsManager.get_nohits_queries();@@ -2036,4 +2063,109 @@ TEST_F(AnalyticsManagerTest, AddSuggestionByEvent) {create_op = analyticsManager.create_rule(analytics_rule, false, true);ASSERT_FALSE(create_op.ok());ASSERT_EQ("Events must contain a unique name.", create_op.error());+}++TEST_F(AnalyticsManagerTest, EventsOnlySearchTest) {+ nlohmann::json titles_schema = R"({+ "name": "titles",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection* titles_coll = collectionManager.create_collection(titles_schema).get();++ nlohmann::json doc;+ doc["title"] = "Cool trousers";+ ASSERT_TRUE(titles_coll->add(doc.dump()).ok());++ // create a collection to store suggestions+ nlohmann::json suggestions_schema = R"({+ "name": "top_queries",+ "fields": [+ {"name": "q", "type": "string" },+ {"name": "count", "type": "int32" }+ ]+ })"_json;++ Collection* suggestions_coll = collectionManager.create_collection(suggestions_schema).get();++ //enable_auto_aggregation flag enables query aggregation via events only+ nlohmann::json analytics_rule = R"({+ "name": "top_search_queries",+ "type": "popular_queries",+ "params": {+ "limit": 100,+ "enable_auto_aggregation": false,+ "source": {+ "collections": ["titles"],+ "events": [{"type": "search", "name": "coll_search"}]+ },+ "destination": {+ "collection": "top_queries"+ }+ }+ })"_json;++ auto create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_TRUE(create_op.ok());++ std::string q = "coo";+ analyticsManager.add_suggestion("titles", q, "cool", true, "1");++ auto popularQueries = analyticsManager.get_popular_queries();+ auto userQueries = popularQueries["top_queries"]->get_user_prefix_queries();+ ASSERT_EQ(0, userQueries.size());++ //try sending via events api+ nlohmann::json event_data;+ event_data["q"] = "coo";+ event_data["user_id"] = "1";++ analyticsManager.add_event("127.0.0.1", "search", "coll_search", event_data);++ popularQueries = analyticsManager.get_popular_queries();+ auto localCounts = popularQueries["top_queries"]->get_local_counts();+ ASSERT_EQ(1, localCounts.size());+ ASSERT_EQ(1, localCounts.count("coo"));+ ASSERT_EQ(1, localCounts["coo"]);++ //try with nohits analytic rule+ analytics_rule = R"({+ "name": "noresults_queries",+ "type": "nohits_queries",+ "params": {+ "limit": 100,+ "enable_auto_aggregation": false,+ "source": {+ "collections": ["titles"],+ "events": [{"type": "search", "name": "nohits_search"}]+ },+ "destination": {+ "collection": "top_queries"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_TRUE(create_op.ok());++ q = "foobar";+ analyticsManager.add_nohits_query("titles", q, true, "1");++ auto noresults_queries = analyticsManager.get_nohits_queries();+ userQueries = noresults_queries["top_queries"]->get_user_prefix_queries();++ ASSERT_EQ(0, userQueries.size());++ //send events for same+ event_data["q"] = "foobar";+ analyticsManager.add_event("127.0.0.1", "search", "nohits_search", event_data);++ noresults_queries = analyticsManager.get_nohits_queries();+ localCounts = noresults_queries["top_queries"]->get_local_counts();++ ASSERT_EQ(1, localCounts.size());+ ASSERT_EQ(1, localCounts.count("foobar"));+ ASSERT_EQ(1, localCounts["foobar"]);}\ No newline at end of file