Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Event analytics revised (#1522)
Krunal Gandhi206cecffbd42
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 85e30e2a..427e6609 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -8,25 +8,33 @@#include <shared_mutex>#include "lru/lru.hpp"+struct event_type_collection {+ std::string event_type;+ std::string collection;+};+struct event_t {std::string query;std::string event_type;uint64_t timestamp;std::string user_id;std::string doc_id;- uint64_t position;+ std::string name;+ std::vector<std::pair<std::string, std::string>> data;event_t() = delete;~event_t() = default;- event_t(std::string q, std::string type, uint64_t ts, std::string uid, std::string id, uint64_t pos) {+ event_t(const std::string& q, const std::string& type, uint64_t ts, const std::string& uid, const std::string& id,+ const std::string& event_name, const std::vector<std::pair<std::string, std::string>> datavec) {query = q;event_type = type;timestamp = ts;user_id = uid;doc_id = id;- position = pos;+ name = event_name;+ data = datavec;}event_t& operator=(event_t& other) {@@ -36,18 +44,25 @@ struct event_t {timestamp = other.timestamp;user_id = other.user_id;doc_id = other.doc_id;- position = other.position;+ name = other.name;+ data = other.data;return *this;}}void to_json(nlohmann::json& obj) const {obj["query"] = query;- obj["event_type"] = event_type;+ obj["type"] = event_type;obj["timestamp"] = timestamp;obj["user_id"] = user_id;obj["doc_id"] = doc_id;- obj["position"] = position;+ obj["name"] = name;++ if(event_type == "custom") {+ for(const auto& kv : data) {+ obj[kv.first] = kv.second;+ }+ }}};@@ -119,6 +134,9 @@ private://query collection => eventsstd::unordered_map<std::string, std::vector<event_t>> query_collection_events;+ //event_name => collection+ std::unordered_map<std::string, event_type_collection> event_collection_map;+// per_ip cache for rate limitingLRU::Cache<std::string, event_cache_t> events_cache;@@ -137,14 +155,22 @@ private:bool upsert,bool write_to_disk);+ std::string get_sub_event_type(const std::string& event_type);+public:static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";static constexpr const char* NOHITS_QUERIES_TYPE = "nohits_queries";static constexpr const char* COUNTER_TYPE = "counter";- static constexpr const char* QUERY_CLICK = "query_click";- static constexpr const char* QUERY_PURCHASE = "query_purchase";+ static constexpr const char* CLICKS_TYPE = "clicks";+ static constexpr const char* CONVERSIONS_TYPE = "conversions";+ static constexpr const char* VISITS_TYPE = "visits";+ static constexpr const char* CUSTOM_EVENTS_TYPE = "custom_events";+ static constexpr const char* CLICK_EVENT = "click";+ static constexpr const char* CONVERSION_EVENT = "conversion";+ static constexpr const char* VISIT_EVENT = "visit";+ static constexpr const char* CUSTOM_EVENT = "custom";static AnalyticsManager& get_instance() {static AnalyticsManager instance;@@ -166,6 +192,8 @@ public:Option<bool> remove_rule(const std::string& name);+ Option<bool> remove_all_rules();+void add_suggestion(const std::string& query_collection,const std::string& query, const std::string& expanded_query,bool live_query, const std::string& user_id);@@ -178,8 +206,8 @@ public:std::unordered_map<std::string, QueryAnalytics*> get_popular_queries();- Option<bool> add_event(const std::string& event_type, const std::string& query_collection, const std::string& query, const std::string& user_id,- std::string doc_id, uint64_t position, const std::string& client_ip);+ Option<bool> add_event(const std::string& client_ip, const std::string& event_type,+ const std::string& event_name, const nlohmann::json& event_data);void persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s);diff --git a/include/event_manager.h b/include/event_manager.hindex 34793690..665bf404 100644--- a/include/event_manager.h+++ b/include/event_manager.h@@ -11,6 +11,7 @@ private:static constexpr char* EVENT_TYPE = "type";static constexpr char* EVENT_DATA = "data";+ static constexpr char* EVENT_NAME = "name";public:static EventManager& get_instance() {diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex f2581191..7b952041 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -41,7 +41,8 @@ Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert,}if(payload["type"] == POPULAR_QUERIES_TYPE || payload["type"] == NOHITS_QUERIES_TYPE- || payload["type"] == COUNTER_TYPE) {+ || payload["type"] == COUNTER_TYPE || payload["type"] == CLICKS_TYPE || payload["type"] == CONVERSIONS_TYPE+ || payload["type"] == VISITS_TYPE || payload["type"] == CUSTOM_EVENTS_TYPE) {return create_index(payload, upsert, write_to_disk);}@@ -52,6 +53,8 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert// params and name are validated upstreamconst std::string& suggestion_config_name = payload["name"].get<std::string>();bool already_exists = suggestion_configs.find(suggestion_config_name) != suggestion_configs.end();+ bool is_event_type = (payload["type"] == CLICKS_TYPE || payload["type"] == CONVERSIONS_TYPE+ || payload["type"] == VISITS_TYPE || payload["type"] == CUSTOM_EVENTS_TYPE);if(!upsert && already_exists) {return Option<bool>(400, "There's already another configuration with the name `" +@@ -64,19 +67,6 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsertreturn Option<bool>(400, "Bad or missing source.");}- if(!params.contains("destination") || !params["destination"].is_object()) {- return Option<bool>(400, "Bad or missing destination.");- }-- if(payload["type"] == COUNTER_TYPE) {- if (!params["source"].contains("events") || (params["source"].contains("events") &&- (params["source"]["events"].empty()- || !params["source"]["events"].is_array()- || !params["source"]["events"][0].is_object()))) {- return Option<bool>(400, "Bad or missing events.");- }- }-size_t limit = 1000;bool expand_query = false;@@ -88,61 +78,96 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsertexpand_query = params["expand_query"].get<bool>();}- if(!params["source"].contains("collections") || !params["source"]["collections"].is_array()) {- return Option<bool>(400, "Must contain a valid list of source collections.");- }-- if(!params["destination"].contains("collection") || !params["destination"]["collection"].is_string()) {- return Option<bool>(400, "Must contain a valid destination collection.");- }-std::string counter_field;+ std::string suggestion_collection;- if(params["destination"].contains("counter_field")) {- if (!params["destination"]["counter_field"].is_string()) {- return Option<bool>(400, "Must contain a valid counter_field.");- }- counter_field = params["destination"]["counter_field"].get<std::string>();- }-- const std::string& suggestion_collection = params["destination"]["collection"].get<std::string>();suggestion_config_t suggestion_config;suggestion_config.name = suggestion_config_name;- suggestion_config.suggestion_collection = suggestion_collection;suggestion_config.limit = limit;suggestion_config.expand_query = expand_query;suggestion_config.rule_type = payload["type"];- if(payload["type"] == POPULAR_QUERIES_TYPE) {- if (!upsert && popular_queries.count(suggestion_collection) != 0) {- return Option<bool>(400, "There's already another configuration for this destination collection.");+ if(is_event_type) {+ if (!params.contains("name") || params["name"].empty()) {+ return Option<bool>(400, "Bad or missing name in params");}- } else if(payload["type"] == NOHITS_QUERIES_TYPE) {- if (!upsert && nohits_queries.count(suggestion_collection) != 0) {- return Option<bool>(400, "There's already another configuration for this destination collection.");++ if (!params["source"].contains("collection") || !params["source"]["collection"].is_string()) {+ return Option<bool>(400, "Must contain a valid source collection.");}- } else if(payload["type"] == COUNTER_TYPE) {- if (!upsert && counter_events.count(suggestion_collection) != 0) {- return Option<bool>(400, "There's already another configuration for this destination collection.");++ suggestion_collection = params["source"]["collection"].get<std::string>();+ suggestion_config.suggestion_collection = suggestion_collection;++ if (event_collection_map.count(params["name"]) != 0) {+ return Option<bool>(400, "Event name already exists.");}+ } else {+ if (payload["type"] == COUNTER_TYPE) {+ if (!params["source"].contains("events") || (params["source"].contains("events") &&+ (params["source"]["events"].empty()+ || !params["source"]["events"].is_array()+ || !params["source"]["events"][0].is_object()))) {+ return Option<bool>(400, "Bad or missing events.");+ }+ }++ if (!params.contains("destination") || !params["destination"].is_object()) {+ return Option<bool>(400, "Bad or missing destination.");+ }++ if (!params["source"].contains("collections") || !params["source"]["collections"].is_array()) {+ return Option<bool>(400, "Must contain a valid list of source collections.");+ }++ if (!params["destination"].contains("collection") || !params["destination"]["collection"].is_string()) {+ return Option<bool>(400, "Must contain a valid destination collection.");+ }+- auto coll = CollectionManager::get_instance().get_collection(suggestion_collection).get();- if(coll != nullptr) {- if (!coll->contains_field(counter_field)) {- return Option<bool>(404, "counter_field `" + counter_field + "` not found in destination collection.");+ if (params["destination"].contains("counter_field")) {+ if (!params["destination"]["counter_field"].is_string()) {+ return Option<bool>(400, "Must contain a valid counter_field.");}- } else {- return Option<bool>(404, "Collection `" + suggestion_collection + "` not found.");+ counter_field = params["destination"]["counter_field"].get<std::string>();}- }- for(const auto& coll: params["source"]["collections"]) {- if(!coll.is_string()) {- return Option<bool>(400, "Must contain a valid list of source collection names.");+ for (const auto &coll: params["source"]["collections"]) {+ if (!coll.is_string()) {+ return Option<bool>(400, "Must contain a valid list of source collection names.");+ }++ const std::string &src_collection = coll.get<std::string>();+ suggestion_config.query_collections.push_back(src_collection);}- const std::string& src_collection = coll.get<std::string>();- suggestion_config.query_collections.push_back(src_collection);+ suggestion_collection = params["destination"]["collection"].get<std::string>();+ suggestion_config.suggestion_collection = suggestion_collection;+++ if (payload["type"] == POPULAR_QUERIES_TYPE) {+ if (!upsert && popular_queries.count(suggestion_collection) != 0) {+ return Option<bool>(400, "There's already another configuration for this destination collection.");+ }+ } else if (payload["type"] == NOHITS_QUERIES_TYPE) {+ if (!upsert && nohits_queries.count(suggestion_collection) != 0) {+ return Option<bool>(400, "There's already another configuration for this destination collection.");+ }+ } else if (payload["type"] == COUNTER_TYPE) {+ if (!upsert && counter_events.count(suggestion_collection) != 0) {+ return Option<bool>(400, "There's already another configuration for this destination collection.");+ }++ auto coll = CollectionManager::get_instance().get_collection(suggestion_collection).get();+ if (coll != nullptr) {+ if (!coll->contains_field(counter_field)) {+ return Option<bool>(404,+ "counter_field `" + counter_field + "` not found in destination collection.");+ }+ } else {+ return Option<bool>(404, "Collection `" + suggestion_collection + "` not found.");+ }+ }}std::unique_lock lock(mutex);@@ -174,6 +199,16 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsertevent_weight_map[event["type"]] = event["weight"];}counter_events.emplace(suggestion_collection, counter_event_t{counter_field, {}, event_weight_map});+ } else if(is_event_type) {+ //multiple event types can be used with one collection+ //hence if no events are created for collection then only initialize it+ if(query_collection_events.count(suggestion_collection) == 0) {+ std::vector<event_t> vec;+ query_collection_events.emplace(suggestion_collection, vec);+ }+ auto sub_event_type = get_sub_event_type(payload["type"]);+ event_type_collection ec {sub_event_type, suggestion_collection};+ event_collection_map.emplace(params["name"], ec);}if(write_to_disk) {@@ -238,6 +273,16 @@ Option<bool> AnalyticsManager::remove_rule(const std::string &name) {return Option<bool>(404, "Rule not found.");}+Option<bool> AnalyticsManager::remove_all_rules() {+ std::unique_lock lock(mutex);++ for(auto suggestion_config : suggestion_configs) {+ remove_index(suggestion_config.first);+ }++ return Option<bool>(true);+}+Option<bool> AnalyticsManager::remove_index(const std::string &name) {// lock is held by callerauto suggestion_configs_it = suggestion_configs.find(name);@@ -266,6 +311,10 @@ Option<bool> AnalyticsManager::remove_index(const std::string &name) {counter_events.erase(suggestion_collection);}+ if(query_collection_events.count(suggestion_collection) != 0) {+ query_collection_events.erase(suggestion_collection);+ }+suggestion_configs.erase(name);auto suggestion_key = std::string(ANALYTICS_RULE_PREFIX) + "_" + name;@@ -293,17 +342,28 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection,}}-Option<bool> AnalyticsManager::add_event(const std::string& event_type, const std::string &query_collection, const std::string &query, const std::string &user_id,- std::string doc_id, uint64_t position, const std::string& client_ip) {+Option<bool> AnalyticsManager::add_event(const std::string& client_ip, const std::string& event_type,+ const std::string& event_name, const nlohmann::json& event_json) {std::unique_lock lock(mutex);if(!analytics_logs.is_open()) {return Option<bool>(true);}+ const auto event_collection_map_it = event_collection_map.find(event_name);+ if(event_collection_map_it == event_collection_map.end()) {+ return Option<bool>(404, "No analytics rule defined for event name" + event_name);+ }++ if(event_collection_map_it->second.event_type != event_type) {+ return Option<bool>(400, "event_type mismatch in analytic rules.");+ }- auto &events_vec= query_collection_events[query_collection];+ const auto& query_collection = event_collection_map_it->second.collection;+ const auto& query_collection_events_it = query_collection_events.find(query_collection);+ if(query_collection_events_it != query_collection_events.end()) {+ auto &events_vec = query_collection_events_it->second;#ifdef TEST_BUILD- if (isRateLimitEnabled) {+ if (isRateLimitEnabled) {#endifauto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();@@ -326,29 +386,54 @@ Option<bool> AnalyticsManager::add_event(const std::string& event_type, const stevents_cache.insert(client_ip, eventCache);}#ifdef TEST_BUILD- }+ }#endif- auto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(- std::chrono::system_clock::now().time_since_epoch()).count();-- event_t event(query, event_type, now_ts_useconds, user_id, doc_id, position);- events_vec.emplace_back(event);-- if(!counter_events.empty()) {- auto counter_events_it = counter_events.find(query_collection);- if (counter_events_it != counter_events.end()) {- auto event_weight_map_it = counter_events_it->second.event_weight_map.find(event_type);- if (event_weight_map_it != counter_events_it->second.event_weight_map.end()) {- auto inc_val = event_weight_map_it->second;- counter_events_it->second.docid_counts[doc_id] += inc_val;- } else {- LOG(ERROR) << "event_type " << event_type << " not defined in analytic rule for counter events.";+ auto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ std::string query;+ std::string user_id;+ std::string doc_id;+ std::vector<std::pair<std::string, std::string>> custom_data;++ if (event_type == CUSTOM_EVENT) {+ for(auto itr = event_json.begin(); itr != event_json.end(); ++itr) {+ if (itr.key() == "query") {+ query = itr.value().get<std::string>();+ } else if (itr.key()== "user_id") {+ user_id = itr.value().get<std::string>();+ } else if (itr.key() == "doc_id") {+ doc_id = itr.value().get<std::string>();+ } else {+ auto kv = std::make_pair(itr.key(), itr.value().get<std::string>());+ custom_data.push_back(kv);+ }}} else {- LOG(ERROR) << "collection " << query_collection << " not found in analytics rule.";+ query = event_json.contains("q") ? event_json["q"].get<std::string>() : "";+ user_id = event_json["user_id"].get<std::string>();+ doc_id = event_json["doc_id"].get<std::string>();}- }+ event_t event(query, event_type, now_ts_useconds, user_id, doc_id, event_name, custom_data);+ events_vec.emplace_back(event);++ if (!counter_events.empty()) {+ auto counter_events_it = counter_events.find(query_collection);+ if (counter_events_it != counter_events.end()) {+ auto event_weight_map_it = counter_events_it->second.event_weight_map.find(event_type);+ if (event_weight_map_it != counter_events_it->second.event_weight_map.end()) {+ auto inc_val = event_weight_map_it->second;+ counter_events_it->second.docid_counts[doc_id] += inc_val;+ } else {+ LOG(ERROR) << "event_type " << event_type+ << " not defined in analytic rule for counter events.";+ }+ } else {+ LOG(ERROR) << "collection " << query_collection << " not found in analytics rule.";+ }+ }+ }return Option<bool>(true);}@@ -491,19 +576,19 @@ void AnalyticsManager::persist_query_events(ReplicationState *raft_server, uint6void AnalyticsManager::persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {// lock is held by callerfor (const auto &events_collection_it: query_collection_events) {+ const auto& collection = events_collection_it.first;for (const auto &event: events_collection_it.second) {if (analytics_logs.is_open()) {//store events to log file- const auto collection = events_collection_it.first;- if (event.event_type == QUERY_CLICK) {- analytics_logs << event.timestamp << "\t" << event.user_id << "\t"- << 'C' << "\t" << event.doc_id << "\t"- << collection << "\t" << event.query << "\n";- } else if (event.event_type == QUERY_PURCHASE) {- analytics_logs << event.timestamp << "\t" << event.user_id << "\t"- << 'P' << "\t" << event.doc_id << "\t" << collection << "\n";+ analytics_logs << event.timestamp << "\t" << event.name << "\t"+ << collection << "\t" << event.user_id << "\t" << event.doc_id << "\t"+ << event.query << "\t";++ for(const auto& kv : event.data) {+ analytics_logs << kv.second << "\t";}+ analytics_logs << "\n";analytics_logs << std::flush;}}@@ -594,4 +679,17 @@ void AnalyticsManager::resetToggleRateLimit(bool toggle) {std::unique_lock lk(mutex);events_cache.clear();isRateLimitEnabled = toggle;+}++std::string AnalyticsManager::get_sub_event_type(const std::string &event_type) {+ if(event_type == AnalyticsManager::CLICKS_TYPE) {+ return AnalyticsManager::CLICK_EVENT;+ } else if(event_type == AnalyticsManager::CONVERSIONS_TYPE) {+ return AnalyticsManager::CONVERSION_EVENT;+ } else if(event_type == AnalyticsManager::VISITS_TYPE) {+ return AnalyticsManager::VISIT_EVENT;+ } else if(event_type == AnalyticsManager::CUSTOM_EVENTS_TYPE) {+ return AnalyticsManager::CUSTOM_EVENT;+ }+ return "";}\ No newline at end of filediff --git a/src/event_manager.cpp b/src/event_manager.cppindex 14dddaad..a0f772fe 100644--- a/src/event_manager.cpp+++ b/src/event_manager.cpp@@ -47,49 +47,47 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::strconst std::string& query = event_data_query_it.get<std::string>();AnalyticsManager::get_instance().add_suggestion(coll.get<std::string>(), query, query, false, "");}- } else if(event_type == AnalyticsManager::QUERY_CLICK || event_type == AnalyticsManager::QUERY_PURCHASE) {- if (!event.contains("data")) {+ } else if(event_type == AnalyticsManager::CLICK_EVENT || event_type == AnalyticsManager::CONVERSION_EVENT+ || event_type == AnalyticsManager::VISIT_EVENT || event_type == AnalyticsManager::CUSTOM_EVENT) {+ if (!event.contains(EVENT_DATA)) {return Option<bool>(404, "key `data` not found.");}const auto &event_data_val = event[EVENT_DATA];+ if (!event.contains(EVENT_NAME)) {+ return Option<bool>(404, "key `name` not found.");+ }+ const auto &event_name = event[EVENT_NAME];if (!event_data_val.is_object()) {return Option<bool>(500, "event_data_val is not object.");}- if (!event_data_val.contains("q") || !event_data_val.contains("doc_id") || !event_data_val.contains("user_id")- || !event_data_val.contains("position") || !event_data_val.contains("collection")) {- return Option<bool>(500, "event json data fields should contain `q`, `doc_id`, `position`, `user_id`, and `collection`.");- }+ if(event_type != AnalyticsManager::CUSTOM_EVENT) {+ //visit event might not contain query so exclude check+ if(event_type != AnalyticsManager::VISIT_EVENT && !event_data_val.contains("q")) {+ return Option<bool>(500,"event json data fields should contain `q`.");+ }- if (!event_data_val["q"].is_string()) {- return Option<bool>(500, "`q` value should be string.");- }+ if (event_data_val.contains("q") && !event_data_val["q"].is_string()) {+ return Option<bool>(500, "`q` value should be string.");+ }- if(!event_data_val["doc_id"].is_string()) {- return Option<bool>(500, "`doc_id` value should be string.");- }+ if (!event_data_val.contains("doc_id") || !event_data_val.contains("user_id")) {+ return Option<bool>(500,+ "event json data fields should contain `doc_id`, `user_id`.");+ }- if(!event_data_val["user_id"].is_string()) {- return Option<bool>(500, "`user_id` value should be string.");- }-- if(!event_data_val["position"].is_number_unsigned()){- return Option<bool>(500, "`position` value should be unsigned int.");- }+ if (!event_data_val["doc_id"].is_string()) {+ return Option<bool>(500, "`doc_id` value should be string.");+ }- if(!event_data_val["collection"].is_string()) {- return Option<bool>(500, "`collection` value should be string.");+ if (!event_data_val["user_id"].is_string()) {+ return Option<bool>(500, "`user_id` value should be string.");+ }}- const std::string& query = event_data_val["q"].get<std::string>();- const std::string& user_id = event_data_val["user_id"].get<std::string>();- const std::string& doc_id = event_data_val["doc_id"].get<std::string>();- uint64_t position = event_data_val["position"].get<uint64_t>();- const std::string& collection = event_data_val["collection"].get<std::string>();-- auto op = AnalyticsManager::get_instance().add_event(event_type, collection, query, user_id, doc_id, position, client_ip);+ auto op = AnalyticsManager::get_instance().add_event(client_ip, event_type, event_name, event_data_val);if(!op.ok()) {return Option<bool>(op.code(), op.error());}diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex ca779ea7..b4f8a5f0 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -196,6 +196,24 @@ TEST_F(AnalyticsManagerTest, GetAndDeleteSuggestions) {ASSERT_FALSE(create_op.ok());ASSERT_EQ("There's already another configuration for this destination collection.", create_op.error());+ analytics_rule = R"({+ "name": "top_search_queries3",+ "type": "popular_queries",+ "params": {+ "limit": 100,+ "source": {+ "collections": [241, 2353]+ },+ "destination": {+ "collection": "top_queries"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_FALSE(create_op.ok());+ ASSERT_EQ("Must contain a valid list of source collection names.", create_op.error());+analytics_rule = R"({"name": "top_search_queries2","type": "popular_queries",@@ -270,45 +288,85 @@ TEST_F(AnalyticsManagerTest, EventsValidation) {std::shared_ptr<http_req> req = std::make_shared<http_req>();std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);+ auto analytics_rule = R"({+ "name": "product_click_events",+ "type": "clicks",+ "params": {+ "name": "AP",+ "source": {+ "collection": "titles"+ }+ }+ })"_json;++ auto create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_TRUE(create_op.ok());++ analytics_rule = R"({+ "name": "product_visitors",+ "type": "visits",+ "params": {+ "name": "VP",+ "source": {+ "collection": "titles"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_TRUE(create_op.ok());+//wrong typenlohmann::json event1 = R"({- "type": "click",+ "type": "query_click",+ "name": "AP","data": {"q": "technology","collection": "titles","doc_id": "21",- "position": 2,"user_id": "13"}})"_json;req->body = event1.dump();ASSERT_FALSE(post_create_event(req, res));- ASSERT_EQ("{\"message\": \"event_type click not found.\"}", res->body);+ ASSERT_EQ("{\"message\": \"event_type query_click not found.\"}", res->body);++ //missing name+ event1 = R"({+ "type": "click",+ "data": {+ "collection": "titles",+ "doc_id": "21",+ "user_id": "13"+ }+ })"_json;++ req->body = event1.dump();+ ASSERT_FALSE(post_create_event(req, res));+ ASSERT_EQ("{\"message\": \"key `name` not found.\"}", res->body);//missing query paramnlohmann::json event2 = R"({- "type": "query_click",+ "type": "click",+ "name": "AP","data": {- "collection": "titles","doc_id": "21",- "position": 2,"user_id": "13"}})"_json;req->body = event2.dump();ASSERT_FALSE(post_create_event(req, res));- ASSERT_EQ("{\"message\": \"event json data fields should contain `q`, `doc_id`, `position`, `user_id`, and `collection`.\"}", res->body);+ ASSERT_EQ("{\"message\": \"event json data fields should contain `q`.\"}", res->body);//should be string typenlohmann::json event3 = R"({- "type": "query_purchase",+ "type": "conversion",+ "name": "AP","data": {"q": "technology",- "collection": "titles","doc_id": 21,- "position": 2,"user_id": "13"}})"_json;@@ -317,34 +375,106 @@ TEST_F(AnalyticsManagerTest, EventsValidation) {ASSERT_FALSE(post_create_event(req, res));ASSERT_EQ("{\"message\": \"`doc_id` value should be string.\"}", res->body);- //correct params+ //event name should be unique+ analytics_rule = R"({+ "name": "product_click_events2",+ "type": "clicks",+ "params": {+ "name": "AP",+ "source": {+ "collection": "titles"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_FALSE(create_op.ok());+ ASSERT_EQ("Event name already exists.", create_op.error());++ //wrong event namenlohmann::json event4 = R"({- "type": "query_click",+ "type": "visit",+ "name": "AB","data": {"q": "technology",- "collection": "titles","doc_id": "21",- "position": 2,- "user_id": "13"+ "user_id": "11"}})"_json;req->body = event4.dump();- ASSERT_TRUE(post_create_event(req, res));+ ASSERT_FALSE(post_create_event(req, res));+ //correct paramsnlohmann::json event5 = R"({- "type": "query_purchase",+ "type": "click",+ "name": "AP","data": {"q": "technology",- "collection": "titles","doc_id": "21",- "position": 4,- "user_id": "11"+ "user_id": "13"}})"_json;req->body = event5.dump();ASSERT_TRUE(post_create_event(req, res));++ nlohmann::json event6 = R"({+ "type": "visit",+ "name": "VP",+ "data": {+ "q": "technology",+ "doc_id": "21",+ "user_id": "11"+ }+ })"_json;++ req->body = event6.dump();+ ASSERT_TRUE(post_create_event(req, res));++ //wrong event type+ nlohmann::json event7 = R"({+ "type": "conversion",+ "name": "VP",+ "data": {+ "q": "technology",+ "doc_id": "21",+ "user_id": "11"+ }+ })"_json;++ req->body = event7.dump();+ ASSERT_FALSE(post_create_event(req, res));++ //custom event+ analytics_rule = R"({+ "name": "product_custom_events",+ "type": "custom_events",+ "params": {+ "name": "CP",+ "source": {+ "collection": "titles"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_TRUE(create_op.ok());++ nlohmann::json event8 = R"({+ "type": "custom",+ "name": "CP",+ "data": {+ "q": "technology",+ "doc_id": "21",+ "user_id": "11",+ "label1": "foo",+ "label2": "bar",+ "info": "xyz"+ }+ })"_json;+ req->body = event8.dump();+ ASSERT_TRUE(post_create_event(req, res));}TEST_F(AnalyticsManagerTest, EventsRateLimitTest) {@@ -360,38 +490,35 @@ TEST_F(AnalyticsManagerTest, EventsRateLimitTest) {std::shared_ptr<http_req> req = std::make_shared<http_req>();std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);- std::vector<nlohmann::json> events;-- nlohmann::json event1 = R"({- "type": "query_click",- "data": {- "q": "technology",- "collection": "titles",- "doc_id": "21",- "position": 2,- "user_id": "13"+ auto analytics_rule = R"({+ "name": "rate_limit",+ "type": "clicks",+ "params": {+ "name": "AB",+ "source": {+ "collection": "titles"+ }}})"_json;- nlohmann::json event2 = R"({- "type": "query_purchase",+ auto create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_TRUE(create_op.ok());++ nlohmann::json event1 = R"({+ "type": "click",+ "name": "AB","data": {"q": "technology",- "collection": "titles","doc_id": "21",- "position": 4,- "user_id": "11"+ "user_id": "13"}})"_json;- events.push_back(event1);- events.push_back(event2);-//reset the LRU cache to test the rate limitanalyticsManager.resetToggleRateLimit(true);for(auto i = 0; i < 5; ++i) {- req->body = events[i%2].dump();+ req->body = event1.dump();ASSERT_TRUE(post_create_event(req, res));}@@ -460,6 +587,9 @@ TEST_F(AnalyticsManagerTest, NoresultsQueries) {}TEST_F(AnalyticsManagerTest, SuggestionConfigRule) {+ //clear all rules first+ analyticsManager.remove_all_rules();+nlohmann::json titles_schema = R"({"name": "titles","fields": [@@ -566,13 +696,41 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {doc["title"] = "Formal pants";ASSERT_TRUE(products_coll->add(doc.dump()).ok());- nlohmann::json analytics_rule = R"({+ auto analytics_rule = R"({+ "name": "click_events",+ "type": "clicks",+ "params": {+ "name": "XY",+ "source": {+ "collection": "products"+ }+ }+ })"_json;++ auto create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_TRUE(create_op.ok());++ analytics_rule = R"({+ "name": "conversion_events",+ "type": "conversions",+ "params": {+ "name": "YZ",+ "source": {+ "collection": "products"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_TRUE(create_op.ok());++ analytics_rule = R"({"name": "product_popularity","type": "counter","params": {"source": {"collections": ["products"],- "events": [{"type": "query_click", "weight": 1}, {"type": "query_purchase", "weight": 5} ]+ "events": [{"type": "click", "weight": 1}, {"type": "conversion", "weight": 5} ]},"destination": {"collection": "products",@@ -581,19 +739,18 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {}})"_json;- auto create_op = analyticsManager.create_rule(analytics_rule, false, true);+ create_op = analyticsManager.create_rule(analytics_rule, false, true);ASSERT_TRUE(create_op.ok());std::shared_ptr<http_req> req = std::make_shared<http_req>();std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);nlohmann::json event1 = R"({- "type": "query_purchase",+ "type": "conversion",+ "name": "YZ","data": {"q": "trousers",- "collection": "products","doc_id": "1",- "position": 2,"user_id": "13"}})"_json;@@ -602,12 +759,11 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {ASSERT_TRUE(post_create_event(req, res));nlohmann::json event2 = R"({- "type": "query_click",+ "type": "click",+ "name": "XY","data": {"q": "shorts",- "collection": "products","doc_id": "3",- "position": 4,"user_id": "11"}})"_json;@@ -625,12 +781,11 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {ASSERT_EQ(2, popular_clicks["products"].docid_counts["3"]);nlohmann::json event3 = R"({- "type": "query_click",+ "type": "click",+ "name": "XY","data": {"q": "shorts",- "collection": "products","doc_id": "1",- "position": 4,"user_id": "11"}})"_json;@@ -639,12 +794,11 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {ASSERT_TRUE(post_create_event(req, res));nlohmann::json event4 = R"({- "type": "query_purchase",+ "type": "conversion",+ "name": "YZ","data": {"q": "shorts",- "collection": "products","doc_id": "3",- "position": 4,"user_id": "11"}})"_json;