Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Event analytics revised (#1522)

Krunal Gandhi206cecffbd42
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index 85e30e2a..427e6609 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -8,25 +8,33 @@
#include <shared_mutex>
#include "lru/lru.hpp"
+struct event_type_collection {
+ std::string event_type;
+ std::string collection;
+};
+
struct event_t {
std::string query;
std::string event_type;
uint64_t timestamp;
std::string user_id;
std::string doc_id;
- uint64_t position;
+ std::string name;
+ std::vector<std::pair<std::string, std::string>> data;
event_t() = delete;
~event_t() = default;
- event_t(std::string q, std::string type, uint64_t ts, std::string uid, std::string id, uint64_t pos) {
+ event_t(const std::string& q, const std::string& type, uint64_t ts, const std::string& uid, const std::string& id,
+ const std::string& event_name, const std::vector<std::pair<std::string, std::string>> datavec) {
query = q;
event_type = type;
timestamp = ts;
user_id = uid;
doc_id = id;
- position = pos;
+ name = event_name;
+ data = datavec;
}
event_t& operator=(event_t& other) {
@@ -36,18 +44,25 @@ struct event_t {
timestamp = other.timestamp;
user_id = other.user_id;
doc_id = other.doc_id;
- position = other.position;
+ name = other.name;
+ data = other.data;
return *this;
}
}
void to_json(nlohmann::json& obj) const {
obj["query"] = query;
- obj["event_type"] = event_type;
+ obj["type"] = event_type;
obj["timestamp"] = timestamp;
obj["user_id"] = user_id;
obj["doc_id"] = doc_id;
- obj["position"] = position;
+ obj["name"] = name;
+
+ if(event_type == "custom") {
+ for(const auto& kv : data) {
+ obj[kv.first] = kv.second;
+ }
+ }
}
};
@@ -119,6 +134,9 @@ private:
//query collection => events
std::unordered_map<std::string, std::vector<event_t>> query_collection_events;
+ //event_name => collection
+ std::unordered_map<std::string, event_type_collection> event_collection_map;
+
// per_ip cache for rate limiting
LRU::Cache<std::string, event_cache_t> events_cache;
@@ -137,14 +155,22 @@ private:
bool upsert,
bool write_to_disk);
+ std::string get_sub_event_type(const std::string& event_type);
+
public:
static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";
static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";
static constexpr const char* NOHITS_QUERIES_TYPE = "nohits_queries";
static constexpr const char* COUNTER_TYPE = "counter";
- static constexpr const char* QUERY_CLICK = "query_click";
- static constexpr const char* QUERY_PURCHASE = "query_purchase";
+ static constexpr const char* CLICKS_TYPE = "clicks";
+ static constexpr const char* CONVERSIONS_TYPE = "conversions";
+ static constexpr const char* VISITS_TYPE = "visits";
+ static constexpr const char* CUSTOM_EVENTS_TYPE = "custom_events";
+ static constexpr const char* CLICK_EVENT = "click";
+ static constexpr const char* CONVERSION_EVENT = "conversion";
+ static constexpr const char* VISIT_EVENT = "visit";
+ static constexpr const char* CUSTOM_EVENT = "custom";
static AnalyticsManager& get_instance() {
static AnalyticsManager instance;
@@ -166,6 +192,8 @@ public:
Option<bool> remove_rule(const std::string& name);
+ Option<bool> remove_all_rules();
+
void add_suggestion(const std::string& query_collection,
const std::string& query, const std::string& expanded_query,
bool live_query, const std::string& user_id);
@@ -178,8 +206,8 @@ public:
std::unordered_map<std::string, QueryAnalytics*> get_popular_queries();
- Option<bool> add_event(const std::string& event_type, const std::string& query_collection, const std::string& query, const std::string& user_id,
- std::string doc_id, uint64_t position, const std::string& client_ip);
+ Option<bool> add_event(const std::string& client_ip, const std::string& event_type,
+ const std::string& event_name, const nlohmann::json& event_data);
void persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s);
diff --git a/include/event_manager.h b/include/event_manager.h
index 34793690..665bf404 100644
--- a/include/event_manager.h
+++ b/include/event_manager.h
@@ -11,6 +11,7 @@ private:
static constexpr char* EVENT_TYPE = "type";
static constexpr char* EVENT_DATA = "data";
+ static constexpr char* EVENT_NAME = "name";
public:
static EventManager& get_instance() {
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index f2581191..7b952041 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -41,7 +41,8 @@ Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert,
}
if(payload["type"] == POPULAR_QUERIES_TYPE || payload["type"] == NOHITS_QUERIES_TYPE
- || payload["type"] == COUNTER_TYPE) {
+ || payload["type"] == COUNTER_TYPE || payload["type"] == CLICKS_TYPE || payload["type"] == CONVERSIONS_TYPE
+ || payload["type"] == VISITS_TYPE || payload["type"] == CUSTOM_EVENTS_TYPE) {
return create_index(payload, upsert, write_to_disk);
}
@@ -52,6 +53,8 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert
// params and name are validated upstream
const std::string& suggestion_config_name = payload["name"].get<std::string>();
bool already_exists = suggestion_configs.find(suggestion_config_name) != suggestion_configs.end();
+ bool is_event_type = (payload["type"] == CLICKS_TYPE || payload["type"] == CONVERSIONS_TYPE
+ || payload["type"] == VISITS_TYPE || payload["type"] == CUSTOM_EVENTS_TYPE);
if(!upsert && already_exists) {
return Option<bool>(400, "There's already another configuration with the name `" +
@@ -64,19 +67,6 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert
return Option<bool>(400, "Bad or missing source.");
}
- if(!params.contains("destination") || !params["destination"].is_object()) {
- return Option<bool>(400, "Bad or missing destination.");
- }
-
- if(payload["type"] == COUNTER_TYPE) {
- if (!params["source"].contains("events") || (params["source"].contains("events") &&
- (params["source"]["events"].empty()
- || !params["source"]["events"].is_array()
- || !params["source"]["events"][0].is_object()))) {
- return Option<bool>(400, "Bad or missing events.");
- }
- }
-
size_t limit = 1000;
bool expand_query = false;
@@ -88,61 +78,96 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert
expand_query = params["expand_query"].get<bool>();
}
- if(!params["source"].contains("collections") || !params["source"]["collections"].is_array()) {
- return Option<bool>(400, "Must contain a valid list of source collections.");
- }
-
- if(!params["destination"].contains("collection") || !params["destination"]["collection"].is_string()) {
- return Option<bool>(400, "Must contain a valid destination collection.");
- }
-
std::string counter_field;
+ std::string suggestion_collection;
- if(params["destination"].contains("counter_field")) {
- if (!params["destination"]["counter_field"].is_string()) {
- return Option<bool>(400, "Must contain a valid counter_field.");
- }
- counter_field = params["destination"]["counter_field"].get<std::string>();
- }
-
- const std::string& suggestion_collection = params["destination"]["collection"].get<std::string>();
suggestion_config_t suggestion_config;
suggestion_config.name = suggestion_config_name;
- suggestion_config.suggestion_collection = suggestion_collection;
suggestion_config.limit = limit;
suggestion_config.expand_query = expand_query;
suggestion_config.rule_type = payload["type"];
- if(payload["type"] == POPULAR_QUERIES_TYPE) {
- if (!upsert && popular_queries.count(suggestion_collection) != 0) {
- return Option<bool>(400, "There's already another configuration for this destination collection.");
+ if(is_event_type) {
+ if (!params.contains("name") || params["name"].empty()) {
+ return Option<bool>(400, "Bad or missing name in params");
}
- } else if(payload["type"] == NOHITS_QUERIES_TYPE) {
- if (!upsert && nohits_queries.count(suggestion_collection) != 0) {
- return Option<bool>(400, "There's already another configuration for this destination collection.");
+
+ if (!params["source"].contains("collection") || !params["source"]["collection"].is_string()) {
+ return Option<bool>(400, "Must contain a valid source collection.");
}
- } else if(payload["type"] == COUNTER_TYPE) {
- if (!upsert && counter_events.count(suggestion_collection) != 0) {
- return Option<bool>(400, "There's already another configuration for this destination collection.");
+
+ suggestion_collection = params["source"]["collection"].get<std::string>();
+ suggestion_config.suggestion_collection = suggestion_collection;
+
+ if (event_collection_map.count(params["name"]) != 0) {
+ return Option<bool>(400, "Event name already exists.");
}
+ } else {
+ if (payload["type"] == COUNTER_TYPE) {
+ if (!params["source"].contains("events") || (params["source"].contains("events") &&
+ (params["source"]["events"].empty()
+ || !params["source"]["events"].is_array()
+ || !params["source"]["events"][0].is_object()))) {
+ return Option<bool>(400, "Bad or missing events.");
+ }
+ }
+
+ if (!params.contains("destination") || !params["destination"].is_object()) {
+ return Option<bool>(400, "Bad or missing destination.");
+ }
+
+ if (!params["source"].contains("collections") || !params["source"]["collections"].is_array()) {
+ return Option<bool>(400, "Must contain a valid list of source collections.");
+ }
+
+ if (!params["destination"].contains("collection") || !params["destination"]["collection"].is_string()) {
+ return Option<bool>(400, "Must contain a valid destination collection.");
+ }
+
- auto coll = CollectionManager::get_instance().get_collection(suggestion_collection).get();
- if(coll != nullptr) {
- if (!coll->contains_field(counter_field)) {
- return Option<bool>(404, "counter_field `" + counter_field + "` not found in destination collection.");
+ if (params["destination"].contains("counter_field")) {
+ if (!params["destination"]["counter_field"].is_string()) {
+ return Option<bool>(400, "Must contain a valid counter_field.");
}
- } else {
- return Option<bool>(404, "Collection `" + suggestion_collection + "` not found.");
+ counter_field = params["destination"]["counter_field"].get<std::string>();
}
- }
- for(const auto& coll: params["source"]["collections"]) {
- if(!coll.is_string()) {
- return Option<bool>(400, "Must contain a valid list of source collection names.");
+ for (const auto &coll: params["source"]["collections"]) {
+ if (!coll.is_string()) {
+ return Option<bool>(400, "Must contain a valid list of source collection names.");
+ }
+
+ const std::string &src_collection = coll.get<std::string>();
+ suggestion_config.query_collections.push_back(src_collection);
}
- const std::string& src_collection = coll.get<std::string>();
- suggestion_config.query_collections.push_back(src_collection);
+ suggestion_collection = params["destination"]["collection"].get<std::string>();
+ suggestion_config.suggestion_collection = suggestion_collection;
+
+
+ if (payload["type"] == POPULAR_QUERIES_TYPE) {
+ if (!upsert && popular_queries.count(suggestion_collection) != 0) {
+ return Option<bool>(400, "There's already another configuration for this destination collection.");
+ }
+ } else if (payload["type"] == NOHITS_QUERIES_TYPE) {
+ if (!upsert && nohits_queries.count(suggestion_collection) != 0) {
+ return Option<bool>(400, "There's already another configuration for this destination collection.");
+ }
+ } else if (payload["type"] == COUNTER_TYPE) {
+ if (!upsert && counter_events.count(suggestion_collection) != 0) {
+ return Option<bool>(400, "There's already another configuration for this destination collection.");
+ }
+
+ auto coll = CollectionManager::get_instance().get_collection(suggestion_collection).get();
+ if (coll != nullptr) {
+ if (!coll->contains_field(counter_field)) {
+ return Option<bool>(404,
+ "counter_field `" + counter_field + "` not found in destination collection.");
+ }
+ } else {
+ return Option<bool>(404, "Collection `" + suggestion_collection + "` not found.");
+ }
+ }
}
std::unique_lock lock(mutex);
@@ -174,6 +199,16 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert
event_weight_map[event["type"]] = event["weight"];
}
counter_events.emplace(suggestion_collection, counter_event_t{counter_field, {}, event_weight_map});
+ } else if(is_event_type) {
+ //multiple event types can be used with one collection
+ //hence if no events are created for collection then only initialize it
+ if(query_collection_events.count(suggestion_collection) == 0) {
+ std::vector<event_t> vec;
+ query_collection_events.emplace(suggestion_collection, vec);
+ }
+ auto sub_event_type = get_sub_event_type(payload["type"]);
+ event_type_collection ec {sub_event_type, suggestion_collection};
+ event_collection_map.emplace(params["name"], ec);
}
if(write_to_disk) {
@@ -238,6 +273,16 @@ Option<bool> AnalyticsManager::remove_rule(const std::string &name) {
return Option<bool>(404, "Rule not found.");
}
+Option<bool> AnalyticsManager::remove_all_rules() {
+ std::unique_lock lock(mutex);
+
+ for(auto suggestion_config : suggestion_configs) {
+ remove_index(suggestion_config.first);
+ }
+
+ return Option<bool>(true);
+}
+
Option<bool> AnalyticsManager::remove_index(const std::string &name) {
// lock is held by caller
auto suggestion_configs_it = suggestion_configs.find(name);
@@ -266,6 +311,10 @@ Option<bool> AnalyticsManager::remove_index(const std::string &name) {
counter_events.erase(suggestion_collection);
}
+ if(query_collection_events.count(suggestion_collection) != 0) {
+ query_collection_events.erase(suggestion_collection);
+ }
+
suggestion_configs.erase(name);
auto suggestion_key = std::string(ANALYTICS_RULE_PREFIX) + "_" + name;
@@ -293,17 +342,28 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection,
}
}
-Option<bool> AnalyticsManager::add_event(const std::string& event_type, const std::string &query_collection, const std::string &query, const std::string &user_id,
- std::string doc_id, uint64_t position, const std::string& client_ip) {
+Option<bool> AnalyticsManager::add_event(const std::string& client_ip, const std::string& event_type,
+ const std::string& event_name, const nlohmann::json& event_json) {
std::unique_lock lock(mutex);
if(!analytics_logs.is_open()) {
return Option<bool>(true);
}
+ const auto event_collection_map_it = event_collection_map.find(event_name);
+ if(event_collection_map_it == event_collection_map.end()) {
+ return Option<bool>(404, "No analytics rule defined for event name" + event_name);
+ }
+
+ if(event_collection_map_it->second.event_type != event_type) {
+ return Option<bool>(400, "event_type mismatch in analytic rules.");
+ }
- auto &events_vec= query_collection_events[query_collection];
+ const auto& query_collection = event_collection_map_it->second.collection;
+ const auto& query_collection_events_it = query_collection_events.find(query_collection);
+ if(query_collection_events_it != query_collection_events.end()) {
+ auto &events_vec = query_collection_events_it->second;
#ifdef TEST_BUILD
- if (isRateLimitEnabled) {
+ if (isRateLimitEnabled) {
#endif
auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
@@ -326,29 +386,54 @@ Option<bool> AnalyticsManager::add_event(const std::string& event_type, const st
events_cache.insert(client_ip, eventCache);
}
#ifdef TEST_BUILD
- }
+ }
#endif
- auto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(
- std::chrono::system_clock::now().time_since_epoch()).count();
-
- event_t event(query, event_type, now_ts_useconds, user_id, doc_id, position);
- events_vec.emplace_back(event);
-
- if(!counter_events.empty()) {
- auto counter_events_it = counter_events.find(query_collection);
- if (counter_events_it != counter_events.end()) {
- auto event_weight_map_it = counter_events_it->second.event_weight_map.find(event_type);
- if (event_weight_map_it != counter_events_it->second.event_weight_map.end()) {
- auto inc_val = event_weight_map_it->second;
- counter_events_it->second.docid_counts[doc_id] += inc_val;
- } else {
- LOG(ERROR) << "event_type " << event_type << " not defined in analytic rule for counter events.";
+ auto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ std::string query;
+ std::string user_id;
+ std::string doc_id;
+ std::vector<std::pair<std::string, std::string>> custom_data;
+
+ if (event_type == CUSTOM_EVENT) {
+ for(auto itr = event_json.begin(); itr != event_json.end(); ++itr) {
+ if (itr.key() == "query") {
+ query = itr.value().get<std::string>();
+ } else if (itr.key()== "user_id") {
+ user_id = itr.value().get<std::string>();
+ } else if (itr.key() == "doc_id") {
+ doc_id = itr.value().get<std::string>();
+ } else {
+ auto kv = std::make_pair(itr.key(), itr.value().get<std::string>());
+ custom_data.push_back(kv);
+ }
}
} else {
- LOG(ERROR) << "collection " << query_collection << " not found in analytics rule.";
+ query = event_json.contains("q") ? event_json["q"].get<std::string>() : "";
+ user_id = event_json["user_id"].get<std::string>();
+ doc_id = event_json["doc_id"].get<std::string>();
}
- }
+ event_t event(query, event_type, now_ts_useconds, user_id, doc_id, event_name, custom_data);
+ events_vec.emplace_back(event);
+
+ if (!counter_events.empty()) {
+ auto counter_events_it = counter_events.find(query_collection);
+ if (counter_events_it != counter_events.end()) {
+ auto event_weight_map_it = counter_events_it->second.event_weight_map.find(event_type);
+ if (event_weight_map_it != counter_events_it->second.event_weight_map.end()) {
+ auto inc_val = event_weight_map_it->second;
+ counter_events_it->second.docid_counts[doc_id] += inc_val;
+ } else {
+ LOG(ERROR) << "event_type " << event_type
+ << " not defined in analytic rule for counter events.";
+ }
+ } else {
+ LOG(ERROR) << "collection " << query_collection << " not found in analytics rule.";
+ }
+ }
+ }
return Option<bool>(true);
}
@@ -491,19 +576,19 @@ void AnalyticsManager::persist_query_events(ReplicationState *raft_server, uint6
void AnalyticsManager::persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {
// lock is held by caller
for (const auto &events_collection_it: query_collection_events) {
+ const auto& collection = events_collection_it.first;
for (const auto &event: events_collection_it.second) {
if (analytics_logs.is_open()) {
//store events to log file
- const auto collection = events_collection_it.first;
- if (event.event_type == QUERY_CLICK) {
- analytics_logs << event.timestamp << "\t" << event.user_id << "\t"
- << 'C' << "\t" << event.doc_id << "\t"
- << collection << "\t" << event.query << "\n";
- } else if (event.event_type == QUERY_PURCHASE) {
- analytics_logs << event.timestamp << "\t" << event.user_id << "\t"
- << 'P' << "\t" << event.doc_id << "\t" << collection << "\n";
+ analytics_logs << event.timestamp << "\t" << event.name << "\t"
+ << collection << "\t" << event.user_id << "\t" << event.doc_id << "\t"
+ << event.query << "\t";
+
+ for(const auto& kv : event.data) {
+ analytics_logs << kv.second << "\t";
}
+ analytics_logs << "\n";
analytics_logs << std::flush;
}
}
@@ -594,4 +679,17 @@ void AnalyticsManager::resetToggleRateLimit(bool toggle) {
std::unique_lock lk(mutex);
events_cache.clear();
isRateLimitEnabled = toggle;
+}
+
+std::string AnalyticsManager::get_sub_event_type(const std::string &event_type) {
+ if(event_type == AnalyticsManager::CLICKS_TYPE) {
+ return AnalyticsManager::CLICK_EVENT;
+ } else if(event_type == AnalyticsManager::CONVERSIONS_TYPE) {
+ return AnalyticsManager::CONVERSION_EVENT;
+ } else if(event_type == AnalyticsManager::VISITS_TYPE) {
+ return AnalyticsManager::VISIT_EVENT;
+ } else if(event_type == AnalyticsManager::CUSTOM_EVENTS_TYPE) {
+ return AnalyticsManager::CUSTOM_EVENT;
+ }
+ return "";
}
\ No newline at end of file
diff --git a/src/event_manager.cpp b/src/event_manager.cpp
index 14dddaad..a0f772fe 100644
--- a/src/event_manager.cpp
+++ b/src/event_manager.cpp
@@ -47,49 +47,47 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::str
const std::string& query = event_data_query_it.get<std::string>();
AnalyticsManager::get_instance().add_suggestion(coll.get<std::string>(), query, query, false, "");
}
- } else if(event_type == AnalyticsManager::QUERY_CLICK || event_type == AnalyticsManager::QUERY_PURCHASE) {
- if (!event.contains("data")) {
+ } else if(event_type == AnalyticsManager::CLICK_EVENT || event_type == AnalyticsManager::CONVERSION_EVENT
+ || event_type == AnalyticsManager::VISIT_EVENT || event_type == AnalyticsManager::CUSTOM_EVENT) {
+ if (!event.contains(EVENT_DATA)) {
return Option<bool>(404, "key `data` not found.");
}
const auto &event_data_val = event[EVENT_DATA];
+ if (!event.contains(EVENT_NAME)) {
+ return Option<bool>(404, "key `name` not found.");
+ }
+ const auto &event_name = event[EVENT_NAME];
if (!event_data_val.is_object()) {
return Option<bool>(500, "event_data_val is not object.");
}
- if (!event_data_val.contains("q") || !event_data_val.contains("doc_id") || !event_data_val.contains("user_id")
- || !event_data_val.contains("position") || !event_data_val.contains("collection")) {
- return Option<bool>(500, "event json data fields should contain `q`, `doc_id`, `position`, `user_id`, and `collection`.");
- }
+ if(event_type != AnalyticsManager::CUSTOM_EVENT) {
+ //visit event might not contain query so exclude check
+ if(event_type != AnalyticsManager::VISIT_EVENT && !event_data_val.contains("q")) {
+ return Option<bool>(500,"event json data fields should contain `q`.");
+ }
- if (!event_data_val["q"].is_string()) {
- return Option<bool>(500, "`q` value should be string.");
- }
+ if (event_data_val.contains("q") && !event_data_val["q"].is_string()) {
+ return Option<bool>(500, "`q` value should be string.");
+ }
- if(!event_data_val["doc_id"].is_string()) {
- return Option<bool>(500, "`doc_id` value should be string.");
- }
+ if (!event_data_val.contains("doc_id") || !event_data_val.contains("user_id")) {
+ return Option<bool>(500,
+ "event json data fields should contain `doc_id`, `user_id`.");
+ }
- if(!event_data_val["user_id"].is_string()) {
- return Option<bool>(500, "`user_id` value should be string.");
- }
-
- if(!event_data_val["position"].is_number_unsigned()){
- return Option<bool>(500, "`position` value should be unsigned int.");
- }
+ if (!event_data_val["doc_id"].is_string()) {
+ return Option<bool>(500, "`doc_id` value should be string.");
+ }
- if(!event_data_val["collection"].is_string()) {
- return Option<bool>(500, "`collection` value should be string.");
+ if (!event_data_val["user_id"].is_string()) {
+ return Option<bool>(500, "`user_id` value should be string.");
+ }
}
- const std::string& query = event_data_val["q"].get<std::string>();
- const std::string& user_id = event_data_val["user_id"].get<std::string>();
- const std::string& doc_id = event_data_val["doc_id"].get<std::string>();
- uint64_t position = event_data_val["position"].get<uint64_t>();
- const std::string& collection = event_data_val["collection"].get<std::string>();
-
- auto op = AnalyticsManager::get_instance().add_event(event_type, collection, query, user_id, doc_id, position, client_ip);
+ auto op = AnalyticsManager::get_instance().add_event(client_ip, event_type, event_name, event_data_val);
if(!op.ok()) {
return Option<bool>(op.code(), op.error());
}
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index ca779ea7..b4f8a5f0 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -196,6 +196,24 @@ TEST_F(AnalyticsManagerTest, GetAndDeleteSuggestions) {
ASSERT_FALSE(create_op.ok());
ASSERT_EQ("There's already another configuration for this destination collection.", create_op.error());
+ analytics_rule = R"({
+ "name": "top_search_queries3",
+ "type": "popular_queries",
+ "params": {
+ "limit": 100,
+ "source": {
+ "collections": [241, 2353]
+ },
+ "destination": {
+ "collection": "top_queries"
+ }
+ }
+ })"_json;
+
+ create_op = analyticsManager.create_rule(analytics_rule, false, true);
+ ASSERT_FALSE(create_op.ok());
+ ASSERT_EQ("Must contain a valid list of source collection names.", create_op.error());
+
analytics_rule = R"({
"name": "top_search_queries2",
"type": "popular_queries",
@@ -270,45 +288,85 @@ TEST_F(AnalyticsManagerTest, EventsValidation) {
std::shared_ptr<http_req> req = std::make_shared<http_req>();
std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+ auto analytics_rule = R"({
+ "name": "product_click_events",
+ "type": "clicks",
+ "params": {
+ "name": "AP",
+ "source": {
+ "collection": "titles"
+ }
+ }
+ })"_json;
+
+ auto create_op = analyticsManager.create_rule(analytics_rule, true, true);
+ ASSERT_TRUE(create_op.ok());
+
+ analytics_rule = R"({
+ "name": "product_visitors",
+ "type": "visits",
+ "params": {
+ "name": "VP",
+ "source": {
+ "collection": "titles"
+ }
+ }
+ })"_json;
+
+ create_op = analyticsManager.create_rule(analytics_rule, true, true);
+ ASSERT_TRUE(create_op.ok());
+
//wrong type
nlohmann::json event1 = R"({
- "type": "click",
+ "type": "query_click",
+ "name": "AP",
"data": {
"q": "technology",
"collection": "titles",
"doc_id": "21",
- "position": 2,
"user_id": "13"
}
})"_json;
req->body = event1.dump();
ASSERT_FALSE(post_create_event(req, res));
- ASSERT_EQ("{\"message\": \"event_type click not found.\"}", res->body);
+ ASSERT_EQ("{\"message\": \"event_type query_click not found.\"}", res->body);
+
+ //missing name
+ event1 = R"({
+ "type": "click",
+ "data": {
+ "collection": "titles",
+ "doc_id": "21",
+ "user_id": "13"
+ }
+ })"_json;
+
+ req->body = event1.dump();
+ ASSERT_FALSE(post_create_event(req, res));
+ ASSERT_EQ("{\"message\": \"key `name` not found.\"}", res->body);
//missing query param
nlohmann::json event2 = R"({
- "type": "query_click",
+ "type": "click",
+ "name": "AP",
"data": {
- "collection": "titles",
"doc_id": "21",
- "position": 2,
"user_id": "13"
}
})"_json;
req->body = event2.dump();
ASSERT_FALSE(post_create_event(req, res));
- ASSERT_EQ("{\"message\": \"event json data fields should contain `q`, `doc_id`, `position`, `user_id`, and `collection`.\"}", res->body);
+ ASSERT_EQ("{\"message\": \"event json data fields should contain `q`.\"}", res->body);
//should be string type
nlohmann::json event3 = R"({
- "type": "query_purchase",
+ "type": "conversion",
+ "name": "AP",
"data": {
"q": "technology",
- "collection": "titles",
"doc_id": 21,
- "position": 2,
"user_id": "13"
}
})"_json;
@@ -317,34 +375,106 @@ TEST_F(AnalyticsManagerTest, EventsValidation) {
ASSERT_FALSE(post_create_event(req, res));
ASSERT_EQ("{\"message\": \"`doc_id` value should be string.\"}", res->body);
- //correct params
+ //event name should be unique
+ analytics_rule = R"({
+ "name": "product_click_events2",
+ "type": "clicks",
+ "params": {
+ "name": "AP",
+ "source": {
+ "collection": "titles"
+ }
+ }
+ })"_json;
+
+ create_op = analyticsManager.create_rule(analytics_rule, true, true);
+ ASSERT_FALSE(create_op.ok());
+ ASSERT_EQ("Event name already exists.", create_op.error());
+
+ //wrong event name
nlohmann::json event4 = R"({
- "type": "query_click",
+ "type": "visit",
+ "name": "AB",
"data": {
"q": "technology",
- "collection": "titles",
"doc_id": "21",
- "position": 2,
- "user_id": "13"
+ "user_id": "11"
}
})"_json;
req->body = event4.dump();
- ASSERT_TRUE(post_create_event(req, res));
+ ASSERT_FALSE(post_create_event(req, res));
+ //correct params
nlohmann::json event5 = R"({
- "type": "query_purchase",
+ "type": "click",
+ "name": "AP",
"data": {
"q": "technology",
- "collection": "titles",
"doc_id": "21",
- "position": 4,
- "user_id": "11"
+ "user_id": "13"
}
})"_json;
req->body = event5.dump();
ASSERT_TRUE(post_create_event(req, res));
+
+ nlohmann::json event6 = R"({
+ "type": "visit",
+ "name": "VP",
+ "data": {
+ "q": "technology",
+ "doc_id": "21",
+ "user_id": "11"
+ }
+ })"_json;
+
+ req->body = event6.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+
+ //wrong event type
+ nlohmann::json event7 = R"({
+ "type": "conversion",
+ "name": "VP",
+ "data": {
+ "q": "technology",
+ "doc_id": "21",
+ "user_id": "11"
+ }
+ })"_json;
+
+ req->body = event7.dump();
+ ASSERT_FALSE(post_create_event(req, res));
+
+ //custom event
+ analytics_rule = R"({
+ "name": "product_custom_events",
+ "type": "custom_events",
+ "params": {
+ "name": "CP",
+ "source": {
+ "collection": "titles"
+ }
+ }
+ })"_json;
+
+ create_op = analyticsManager.create_rule(analytics_rule, true, true);
+ ASSERT_TRUE(create_op.ok());
+
+ nlohmann::json event8 = R"({
+ "type": "custom",
+ "name": "CP",
+ "data": {
+ "q": "technology",
+ "doc_id": "21",
+ "user_id": "11",
+ "label1": "foo",
+ "label2": "bar",
+ "info": "xyz"
+ }
+ })"_json;
+ req->body = event8.dump();
+ ASSERT_TRUE(post_create_event(req, res));
}
TEST_F(AnalyticsManagerTest, EventsRateLimitTest) {
@@ -360,38 +490,35 @@ TEST_F(AnalyticsManagerTest, EventsRateLimitTest) {
std::shared_ptr<http_req> req = std::make_shared<http_req>();
std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
- std::vector<nlohmann::json> events;
-
- nlohmann::json event1 = R"({
- "type": "query_click",
- "data": {
- "q": "technology",
- "collection": "titles",
- "doc_id": "21",
- "position": 2,
- "user_id": "13"
+ auto analytics_rule = R"({
+ "name": "rate_limit",
+ "type": "clicks",
+ "params": {
+ "name": "AB",
+ "source": {
+ "collection": "titles"
+ }
}
})"_json;
- nlohmann::json event2 = R"({
- "type": "query_purchase",
+ auto create_op = analyticsManager.create_rule(analytics_rule, true, true);
+ ASSERT_TRUE(create_op.ok());
+
+ nlohmann::json event1 = R"({
+ "type": "click",
+ "name": "AB",
"data": {
"q": "technology",
- "collection": "titles",
"doc_id": "21",
- "position": 4,
- "user_id": "11"
+ "user_id": "13"
}
})"_json;
- events.push_back(event1);
- events.push_back(event2);
-
//reset the LRU cache to test the rate limit
analyticsManager.resetToggleRateLimit(true);
for(auto i = 0; i < 5; ++i) {
- req->body = events[i%2].dump();
+ req->body = event1.dump();
ASSERT_TRUE(post_create_event(req, res));
}
@@ -460,6 +587,9 @@ TEST_F(AnalyticsManagerTest, NoresultsQueries) {
}
TEST_F(AnalyticsManagerTest, SuggestionConfigRule) {
+ //clear all rules first
+ analyticsManager.remove_all_rules();
+
nlohmann::json titles_schema = R"({
"name": "titles",
"fields": [
@@ -566,13 +696,41 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {
doc["title"] = "Formal pants";
ASSERT_TRUE(products_coll->add(doc.dump()).ok());
- nlohmann::json analytics_rule = R"({
+ auto analytics_rule = R"({
+ "name": "click_events",
+ "type": "clicks",
+ "params": {
+ "name": "XY",
+ "source": {
+ "collection": "products"
+ }
+ }
+ })"_json;
+
+ auto create_op = analyticsManager.create_rule(analytics_rule, true, true);
+ ASSERT_TRUE(create_op.ok());
+
+ analytics_rule = R"({
+ "name": "conversion_events",
+ "type": "conversions",
+ "params": {
+ "name": "YZ",
+ "source": {
+ "collection": "products"
+ }
+ }
+ })"_json;
+
+ create_op = analyticsManager.create_rule(analytics_rule, true, true);
+ ASSERT_TRUE(create_op.ok());
+
+ analytics_rule = R"({
"name": "product_popularity",
"type": "counter",
"params": {
"source": {
"collections": ["products"],
- "events": [{"type": "query_click", "weight": 1}, {"type": "query_purchase", "weight": 5} ]
+ "events": [{"type": "click", "weight": 1}, {"type": "conversion", "weight": 5} ]
},
"destination": {
"collection": "products",
@@ -581,19 +739,18 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {
}
})"_json;
- auto create_op = analyticsManager.create_rule(analytics_rule, false, true);
+ create_op = analyticsManager.create_rule(analytics_rule, false, true);
ASSERT_TRUE(create_op.ok());
std::shared_ptr<http_req> req = std::make_shared<http_req>();
std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
nlohmann::json event1 = R"({
- "type": "query_purchase",
+ "type": "conversion",
+ "name": "YZ",
"data": {
"q": "trousers",
- "collection": "products",
"doc_id": "1",
- "position": 2,
"user_id": "13"
}
})"_json;
@@ -602,12 +759,11 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {
ASSERT_TRUE(post_create_event(req, res));
nlohmann::json event2 = R"({
- "type": "query_click",
+ "type": "click",
+ "name": "XY",
"data": {
"q": "shorts",
- "collection": "products",
"doc_id": "3",
- "position": 4,
"user_id": "11"
}
})"_json;
@@ -625,12 +781,11 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {
ASSERT_EQ(2, popular_clicks["products"].docid_counts["3"]);
nlohmann::json event3 = R"({
- "type": "query_click",
+ "type": "click",
+ "name": "XY",
"data": {
"q": "shorts",
- "collection": "products",
"doc_id": "1",
- "position": 4,
"user_id": "11"
}
})"_json;
@@ -639,12 +794,11 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {
ASSERT_TRUE(post_create_event(req, res));
nlohmann::json event4 = R"({
- "type": "query_purchase",
+ "type": "conversion",
+ "name": "YZ",
"data": {
"q": "shorts",
- "collection": "products",
"doc_id": "3",
- "position": 4,
"user_id": "11"
}
})"_json;