Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor analytics changes (#1502)

Krunal Gandhicde131f4b30a
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index c3642c6b..b5bdf4b2 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -121,7 +121,9 @@ private:
Store* store = nullptr;
std::ofstream analytics_logs;
- bool isRateLimitEnabled = false;
+ bool isRateLimitTestEnabled = false;
+ uint64_t analytics_logs_count = 0;
+
AnalyticsManager() {}
~AnalyticsManager();
@@ -138,6 +140,8 @@ public:
static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";
static constexpr const char* NOHITS_QUERIES_TYPE = "nohits_queries";
static constexpr const char* COUNTER_TYPE = "counter";
+ static constexpr const char* QUERY_CLICK = "query_click";
+ static constexpr const char* QUERY_PURCHASE = "query_purchase";
static AnalyticsManager& get_instance() {
static AnalyticsManager instance;
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index a897c744..53b15b74 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -10,7 +10,6 @@
LRU::Cache<std::string, event_cache_t> events_cache;
#define EVENTS_RATE_LIMIT_SEC 60
#define EVENTS_RATE_LIMIT_COUNT 5
-#define EVENTS_TTL_INTERVAL_US 2592000000000 //30days
Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk) {
/*
@@ -303,7 +302,7 @@ Option<bool> AnalyticsManager::add_event(const std::string& event_type, const st
auto &events_vec= query_collection_events[query_collection];
#ifdef TEST_BUILD
- if (isRateLimitEnabled) {
+ if (isRateLimitTestEnabled) {
#endif
auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
@@ -334,17 +333,19 @@ Option<bool> AnalyticsManager::add_event(const std::string& event_type, const st
event_t event(query, event_type, now_ts_useconds, user_id, doc_id, position);
events_vec.emplace_back(event);
- auto counter_events_it = counter_events.find(query_collection);
- if(counter_events_it != counter_events.end()) {
- auto event_weight_map_it = counter_events_it->second.event_weight_map.find(event_type);
- if(event_weight_map_it != counter_events_it->second.event_weight_map.end()) {
- auto inc_val = event_weight_map_it->second;
- counter_events_it->second.docid_counts[doc_id]+= inc_val;
+ if(!counter_events.empty()) {
+ auto counter_events_it = counter_events.find(query_collection);
+ if (counter_events_it != counter_events.end()) {
+ auto event_weight_map_it = counter_events_it->second.event_weight_map.find(event_type);
+ if (event_weight_map_it != counter_events_it->second.event_weight_map.end()) {
+ auto inc_val = event_weight_map_it->second;
+ counter_events_it->second.docid_counts[doc_id] += inc_val;
+ } else {
+ LOG(ERROR) << "event_type " << event_type << " not defined in analytic rule for counter events.";
+ }
} else {
- LOG(ERROR) << "event_type " << event_type << " not defined in analytic rule for counter events.";
+ LOG(ERROR) << "collection " << query_collection << " not found in analytics rule.";
}
- } else {
- LOG(ERROR) << "collection " << query_collection << " not found in analytics rule.";
}
return Option<bool>(true);
@@ -492,17 +493,27 @@ void AnalyticsManager::persist_query_events(ReplicationState *raft_server, uint6
void AnalyticsManager::persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {
// lock is held by caller
- for (const auto &events_collection_it: query_collection_events) {
+ for (const auto &events_collection_it: query_collection_events) {
for (const auto &event: events_collection_it.second) {
- if(analytics_logs.is_open()) {
+ if (analytics_logs.is_open()) {
//store events to log file
- char event_type_short = event.event_type == "query_click" ? 'C' : 'P';
-
- analytics_logs << event.timestamp << "\t" << event.user_id << "\t"
- << event_type_short << "\t" << event.query << "\t" << event.doc_id << "\n";
+ const auto collection = events_collection_it.first;
+ if (event.event_type == QUERY_CLICK) {
+ analytics_logs << event.timestamp << "\t" << event.user_id << "\t"
+ << 'C' << "\t" << event.doc_id << "\t"
+ << collection << event.query << "\t" << "\n";
+ } else if (event.event_type == QUERY_PURCHASE) {
+ analytics_logs << event.timestamp << "\t" << event.user_id << "\t"
+ << 'P' << "\t" << event.doc_id << "\t" << collection << "\n";
+ }
+ ++analytics_logs_count;
+ if(analytics_logs_count % 10 == 0) {
+ analytics_logs << std::flush;
+ }
}
}
}
+ query_collection_events.clear();
}
void AnalyticsManager::persist_popular_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {
@@ -534,6 +545,7 @@ void AnalyticsManager::persist_popular_events(ReplicationState *raft_server, uin
send_http_response(doc.dump(), coll);
}
}
+ counter_events.clear();
}
void AnalyticsManager::stop() {
@@ -604,5 +616,5 @@ nlohmann::json AnalyticsManager::get_events(const std::string& coll, const std::
void AnalyticsManager::resetToggleRateLimit(bool toggle) {
events_cache.clear();
- isRateLimitEnabled = toggle;
+ isRateLimitTestEnabled = toggle;
}
\ No newline at end of file
diff --git a/src/event_manager.cpp b/src/event_manager.cpp
index 3e4b6409..14dddaad 100644
--- a/src/event_manager.cpp
+++ b/src/event_manager.cpp
@@ -44,10 +44,10 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::str
}
for(const auto& coll: event_data_val["collections"]) {
- std::string query = event_data_query_it.get<std::string>();
+ const std::string& query = event_data_query_it.get<std::string>();
AnalyticsManager::get_instance().add_suggestion(coll.get<std::string>(), query, query, false, "");
}
- } else if(event_type == "query_click" || event_type == "query_purchase") {
+ } else if(event_type == AnalyticsManager::QUERY_CLICK || event_type == AnalyticsManager::QUERY_PURCHASE) {
if (!event.contains("data")) {
return Option<bool>(404, "key `data` not found.");
}
@@ -83,9 +83,9 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::str
return Option<bool>(500, "`collection` value should be string.");
}
- const std::string query = event_data_val["q"].get<std::string>();
- const std::string user_id = event_data_val["user_id"].get<std::string>();
- const std::string doc_id = event_data_val["doc_id"].get<std::string>();
+ const std::string& query = event_data_val["q"].get<std::string>();
+ const std::string& user_id = event_data_val["user_id"].get<std::string>();
+ const std::string& doc_id = event_data_val["doc_id"].get<std::string>();
uint64_t position = event_data_val["position"].get<uint64_t>();
const std::string& collection = event_data_val["collection"].get<std::string>();