Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor analytics changes (#1502)
Krunal Gandhicde131f4b30a
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex c3642c6b..b5bdf4b2 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -121,7 +121,9 @@ private:Store* store = nullptr;std::ofstream analytics_logs;- bool isRateLimitEnabled = false;+ bool isRateLimitTestEnabled = false;+ uint64_t analytics_logs_count = 0;+AnalyticsManager() {}~AnalyticsManager();@@ -138,6 +140,8 @@ public:static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";static constexpr const char* NOHITS_QUERIES_TYPE = "nohits_queries";static constexpr const char* COUNTER_TYPE = "counter";+ static constexpr const char* QUERY_CLICK = "query_click";+ static constexpr const char* QUERY_PURCHASE = "query_purchase";static AnalyticsManager& get_instance() {static AnalyticsManager instance;diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex a897c744..53b15b74 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -10,7 +10,6 @@LRU::Cache<std::string, event_cache_t> events_cache;#define EVENTS_RATE_LIMIT_SEC 60#define EVENTS_RATE_LIMIT_COUNT 5-#define EVENTS_TTL_INTERVAL_US 2592000000000 //30daysOption<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk) {/*@@ -303,7 +302,7 @@ Option<bool> AnalyticsManager::add_event(const std::string& event_type, const stauto &events_vec= query_collection_events[query_collection];#ifdef TEST_BUILD- if (isRateLimitEnabled) {+ if (isRateLimitTestEnabled) {#endifauto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();@@ -334,17 +333,19 @@ Option<bool> AnalyticsManager::add_event(const std::string& event_type, const stevent_t event(query, event_type, now_ts_useconds, user_id, doc_id, position);events_vec.emplace_back(event);- auto counter_events_it = counter_events.find(query_collection);- if(counter_events_it != counter_events.end()) {- auto event_weight_map_it = counter_events_it->second.event_weight_map.find(event_type);- if(event_weight_map_it != counter_events_it->second.event_weight_map.end()) {- auto inc_val = event_weight_map_it->second;- counter_events_it->second.docid_counts[doc_id]+= inc_val;+ if(!counter_events.empty()) {+ auto counter_events_it = counter_events.find(query_collection);+ if (counter_events_it != counter_events.end()) {+ auto event_weight_map_it = counter_events_it->second.event_weight_map.find(event_type);+ if (event_weight_map_it != counter_events_it->second.event_weight_map.end()) {+ auto inc_val = event_weight_map_it->second;+ counter_events_it->second.docid_counts[doc_id] += inc_val;+ } else {+ LOG(ERROR) << "event_type " << event_type << " not defined in analytic rule for counter events.";+ }} else {- LOG(ERROR) << "event_type " << event_type << " not defined in analytic rule for counter events.";+ LOG(ERROR) << "collection " << query_collection << " not found in analytics rule.";}- } else {- LOG(ERROR) << "collection " << query_collection << " not found in analytics rule.";}return Option<bool>(true);@@ -492,17 +493,27 @@ void AnalyticsManager::persist_query_events(ReplicationState *raft_server, uint6void AnalyticsManager::persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {// lock is held by caller- for (const auto &events_collection_it: query_collection_events) {+ for (const auto &events_collection_it: query_collection_events) {for (const auto &event: events_collection_it.second) {- if(analytics_logs.is_open()) {+ if (analytics_logs.is_open()) {//store events to log file- char event_type_short = event.event_type == "query_click" ? 'C' : 'P';-- analytics_logs << event.timestamp << "\t" << event.user_id << "\t"- << event_type_short << "\t" << event.query << "\t" << event.doc_id << "\n";+ const auto collection = events_collection_it.first;+ if (event.event_type == QUERY_CLICK) {+ analytics_logs << event.timestamp << "\t" << event.user_id << "\t"+ << 'C' << "\t" << event.doc_id << "\t"+ << collection << event.query << "\t" << "\n";+ } else if (event.event_type == QUERY_PURCHASE) {+ analytics_logs << event.timestamp << "\t" << event.user_id << "\t"+ << 'P' << "\t" << event.doc_id << "\t" << collection << "\n";+ }+ ++analytics_logs_count;+ if(analytics_logs_count % 10 == 0) {+ analytics_logs << std::flush;+ }}}}+ query_collection_events.clear();}void AnalyticsManager::persist_popular_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {@@ -534,6 +545,7 @@ void AnalyticsManager::persist_popular_events(ReplicationState *raft_server, uinsend_http_response(doc.dump(), coll);}}+ counter_events.clear();}void AnalyticsManager::stop() {@@ -604,5 +616,5 @@ nlohmann::json AnalyticsManager::get_events(const std::string& coll, const std::void AnalyticsManager::resetToggleRateLimit(bool toggle) {events_cache.clear();- isRateLimitEnabled = toggle;+ isRateLimitTestEnabled = toggle;}\ No newline at end of filediff --git a/src/event_manager.cpp b/src/event_manager.cppindex 3e4b6409..14dddaad 100644--- a/src/event_manager.cpp+++ b/src/event_manager.cpp@@ -44,10 +44,10 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::str}for(const auto& coll: event_data_val["collections"]) {- std::string query = event_data_query_it.get<std::string>();+ const std::string& query = event_data_query_it.get<std::string>();AnalyticsManager::get_instance().add_suggestion(coll.get<std::string>(), query, query, false, "");}- } else if(event_type == "query_click" || event_type == "query_purchase") {+ } else if(event_type == AnalyticsManager::QUERY_CLICK || event_type == AnalyticsManager::QUERY_PURCHASE) {if (!event.contains("data")) {return Option<bool>(404, "key `data` not found.");}@@ -83,9 +83,9 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::strreturn Option<bool>(500, "`collection` value should be string.");}- const std::string query = event_data_val["q"].get<std::string>();- const std::string user_id = event_data_val["user_id"].get<std::string>();- const std::string doc_id = event_data_val["doc_id"].get<std::string>();+ const std::string& query = event_data_val["q"].get<std::string>();+ const std::string& user_id = event_data_val["user_id"].get<std::string>();+ const std::string& doc_id = event_data_val["doc_id"].get<std::string>();uint64_t position = event_data_val["position"].get<uint64_t>();const std::string& collection = event_data_val["collection"].get<std::string>();