Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
generify all events
krunal5115419e1ca4
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 28a45293..5beee7e6 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -7,28 +7,31 @@#include <unordered_map>#include <shared_mutex>-struct click_event_t {+struct event_t {std::string query;+ std::string event_type;uint64_t timestamp;std::string user_id;std::string doc_id;uint64_t position;- click_event_t() = delete;+ event_t() = delete;- ~click_event_t() = default;+ ~event_t() = default;- click_event_t(std::string q, uint64_t ts, std::string uid, std::string id, uint64_t pos) {+ event_t(std::string q, std::string type, uint64_t ts, std::string uid, std::string id, uint64_t pos) {query = q;+ event_type = type;timestamp = ts;user_id = uid;doc_id = id;position = pos;}- click_event_t& operator=(click_event_t& other) {+ event_t& operator=(event_t& other) {if (this != &other) {query = other.query;+ event_type = other.event_type;timestamp = other.timestamp;user_id = other.user_id;doc_id = other.doc_id;@@ -39,6 +42,7 @@ struct click_event_t {void to_json(nlohmann::json& obj) const {obj["query"] = query;+ obj["event_type"] = event_type;obj["timestamp"] = timestamp;obj["user_id"] = user_id;obj["doc_id"] = doc_id;@@ -151,11 +155,8 @@ private:// collection => popular clicksstd::unordered_map<std::string, popular_clicks_t> popular_clicks;- //query collection => click events- std::unordered_map<std::string, std::vector<click_event_t>> query_collection_click_events;-- //query collection=>purchase events- std::unordered_map<std::string, std::vector<click_event_t>> query_collection_special_events;+ //query collection => events+ std::unordered_map<std::string, std::vector<event_t>> query_collection_events;//query collection => query hits countstd::unordered_map<std::string, std::set<query_hits_count_t, query_hits_count_comp>> query_collection_hits_count;@@ -178,7 +179,7 @@ public:static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";static constexpr const char* CLICK_EVENT = "$CE";static constexpr const char* QUERY_HITS_COUNT = "$QH";- static constexpr const char* SPECIAL_EVENT = "$SE";+ static constexpr const char* PURCHASE_EVENT = "$PE";static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";static constexpr const char* NOHITS_QUERIES_TYPE = "nohits_queries";static constexpr const char* POPULAR_CLICKS_TYPE = "popular_clicks";@@ -217,14 +218,14 @@ public:std::unordered_map<std::string, QueryAnalytics*> get_popular_queries();- Option<bool> add_click_special_event(const std::string& event_type, const std::string& query_collection, const std::string& query, const std::string& user_id,+ Option<bool> add_event(const std::string& event_type, const std::string& query_collection, const std::string& query, const std::string& user_id,std::string doc_id, uint64_t position, const std::string& client_ip);- void persist_other_events(ReplicationState *raft_server, uint64_t prev_persistence_s);+ void persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s);void persist_popular_clicks(ReplicationState *raft_server, uint64_t prev_persistence_s);- nlohmann::json get_other_events(const std::string& event_type);+ nlohmann::json get_events(const std::string& event_type);std::unordered_map<std::string, popular_clicks_t> get_popular_clicks();diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex a705cd66..b14cd2c6 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -8,8 +8,8 @@#include "string_utils.h"LRU::Cache<std::string, event_cache_t> events_cache;-#define CLICK_EVENTS_RATE_LIMIT_SEC 60-#define CLICK_EVENTS_RATE_LIMIT_COUNT 5+#define EVENTS_RATE_LIMIT_SEC 60+#define EVENTS_RATE_LIMIT_COUNT 5#define EVENTS_TTL_INTERVAL_US 2592000000000 //30daysOption<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk) {@@ -279,12 +279,11 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection,}}-Option<bool> AnalyticsManager::add_click_special_event(const std::string& event_type, const std::string &query_collection, const std::string &query, const std::string &user_id,+Option<bool> AnalyticsManager::add_event(const std::string& event_type, const std::string &query_collection, const std::string &query, const std::string &user_id,std::string doc_id, uint64_t position, const std::string& client_ip) {std::unique_lock lock(mutex);if(analytics_store) {- auto &events_vec = event_type == "query_click" ?- query_collection_click_events[query_collection] : query_collection_special_events[query_collection];+ auto &events_vec= query_collection_events[query_collection];auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();@@ -292,9 +291,9 @@ Option<bool> AnalyticsManager::add_click_special_event(const std::string& event_if (events_cache_it != events_cache.end()) {//event found in events cache- if ((now_ts_seconds - events_cache_it->second.last_update_time) < CLICK_EVENTS_RATE_LIMIT_SEC) {- if (events_cache_it->second.count >= CLICK_EVENTS_RATE_LIMIT_COUNT) {- return Option<bool>(500, "click event rate limit reached.");+ if ((now_ts_seconds - events_cache_it->second.last_update_time) < EVENTS_RATE_LIMIT_SEC) {+ if (events_cache_it->second.count >= EVENTS_RATE_LIMIT_COUNT) {+ return Option<bool>(500, "event rate limit reached.");} else {events_cache_it->second.count++;}@@ -310,8 +309,8 @@ Option<bool> AnalyticsManager::add_click_special_event(const std::string& event_auto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();- click_event_t click_event(query, now_ts_useconds, user_id, doc_id, position);- events_vec.emplace_back(click_event);+ event_t event(query, event_type, now_ts_useconds, user_id, doc_id, position);+ events_vec.emplace_back(event);auto popular_clicks_it = popular_clicks.find(query_collection);if(popular_clicks_it != popular_clicks.end()) {@@ -387,7 +386,7 @@ void AnalyticsManager::run(ReplicationState* raft_server) {checkEventsExpiry();persist_query_events(raft_server, prev_persistence_s);- persist_other_events(raft_server, prev_persistence_s);+ persist_events(raft_server, prev_persistence_s);persist_popular_clicks(raft_server, prev_persistence_s);prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(@@ -483,11 +482,11 @@ void AnalyticsManager::persist_query_events(ReplicationState *raft_server, uint6}}-void AnalyticsManager::persist_other_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {+void AnalyticsManager::persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {// lock is held by callernlohmann::json payload_json = nlohmann::json::array();- auto send_http_response = [&](const std::string& event_type)->bool {+ auto send_http_response = [&]()->bool {if(payload_json.empty()) {return false;}@@ -505,7 +504,7 @@ void AnalyticsManager::persist_other_events(ReplicationState *raft_server, uint6res, res_headers, {}, 10 * 1000, true);if (status_code != 200) {- LOG(ERROR) << "Error while sending " << event_type <<" to leader. "+ LOG(ERROR) << "Error while sending events to leader. "<< "Status code: " << status_code << ", response: " << res;return false;}@@ -514,21 +513,20 @@ void AnalyticsManager::persist_other_events(ReplicationState *raft_server, uint6return false;};- for (const auto &click_events_collection_it: query_collection_click_events) {+ for (const auto &events_collection_it: query_collection_events) {auto collection_id = CollectionManager::get_instance().get_collection(- click_events_collection_it.first)->get_collection_id();- for (const auto &click_event: click_events_collection_it.second) {+ events_collection_it.first)->get_collection_id();+ for (const auto &event: events_collection_it.second) {// send http request- nlohmann::json click_event_json;- click_event.to_json(click_event_json);- click_event_json["collection_id"] = std::to_string(collection_id);- click_event_json["event_type"] = "click_events";- payload_json.push_back(click_event_json);+ nlohmann::json event_json;+ event.to_json(event_json);+ event_json["collection_id"] = std::to_string(collection_id);+ payload_json.push_back(event_json);}}- if(send_http_response("click_events")) {- query_collection_click_events.clear();+ if(send_http_response()) {+ query_collection_events.clear();}@@ -546,29 +544,11 @@ void AnalyticsManager::persist_other_events(ReplicationState *raft_server, uint6payload_json.push_back(query_hits_count_json);}}- if(send_http_response("query_hits_counts")) {+ if(send_http_response()) {query_collection_hits_count.clear();}payload_json.clear();--- for (const auto &special_events_collection_it: query_collection_special_events) {- auto collection_id = CollectionManager::get_instance().get_collection(- special_events_collection_it.first)->get_collection_id();- for (const auto &special_event: special_events_collection_it.second) {- // send http request- nlohmann::json special_event_json;- special_event.to_json(special_event_json);- special_event_json["collection_id"] = std::to_string(collection_id);- special_event_json["event_type"] = "special_events";- payload_json.push_back(special_event_json);- }- }-- if(send_http_response("special_events")) {- query_collection_special_events.clear();- }}void AnalyticsManager::persist_popular_clicks(ReplicationState *raft_server, uint64_t prev_persistence_s) {@@ -647,13 +627,14 @@ std::unordered_map<std::string, popular_clicks_t> AnalyticsManager::get_popular_return popular_clicks;}-nlohmann::json AnalyticsManager::get_other_events(const std::string& event_type) {+nlohmann::json AnalyticsManager::get_events(const std::string& event_type) {std::unique_lock lk(mutex);std::vector<std::string> event_jsons;nlohmann::json result_json = nlohmann::json::array();if (analytics_store) {- auto event_prefix = event_type.find("click_events") != std::string::npos ? std::string(CLICK_EVENT) : std::string(SPECIAL_EVENT);+ auto event_prefix = event_type.find("click_events") != std::string::npos ? std::string(CLICK_EVENT)+ : std::string(PURCHASE_EVENT);analytics_store->scan_fill(event_prefix + "_", event_prefix + "`",event_jsons);@@ -691,12 +672,12 @@ Option<bool> AnalyticsManager::write_events_to_store(nlohmann::json &event_jsonsauto timestamp = event_json["timestamp"].get<uint64_t>();std::string key = "_" + StringUtils::serialize_uint64_t(timestamp) + "_" + collection_id;- if(event_json["event_type"] == "click_events") {+ if(event_json["event_type"] == "query_click") {key = std::string(CLICK_EVENT) + key;} else if(event_json["event_type"] == "query_hits_counts") {key = std::string(QUERY_HITS_COUNT) + key;- } else if(event_json["event_type"] == "special_events") {- key = std::string(SPECIAL_EVENT) + key;+ } else if(event_json["event_type"] == "query_purchase") {+ key = std::string(PURCHASE_EVENT) + key;}if(analytics_store) {diff --git a/src/core_api.cpp b/src/core_api.cppindex cdcfe63f..dc7d3859 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -2801,16 +2801,16 @@ bool get_analytics_events(const std::shared_ptr<http_req>& req, const std::shareauto event_type = req->params["name"];if((event_type.find("click_events") == std::string::npos)- && (event_type.find("special_events") == std::string::npos)) {+ && (event_type.find("purchase_events") == std::string::npos)) {LOG(ERROR) << "Unknown event : " << event_type;- LOG(ERROR) << "get_analytics_events supports only click_events and special_events.";+ LOG(ERROR) << "get_analytics_events supports only click_events and purchase_events.";return true;}export_state_t *export_state = nullptr;- auto event_prefix = event_type.find("click_events") != std::string::npos ? std::string(AnalyticsManager::CLICK_EVENT)- : std::string(AnalyticsManager::SPECIAL_EVENT);+ auto event_prefix = event_type.find("click_event") != std::string::npos ? std::string(AnalyticsManager::CLICK_EVENT)+ : std::string(AnalyticsManager::PURCHASE_EVENT);if (req->data == nullptr) {export_state = new export_state_t();req->data = export_state;@@ -2828,7 +2828,7 @@ bool get_analytics_events(const std::shared_ptr<http_req>& req, const std::sharestd::string().swap(res->body);if (!it->Valid()) {- LOG(ERROR) << "No click events found in db.";+ LOG(ERROR) << "No events found in db.";req->last_chunk_aggregate = true;res->final = true;res->set_404();diff --git a/src/event_manager.cpp b/src/event_manager.cppindex 9c097d3a..3e4b6409 100644--- a/src/event_manager.cpp+++ b/src/event_manager.cpp@@ -47,7 +47,7 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::strstd::string query = event_data_query_it.get<std::string>();AnalyticsManager::get_instance().add_suggestion(coll.get<std::string>(), query, query, false, "");}- } else if(event_type == "query_click" || event_type == "query_special") {+ } else if(event_type == "query_click" || event_type == "query_purchase") {if (!event.contains("data")) {return Option<bool>(404, "key `data` not found.");}@@ -89,7 +89,7 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::struint64_t position = event_data_val["position"].get<uint64_t>();const std::string& collection = event_data_val["collection"].get<std::string>();- auto op = AnalyticsManager::get_instance().add_click_special_event(event_type, collection, query, user_id, doc_id, position, client_ip);+ auto op = AnalyticsManager::get_instance().add_event(event_type, collection, query, user_id, doc_id, position, client_ip);if(!op.ok()) {return Option<bool>(op.code(), op.error());}diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex f6832cbf..6ffe8d22 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -377,10 +377,10 @@ TEST_F(AnalyticsManagerTest, ClickEventsStoreRetrieveal) {event1["collection_id"] = "0";event1["timestamp"] = 1521512521;- event1["event_type"] = "click_events";+ event1["event_type"] = "query_click";event2["collection_id"] = "0";event2["timestamp"] = 1521514354;- event2["event_type"] = "click_events";+ event2["event_type"] = "query_click";nlohmann::json click_events = nlohmann::json::array();click_events.push_back(event1);@@ -389,7 +389,7 @@ TEST_F(AnalyticsManagerTest, ClickEventsStoreRetrieveal) {req->body = click_events.dump();ASSERT_TRUE(post_replicate_events(req, res));- auto result = analyticsManager.get_other_events("click_events");+ auto result = analyticsManager.get_events("click_events");ASSERT_EQ("0", result[0]["collection_id"]);ASSERT_EQ("13", result[0]["data"]["user_id"]);@@ -438,7 +438,7 @@ TEST_F(AnalyticsManagerTest, ClickEventsRateLimitTest) {//as rate limit is 5, adding one more event above that should trigger rate limitASSERT_FALSE(post_create_event(req, res));- ASSERT_EQ("{\"message\": \"click event rate limit reached.\"}", res->body);+ ASSERT_EQ("{\"message\": \"event rate limit reached.\"}", res->body);}TEST_F(AnalyticsManagerTest, NoresultsQueries) {@@ -647,7 +647,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {auto ts = 1701851341000000;nlohmann::json event;- event["event_type"] = "click_events";+ event["event_type"] = "query_click";event["q"] = "technology";event["collection_id"] = "0";event["doc_id"] = "21";@@ -663,7 +663,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());- nlohmann::json resp = analyticsManager.get_other_events("click_events");+ nlohmann::json resp = analyticsManager.get_events("click_events");ASSERT_EQ(2, resp.size());ASSERT_EQ("technology", resp[0]["q"]);ASSERT_EQ("21", resp[0]["doc_id"]);@@ -677,7 +677,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {analyticsManager.checkEventsExpiry();- resp = analyticsManager.get_other_events("click_events");+ resp = analyticsManager.get_events("click_events");ASSERT_EQ(0, resp.size());//add query hits events with click events@@ -703,7 +703,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {// assumming ttl is ts + 5 secanalyticsManager.checkEventsExpiry();- resp = analyticsManager.get_other_events("click_events");+ resp = analyticsManager.get_events("click_events");ASSERT_EQ(0, resp.size());resp = analyticsManager.get_query_hits_counts();@@ -730,7 +730,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryAll) {auto ts = 1701851341000000;nlohmann::json event;- event["event_type"] = "click_events";+ event["event_type"] = "query_click";event["q"] = "technology";event["collection_id"] = "0";event["doc_id"] = "21";@@ -756,7 +756,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryAll) {ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());- nlohmann::json resp = analyticsManager.get_other_events("click_events");+ nlohmann::json resp = analyticsManager.get_events("click_events");ASSERT_EQ(2, resp.size());ASSERT_EQ("technology", resp[0]["q"]);ASSERT_EQ("21", resp[0]["doc_id"]);@@ -778,7 +778,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryAll) {// assuming ttl is ts + 5 secanalyticsManager.checkEventsExpiry();- resp = analyticsManager.get_other_events("click_events");+ resp = analyticsManager.get_events("click_events");ASSERT_EQ(0, resp.size());resp = analyticsManager.get_query_hits_counts();@@ -802,7 +802,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryPartial) {auto ts = 1701851341000000;nlohmann::json event;- event["event_type"] = "click_events";+ event["event_type"] = "query_click";event["q"] = "technology";event["collection_id"] = "0";event["doc_id"] = "21";@@ -844,7 +844,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryPartial) {ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());- nlohmann::json resp = analyticsManager.get_other_events("click_events");+ nlohmann::json resp = analyticsManager.get_events("click_events");ASSERT_EQ(3, resp.size());ASSERT_EQ("technology", resp[0]["q"]);ASSERT_EQ("21", resp[0]["doc_id"]);@@ -880,7 +880,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryPartial) {//only events in ttl interval will be removedanalyticsManager.checkEventsExpiry();- resp = analyticsManager.get_other_events("click_events");+ resp = analyticsManager.get_events("click_events");ASSERT_EQ(1, resp.size());ASSERT_EQ("technology", resp[0]["q"]);ASSERT_EQ("19", resp[0]["doc_id"]);@@ -1068,7 +1068,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {ASSERT_EQ("counter_field `popularity_score` not found in destination collection.", create_op.error());}-TEST_F(AnalyticsManagerTest, SpecialEventsStoreRetrieval) {+TEST_F(AnalyticsManagerTest, PurchaseEventsStoreRetrieval) {nlohmann::json titles_schema = R"({"name": "titles","fields": [@@ -1082,7 +1082,7 @@ TEST_F(AnalyticsManagerTest, SpecialEventsStoreRetrieval) {std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);nlohmann::json event1 = R"({- "type": "query_special",+ "type": "query_purchase","data": {"q": "technology","collection": "titles",@@ -1096,7 +1096,7 @@ TEST_F(AnalyticsManagerTest, SpecialEventsStoreRetrieval) {ASSERT_TRUE(post_create_event(req, res));nlohmann::json event2 = R"({- "type": "query_special",+ "type": "query_purchase","data": {"q": "technology","collection": "titles",@@ -1111,19 +1111,19 @@ TEST_F(AnalyticsManagerTest, SpecialEventsStoreRetrieval) {event1["collection_id"] = "0";event1["timestamp"] = 1521512521;- event1["event_type"] = "special_events";+ event1["event_type"] = "query_purchase";event2["collection_id"] = "0";event2["timestamp"] = 1521514354;- event2["event_type"] = "special_events";+ event2["event_type"] = "query_purchase";- nlohmann::json click_events = nlohmann::json::array();- click_events.push_back(event1);- click_events.push_back(event2);+ nlohmann::json events = nlohmann::json::array();+ events.push_back(event1);+ events.push_back(event2);- req->body = click_events.dump();+ req->body = events.dump();ASSERT_TRUE(post_replicate_events(req, res));- auto result = analyticsManager.get_other_events("special_events");+ auto result = analyticsManager.get_events("purchase_events");ASSERT_EQ("0", result[0]["collection_id"]);ASSERT_EQ("13", result[0]["data"]["user_id"]);diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex fd8be7b1..fc81eb11 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -1567,13 +1567,13 @@ TEST_F(CoreAPIUtilsTest, GetClickEvents) {event1["collection_id"] = "0";event1["timestamp"] = 1521512521;- event1["event_type"] = "click_events";+ event1["event_type"] = "query_click";event2["collection_id"] = "0";event2["timestamp"] = 1521514354;- event2["event_type"] = "click_events";+ event2["event_type"] = "query_click";event3["collection_id"] = "0";event3["timestamp"] = 1521515382;- event3["event_type"] = "click_events";+ event3["event_type"] = "query_click";nlohmann::json click_events = nlohmann::json::array();