Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

generify all events

krunal5115419e1ca4
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index 28a45293..5beee7e6 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -7,28 +7,31 @@
#include <unordered_map>
#include <shared_mutex>
-struct click_event_t {
+struct event_t {
std::string query;
+ std::string event_type;
uint64_t timestamp;
std::string user_id;
std::string doc_id;
uint64_t position;
- click_event_t() = delete;
+ event_t() = delete;
- ~click_event_t() = default;
+ ~event_t() = default;
- click_event_t(std::string q, uint64_t ts, std::string uid, std::string id, uint64_t pos) {
+ event_t(std::string q, std::string type, uint64_t ts, std::string uid, std::string id, uint64_t pos) {
query = q;
+ event_type = type;
timestamp = ts;
user_id = uid;
doc_id = id;
position = pos;
}
- click_event_t& operator=(click_event_t& other) {
+ event_t& operator=(event_t& other) {
if (this != &other) {
query = other.query;
+ event_type = other.event_type;
timestamp = other.timestamp;
user_id = other.user_id;
doc_id = other.doc_id;
@@ -39,6 +42,7 @@ struct click_event_t {
void to_json(nlohmann::json& obj) const {
obj["query"] = query;
+ obj["event_type"] = event_type;
obj["timestamp"] = timestamp;
obj["user_id"] = user_id;
obj["doc_id"] = doc_id;
@@ -151,11 +155,8 @@ private:
// collection => popular clicks
std::unordered_map<std::string, popular_clicks_t> popular_clicks;
- //query collection => click events
- std::unordered_map<std::string, std::vector<click_event_t>> query_collection_click_events;
-
- //query collection=>purchase events
- std::unordered_map<std::string, std::vector<click_event_t>> query_collection_special_events;
+ //query collection => events
+ std::unordered_map<std::string, std::vector<event_t>> query_collection_events;
//query collection => query hits count
std::unordered_map<std::string, std::set<query_hits_count_t, query_hits_count_comp>> query_collection_hits_count;
@@ -178,7 +179,7 @@ public:
static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";
static constexpr const char* CLICK_EVENT = "$CE";
static constexpr const char* QUERY_HITS_COUNT = "$QH";
- static constexpr const char* SPECIAL_EVENT = "$SE";
+ static constexpr const char* PURCHASE_EVENT = "$PE";
static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";
static constexpr const char* NOHITS_QUERIES_TYPE = "nohits_queries";
static constexpr const char* POPULAR_CLICKS_TYPE = "popular_clicks";
@@ -217,14 +218,14 @@ public:
std::unordered_map<std::string, QueryAnalytics*> get_popular_queries();
- Option<bool> add_click_special_event(const std::string& event_type, const std::string& query_collection, const std::string& query, const std::string& user_id,
+ Option<bool> add_event(const std::string& event_type, const std::string& query_collection, const std::string& query, const std::string& user_id,
std::string doc_id, uint64_t position, const std::string& client_ip);
- void persist_other_events(ReplicationState *raft_server, uint64_t prev_persistence_s);
+ void persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s);
void persist_popular_clicks(ReplicationState *raft_server, uint64_t prev_persistence_s);
- nlohmann::json get_other_events(const std::string& event_type);
+ nlohmann::json get_events(const std::string& event_type);
std::unordered_map<std::string, popular_clicks_t> get_popular_clicks();
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index a705cd66..b14cd2c6 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -8,8 +8,8 @@
#include "string_utils.h"
LRU::Cache<std::string, event_cache_t> events_cache;
-#define CLICK_EVENTS_RATE_LIMIT_SEC 60
-#define CLICK_EVENTS_RATE_LIMIT_COUNT 5
+#define EVENTS_RATE_LIMIT_SEC 60
+#define EVENTS_RATE_LIMIT_COUNT 5
#define EVENTS_TTL_INTERVAL_US 2592000000000 //30days
Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk) {
@@ -279,12 +279,11 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection,
}
}
-Option<bool> AnalyticsManager::add_click_special_event(const std::string& event_type, const std::string &query_collection, const std::string &query, const std::string &user_id,
+Option<bool> AnalyticsManager::add_event(const std::string& event_type, const std::string &query_collection, const std::string &query, const std::string &user_id,
std::string doc_id, uint64_t position, const std::string& client_ip) {
std::unique_lock lock(mutex);
if(analytics_store) {
- auto &events_vec = event_type == "query_click" ?
- query_collection_click_events[query_collection] : query_collection_special_events[query_collection];
+ auto &events_vec= query_collection_events[query_collection];
auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
@@ -292,9 +291,9 @@ Option<bool> AnalyticsManager::add_click_special_event(const std::string& event_
if (events_cache_it != events_cache.end()) {
//event found in events cache
- if ((now_ts_seconds - events_cache_it->second.last_update_time) < CLICK_EVENTS_RATE_LIMIT_SEC) {
- if (events_cache_it->second.count >= CLICK_EVENTS_RATE_LIMIT_COUNT) {
- return Option<bool>(500, "click event rate limit reached.");
+ if ((now_ts_seconds - events_cache_it->second.last_update_time) < EVENTS_RATE_LIMIT_SEC) {
+ if (events_cache_it->second.count >= EVENTS_RATE_LIMIT_COUNT) {
+ return Option<bool>(500, "event rate limit reached.");
} else {
events_cache_it->second.count++;
}
@@ -310,8 +309,8 @@ Option<bool> AnalyticsManager::add_click_special_event(const std::string& event_
auto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
- click_event_t click_event(query, now_ts_useconds, user_id, doc_id, position);
- events_vec.emplace_back(click_event);
+ event_t event(query, event_type, now_ts_useconds, user_id, doc_id, position);
+ events_vec.emplace_back(event);
auto popular_clicks_it = popular_clicks.find(query_collection);
if(popular_clicks_it != popular_clicks.end()) {
@@ -387,7 +386,7 @@ void AnalyticsManager::run(ReplicationState* raft_server) {
checkEventsExpiry();
persist_query_events(raft_server, prev_persistence_s);
- persist_other_events(raft_server, prev_persistence_s);
+ persist_events(raft_server, prev_persistence_s);
persist_popular_clicks(raft_server, prev_persistence_s);
prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(
@@ -483,11 +482,11 @@ void AnalyticsManager::persist_query_events(ReplicationState *raft_server, uint6
}
}
-void AnalyticsManager::persist_other_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {
+void AnalyticsManager::persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {
// lock is held by caller
nlohmann::json payload_json = nlohmann::json::array();
- auto send_http_response = [&](const std::string& event_type)->bool {
+ auto send_http_response = [&]()->bool {
if(payload_json.empty()) {
return false;
}
@@ -505,7 +504,7 @@ void AnalyticsManager::persist_other_events(ReplicationState *raft_server, uint6
res, res_headers, {}, 10 * 1000, true);
if (status_code != 200) {
- LOG(ERROR) << "Error while sending " << event_type <<" to leader. "
+ LOG(ERROR) << "Error while sending events to leader. "
<< "Status code: " << status_code << ", response: " << res;
return false;
}
@@ -514,21 +513,20 @@ void AnalyticsManager::persist_other_events(ReplicationState *raft_server, uint6
return false;
};
- for (const auto &click_events_collection_it: query_collection_click_events) {
+ for (const auto &events_collection_it: query_collection_events) {
auto collection_id = CollectionManager::get_instance().get_collection(
- click_events_collection_it.first)->get_collection_id();
- for (const auto &click_event: click_events_collection_it.second) {
+ events_collection_it.first)->get_collection_id();
+ for (const auto &event: events_collection_it.second) {
// send http request
- nlohmann::json click_event_json;
- click_event.to_json(click_event_json);
- click_event_json["collection_id"] = std::to_string(collection_id);
- click_event_json["event_type"] = "click_events";
- payload_json.push_back(click_event_json);
+ nlohmann::json event_json;
+ event.to_json(event_json);
+ event_json["collection_id"] = std::to_string(collection_id);
+ payload_json.push_back(event_json);
}
}
- if(send_http_response("click_events")) {
- query_collection_click_events.clear();
+ if(send_http_response()) {
+ query_collection_events.clear();
}
@@ -546,29 +544,11 @@ void AnalyticsManager::persist_other_events(ReplicationState *raft_server, uint6
payload_json.push_back(query_hits_count_json);
}
}
- if(send_http_response("query_hits_counts")) {
+ if(send_http_response()) {
query_collection_hits_count.clear();
}
payload_json.clear();
-
-
- for (const auto &special_events_collection_it: query_collection_special_events) {
- auto collection_id = CollectionManager::get_instance().get_collection(
- special_events_collection_it.first)->get_collection_id();
- for (const auto &special_event: special_events_collection_it.second) {
- // send http request
- nlohmann::json special_event_json;
- special_event.to_json(special_event_json);
- special_event_json["collection_id"] = std::to_string(collection_id);
- special_event_json["event_type"] = "special_events";
- payload_json.push_back(special_event_json);
- }
- }
-
- if(send_http_response("special_events")) {
- query_collection_special_events.clear();
- }
}
void AnalyticsManager::persist_popular_clicks(ReplicationState *raft_server, uint64_t prev_persistence_s) {
@@ -647,13 +627,14 @@ std::unordered_map<std::string, popular_clicks_t> AnalyticsManager::get_popular_
return popular_clicks;
}
-nlohmann::json AnalyticsManager::get_other_events(const std::string& event_type) {
+nlohmann::json AnalyticsManager::get_events(const std::string& event_type) {
std::unique_lock lk(mutex);
std::vector<std::string> event_jsons;
nlohmann::json result_json = nlohmann::json::array();
if (analytics_store) {
- auto event_prefix = event_type.find("click_events") != std::string::npos ? std::string(CLICK_EVENT) : std::string(SPECIAL_EVENT);
+ auto event_prefix = event_type.find("click_events") != std::string::npos ? std::string(CLICK_EVENT)
+ : std::string(PURCHASE_EVENT);
analytics_store->scan_fill(event_prefix + "_", event_prefix + "`",
event_jsons);
@@ -691,12 +672,12 @@ Option<bool> AnalyticsManager::write_events_to_store(nlohmann::json &event_jsons
auto timestamp = event_json["timestamp"].get<uint64_t>();
std::string key = "_" + StringUtils::serialize_uint64_t(timestamp) + "_" + collection_id;
- if(event_json["event_type"] == "click_events") {
+ if(event_json["event_type"] == "query_click") {
key = std::string(CLICK_EVENT) + key;
} else if(event_json["event_type"] == "query_hits_counts") {
key = std::string(QUERY_HITS_COUNT) + key;
- } else if(event_json["event_type"] == "special_events") {
- key = std::string(SPECIAL_EVENT) + key;
+ } else if(event_json["event_type"] == "query_purchase") {
+ key = std::string(PURCHASE_EVENT) + key;
}
if(analytics_store) {
diff --git a/src/core_api.cpp b/src/core_api.cpp
index cdcfe63f..dc7d3859 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -2801,16 +2801,16 @@ bool get_analytics_events(const std::shared_ptr<http_req>& req, const std::share
auto event_type = req->params["name"];
if((event_type.find("click_events") == std::string::npos)
- && (event_type.find("special_events") == std::string::npos)) {
+ && (event_type.find("purchase_events") == std::string::npos)) {
LOG(ERROR) << "Unknown event : " << event_type;
- LOG(ERROR) << "get_analytics_events supports only click_events and special_events.";
+ LOG(ERROR) << "get_analytics_events supports only click_events and purchase_events.";
return true;
}
export_state_t *export_state = nullptr;
- auto event_prefix = event_type.find("click_events") != std::string::npos ? std::string(AnalyticsManager::CLICK_EVENT)
- : std::string(AnalyticsManager::SPECIAL_EVENT);
+ auto event_prefix = event_type.find("click_event") != std::string::npos ? std::string(AnalyticsManager::CLICK_EVENT)
+ : std::string(AnalyticsManager::PURCHASE_EVENT);
if (req->data == nullptr) {
export_state = new export_state_t();
req->data = export_state;
@@ -2828,7 +2828,7 @@ bool get_analytics_events(const std::shared_ptr<http_req>& req, const std::share
std::string().swap(res->body);
if (!it->Valid()) {
- LOG(ERROR) << "No click events found in db.";
+ LOG(ERROR) << "No events found in db.";
req->last_chunk_aggregate = true;
res->final = true;
res->set_404();
diff --git a/src/event_manager.cpp b/src/event_manager.cpp
index 9c097d3a..3e4b6409 100644
--- a/src/event_manager.cpp
+++ b/src/event_manager.cpp
@@ -47,7 +47,7 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::str
std::string query = event_data_query_it.get<std::string>();
AnalyticsManager::get_instance().add_suggestion(coll.get<std::string>(), query, query, false, "");
}
- } else if(event_type == "query_click" || event_type == "query_special") {
+ } else if(event_type == "query_click" || event_type == "query_purchase") {
if (!event.contains("data")) {
return Option<bool>(404, "key `data` not found.");
}
@@ -89,7 +89,7 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::str
uint64_t position = event_data_val["position"].get<uint64_t>();
const std::string& collection = event_data_val["collection"].get<std::string>();
- auto op = AnalyticsManager::get_instance().add_click_special_event(event_type, collection, query, user_id, doc_id, position, client_ip);
+ auto op = AnalyticsManager::get_instance().add_event(event_type, collection, query, user_id, doc_id, position, client_ip);
if(!op.ok()) {
return Option<bool>(op.code(), op.error());
}
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index f6832cbf..6ffe8d22 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -377,10 +377,10 @@ TEST_F(AnalyticsManagerTest, ClickEventsStoreRetrieveal) {
event1["collection_id"] = "0";
event1["timestamp"] = 1521512521;
- event1["event_type"] = "click_events";
+ event1["event_type"] = "query_click";
event2["collection_id"] = "0";
event2["timestamp"] = 1521514354;
- event2["event_type"] = "click_events";
+ event2["event_type"] = "query_click";
nlohmann::json click_events = nlohmann::json::array();
click_events.push_back(event1);
@@ -389,7 +389,7 @@ TEST_F(AnalyticsManagerTest, ClickEventsStoreRetrieveal) {
req->body = click_events.dump();
ASSERT_TRUE(post_replicate_events(req, res));
- auto result = analyticsManager.get_other_events("click_events");
+ auto result = analyticsManager.get_events("click_events");
ASSERT_EQ("0", result[0]["collection_id"]);
ASSERT_EQ("13", result[0]["data"]["user_id"]);
@@ -438,7 +438,7 @@ TEST_F(AnalyticsManagerTest, ClickEventsRateLimitTest) {
//as rate limit is 5, adding one more event above that should trigger rate limit
ASSERT_FALSE(post_create_event(req, res));
- ASSERT_EQ("{\"message\": \"click event rate limit reached.\"}", res->body);
+ ASSERT_EQ("{\"message\": \"event rate limit reached.\"}", res->body);
}
TEST_F(AnalyticsManagerTest, NoresultsQueries) {
@@ -647,7 +647,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {
auto ts = 1701851341000000;
nlohmann::json event;
- event["event_type"] = "click_events";
+ event["event_type"] = "query_click";
event["q"] = "technology";
event["collection_id"] = "0";
event["doc_id"] = "21";
@@ -663,7 +663,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {
ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());
- nlohmann::json resp = analyticsManager.get_other_events("click_events");
+ nlohmann::json resp = analyticsManager.get_events("click_events");
ASSERT_EQ(2, resp.size());
ASSERT_EQ("technology", resp[0]["q"]);
ASSERT_EQ("21", resp[0]["doc_id"]);
@@ -677,7 +677,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {
analyticsManager.checkEventsExpiry();
- resp = analyticsManager.get_other_events("click_events");
+ resp = analyticsManager.get_events("click_events");
ASSERT_EQ(0, resp.size());
//add query hits events with click events
@@ -703,7 +703,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {
// assumming ttl is ts + 5 sec
analyticsManager.checkEventsExpiry();
- resp = analyticsManager.get_other_events("click_events");
+ resp = analyticsManager.get_events("click_events");
ASSERT_EQ(0, resp.size());
resp = analyticsManager.get_query_hits_counts();
@@ -730,7 +730,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryAll) {
auto ts = 1701851341000000;
nlohmann::json event;
- event["event_type"] = "click_events";
+ event["event_type"] = "query_click";
event["q"] = "technology";
event["collection_id"] = "0";
event["doc_id"] = "21";
@@ -756,7 +756,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryAll) {
ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());
- nlohmann::json resp = analyticsManager.get_other_events("click_events");
+ nlohmann::json resp = analyticsManager.get_events("click_events");
ASSERT_EQ(2, resp.size());
ASSERT_EQ("technology", resp[0]["q"]);
ASSERT_EQ("21", resp[0]["doc_id"]);
@@ -778,7 +778,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryAll) {
// assuming ttl is ts + 5 sec
analyticsManager.checkEventsExpiry();
- resp = analyticsManager.get_other_events("click_events");
+ resp = analyticsManager.get_events("click_events");
ASSERT_EQ(0, resp.size());
resp = analyticsManager.get_query_hits_counts();
@@ -802,7 +802,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryPartial) {
auto ts = 1701851341000000;
nlohmann::json event;
- event["event_type"] = "click_events";
+ event["event_type"] = "query_click";
event["q"] = "technology";
event["collection_id"] = "0";
event["doc_id"] = "21";
@@ -844,7 +844,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryPartial) {
ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());
- nlohmann::json resp = analyticsManager.get_other_events("click_events");
+ nlohmann::json resp = analyticsManager.get_events("click_events");
ASSERT_EQ(3, resp.size());
ASSERT_EQ("technology", resp[0]["q"]);
ASSERT_EQ("21", resp[0]["doc_id"]);
@@ -880,7 +880,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryPartial) {
//only events in ttl interval will be removed
analyticsManager.checkEventsExpiry();
- resp = analyticsManager.get_other_events("click_events");
+ resp = analyticsManager.get_events("click_events");
ASSERT_EQ(1, resp.size());
ASSERT_EQ("technology", resp[0]["q"]);
ASSERT_EQ("19", resp[0]["doc_id"]);
@@ -1068,7 +1068,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
ASSERT_EQ("counter_field `popularity_score` not found in destination collection.", create_op.error());
}
-TEST_F(AnalyticsManagerTest, SpecialEventsStoreRetrieval) {
+TEST_F(AnalyticsManagerTest, PurchaseEventsStoreRetrieval) {
nlohmann::json titles_schema = R"({
"name": "titles",
"fields": [
@@ -1082,7 +1082,7 @@ TEST_F(AnalyticsManagerTest, SpecialEventsStoreRetrieval) {
std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
nlohmann::json event1 = R"({
- "type": "query_special",
+ "type": "query_purchase",
"data": {
"q": "technology",
"collection": "titles",
@@ -1096,7 +1096,7 @@ TEST_F(AnalyticsManagerTest, SpecialEventsStoreRetrieval) {
ASSERT_TRUE(post_create_event(req, res));
nlohmann::json event2 = R"({
- "type": "query_special",
+ "type": "query_purchase",
"data": {
"q": "technology",
"collection": "titles",
@@ -1111,19 +1111,19 @@ TEST_F(AnalyticsManagerTest, SpecialEventsStoreRetrieval) {
event1["collection_id"] = "0";
event1["timestamp"] = 1521512521;
- event1["event_type"] = "special_events";
+ event1["event_type"] = "query_purchase";
event2["collection_id"] = "0";
event2["timestamp"] = 1521514354;
- event2["event_type"] = "special_events";
+ event2["event_type"] = "query_purchase";
- nlohmann::json click_events = nlohmann::json::array();
- click_events.push_back(event1);
- click_events.push_back(event2);
+ nlohmann::json events = nlohmann::json::array();
+ events.push_back(event1);
+ events.push_back(event2);
- req->body = click_events.dump();
+ req->body = events.dump();
ASSERT_TRUE(post_replicate_events(req, res));
- auto result = analyticsManager.get_other_events("special_events");
+ auto result = analyticsManager.get_events("purchase_events");
ASSERT_EQ("0", result[0]["collection_id"]);
ASSERT_EQ("13", result[0]["data"]["user_id"]);
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index fd8be7b1..fc81eb11 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -1567,13 +1567,13 @@ TEST_F(CoreAPIUtilsTest, GetClickEvents) {
event1["collection_id"] = "0";
event1["timestamp"] = 1521512521;
- event1["event_type"] = "click_events";
+ event1["event_type"] = "query_click";
event2["collection_id"] = "0";
event2["timestamp"] = 1521514354;
- event2["event_type"] = "click_events";
+ event2["event_type"] = "query_click";
event3["collection_id"] = "0";
event3["timestamp"] = 1521515382;
- event3["event_type"] = "click_events";
+ event3["event_type"] = "query_click";
nlohmann::json click_events = nlohmann::json::array();