Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
generify counter events with different events types
krunale379b72813ea
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 5beee7e6..4448bf9c 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -50,9 +50,10 @@ struct event_t {}};-struct popular_clicks_t {+struct counter_event_t {std::string counter_field;std::map<std::string, uint64_t> docid_counts;+ std::map<std::string, uint16_t> event_weight_map;};struct query_hits_count_t {@@ -153,7 +154,7 @@ private:std::unordered_map<std::string, QueryAnalytics*> nohits_queries;// collection => popular clicks- std::unordered_map<std::string, popular_clicks_t> popular_clicks;+ std::unordered_map<std::string, counter_event_t> counter_events;//query collection => eventsstd::unordered_map<std::string, std::vector<event_t>> query_collection_events;@@ -170,9 +171,9 @@ private:Option<bool> remove_queries_index(const std::string& name);- Option<bool> create_queries_index(nlohmann::json &payload,- bool upsert,- bool write_to_disk);+ Option<bool> create_index(nlohmann::json &payload,+ bool upsert,+ bool write_to_disk);public:@@ -182,7 +183,7 @@ public:static constexpr const char* PURCHASE_EVENT = "$PE";static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";static constexpr const char* NOHITS_QUERIES_TYPE = "nohits_queries";- static constexpr const char* POPULAR_CLICKS_TYPE = "popular_clicks";+ static constexpr const char* COUNTER_TYPE = "counter";static AnalyticsManager& get_instance() {static AnalyticsManager instance;@@ -227,7 +228,7 @@ public:nlohmann::json get_events(const std::string& event_type);- std::unordered_map<std::string, popular_clicks_t> get_popular_clicks();+ std::unordered_map<std::string, counter_event_t> get_popular_clicks();Option<bool> write_events_to_store(nlohmann::json& event_jsons);diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex b14cd2c6..6d4d2383 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -44,14 +44,14 @@ Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert,}if(payload["type"] == POPULAR_QUERIES_TYPE || payload["type"] == NOHITS_QUERIES_TYPE- || payload["type"] == POPULAR_CLICKS_TYPE) {- return create_queries_index(payload, upsert, write_to_disk);+ || payload["type"] == COUNTER_TYPE) {+ return create_index(payload, upsert, write_to_disk);}return Option<bool>(400, "Invalid type.");}-Option<bool> AnalyticsManager::create_queries_index(nlohmann::json &payload, bool upsert, bool write_to_disk) {+Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert, bool write_to_disk) {// params and name are validated upstreamconst std::string& suggestion_config_name = payload["name"].get<std::string>();bool already_exists = suggestion_configs.find(suggestion_config_name) != suggestion_configs.end();@@ -71,6 +71,15 @@ Option<bool> AnalyticsManager::create_queries_index(nlohmann::json &payload, booreturn Option<bool>(400, "Bad or missing destination.");}+ if(payload["type"] == COUNTER_TYPE) {+ if (!params["source"].contains("events") || (params["source"].contains("events") &&+ (params["source"]["events"].empty()+ || !params["source"]["events"].is_array()+ || !params["source"]["events"][0].is_object()))) {+ return Option<bool>(400, "Bad or missing events.");+ }+ }+size_t limit = 1000;bool expand_query = false;@@ -115,8 +124,8 @@ Option<bool> AnalyticsManager::create_queries_index(nlohmann::json &payload, booif (!upsert && nohits_queries.count(suggestion_collection) != 0) {return Option<bool>(400, "There's already another configuration for this destination collection.");}- } else if(payload["type"] == POPULAR_CLICKS_TYPE) {- if (!upsert && popular_clicks.count(suggestion_collection) != 0) {+ } else if(payload["type"] == COUNTER_TYPE) {+ if (!upsert && counter_events.count(suggestion_collection) != 0) {return Option<bool>(400, "There's already another configuration for this destination collection.");}@@ -162,8 +171,12 @@ Option<bool> AnalyticsManager::create_queries_index(nlohmann::json &payload, boo} else if(payload["type"] == NOHITS_QUERIES_TYPE) {QueryAnalytics *noresultsQueries = new QueryAnalytics(limit);nohits_queries.emplace(suggestion_collection, noresultsQueries);- } else if(payload["type"] == POPULAR_CLICKS_TYPE) {- popular_clicks.emplace(suggestion_collection, popular_clicks_t{counter_field, {}});+ } else if(payload["type"] == COUNTER_TYPE) {+ std::map<std::string, uint16_t> event_weight_map;+ for(const auto& event : params["source"]["events"]){+ event_weight_map[event["type"]] = event["weight"];+ }+ counter_events.emplace(suggestion_collection, counter_event_t{counter_field, {}, event_weight_map});}if(write_to_disk) {@@ -312,9 +325,15 @@ Option<bool> AnalyticsManager::add_event(const std::string& event_type, const stevent_t event(query, event_type, now_ts_useconds, user_id, doc_id, position);events_vec.emplace_back(event);- auto popular_clicks_it = popular_clicks.find(query_collection);- if(popular_clicks_it != popular_clicks.end()) {- popular_clicks_it->second.docid_counts[doc_id]++;+ auto counter_events_it = counter_events.find(query_collection);+ if(counter_events_it != counter_events.end()) {+ auto event_weight_map_it = counter_events_it->second.event_weight_map.find(event_type);+ if(event_weight_map_it != counter_events_it->second.event_weight_map.end()) {+ auto inc_val = event_weight_map_it->second;+ counter_events_it->second.docid_counts[doc_id]+= inc_val;+ } else {+ LOG(ERROR) << "event_type " << event_type << " not defined in analytic rule for counter events.";+ }} else {LOG(ERROR) << "collection " << query_collection << " not found in analytics rule.";}@@ -570,13 +589,13 @@ void AnalyticsManager::persist_popular_clicks(ReplicationState *raft_server, uin}};- for(const auto& popular_clicks_it : popular_clicks) {- auto coll = popular_clicks_it.first;+ for(const auto& counter_event_it : counter_events) {+ auto coll = counter_event_it.first;nlohmann::json doc;- auto counter_field = popular_clicks_it.second.counter_field;- for(const auto& popular_click : popular_clicks_it.second.docid_counts) {- doc["id"] = popular_click.first;- doc[counter_field] = popular_click.second;+ auto counter_field = counter_event_it.second.counter_field;+ for(const auto& counter_event : counter_event_it.second.docid_counts) {+ doc["id"] = counter_event.first;+ doc[counter_field] = counter_event.second;send_http_response(doc.dump(), coll);}}@@ -622,9 +641,9 @@ std::unordered_map<std::string, QueryAnalytics*> AnalyticsManager::get_nohits_qureturn nohits_queries;}-std::unordered_map<std::string, popular_clicks_t> AnalyticsManager::get_popular_clicks() {+std::unordered_map<std::string, counter_event_t> AnalyticsManager::get_popular_clicks() {std::unique_lock lk(mutex);- return popular_clicks;+ return counter_events;}nlohmann::json AnalyticsManager::get_events(const std::string& event_type) {diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex 6ffe8d22..e2c656d8 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -933,10 +933,11 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {nlohmann::json analytics_rule = R"({"name": "product_popularity",- "type": "popular_clicks",+ "type": "counter","params": {"source": {- "collections": ["products"]+ "collections": ["products"],+ "events": [{"type": "query_click", "weight": 1}, {"type": "query_purchase", "weight": 5} ]},"destination": {"collection": "products",@@ -952,7 +953,7 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);nlohmann::json event1 = R"({- "type": "query_click",+ "type": "query_purchase","data": {"q": "trousers","collection": "products",@@ -985,7 +986,7 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {ASSERT_EQ(1, popular_clicks.size());ASSERT_EQ("popularity", popular_clicks["products"].counter_field);ASSERT_EQ(2, popular_clicks["products"].docid_counts.size());- ASSERT_EQ(1, popular_clicks["products"].docid_counts["1"]);+ ASSERT_EQ(5, popular_clicks["products"].docid_counts["1"]);ASSERT_EQ(2, popular_clicks["products"].docid_counts["3"]);//trigger persistance event@@ -1011,13 +1012,13 @@ TEST_F(AnalyticsManagerTest, PopularityScore) {ASSERT_EQ(5, results["hits"].size());- ASSERT_EQ("3", results["hits"][0]["document"]["id"]);- ASSERT_EQ(2, results["hits"][0]["document"]["popularity"]);- ASSERT_EQ("Trendy shorts", results["hits"][0]["document"]["title"]);+ ASSERT_EQ("1", results["hits"][0]["document"]["id"]);+ ASSERT_EQ(5, results["hits"][0]["document"]["popularity"]);+ ASSERT_EQ("Funky trousers", results["hits"][0]["document"]["title"]);- ASSERT_EQ("1", results["hits"][1]["document"]["id"]);- ASSERT_EQ(1, results["hits"][1]["document"]["popularity"]);- ASSERT_EQ("Funky trousers", results["hits"][1]["document"]["title"]);+ ASSERT_EQ("3", results["hits"][1]["document"]["id"]);+ ASSERT_EQ(2, results["hits"][1]["document"]["popularity"]);+ ASSERT_EQ("Trendy shorts", results["hits"][1]["document"]["title"]);}TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {@@ -1033,10 +1034,11 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {nlohmann::json analytics_rule = R"({"name": "books_popularity",- "type": "popular_clicks",+ "type": "counter","params": {"source": {- "collections": ["books"]+ "collections": ["books"],+ "events": [{"type": "query_click", "weight": 1}, {"type": "query_purchase", "weight": 5} ]},"destination": {"collection": "popular_books",@@ -1051,10 +1053,11 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {analytics_rule = R"({"name": "books_popularity",- "type": "popular_clicks",+ "type": "counter","params": {"source": {- "collections": ["books"]+ "collections": ["books"],+ "events": [{"type": "query_click", "weight": 1}, {"type": "query_purchase", "weight": 5} ]},"destination": {"collection": "books",@@ -1066,6 +1069,81 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {create_op = analyticsManager.create_rule(analytics_rule, false, true);ASSERT_FALSE(create_op.ok());ASSERT_EQ("counter_field `popularity_score` not found in destination collection.", create_op.error());++ analytics_rule = R"({+ "name": "books_popularity",+ "type": "popular_click",+ "params": {+ "source": {+ "collections": ["books"],+ "events": [{"type": "query_click", "weight": 1}, {"type": "query_purchase", "weight": 5} ]+ },+ "destination": {+ "collection": "books",+ "counter_field": "popularity_score"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_FALSE(create_op.ok());+ ASSERT_EQ("Invalid type.", create_op.error());++ analytics_rule = R"({+ "name": "books_popularity",+ "type": "counter",+ "params": {+ "source": {+ "collections": ["books"]+ },+ "destination": {+ "collection": "books",+ "counter_field": "popularity_score"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_FALSE(create_op.ok());+ ASSERT_EQ("Bad or missing events.", create_op.error());++ analytics_rule = R"({+ "name": "books_popularity",+ "type": "counter",+ "params": {+ "source": {+ "collections": ["books"],+ "events": []+ },+ "destination": {+ "collection": "books",+ "counter_field": "popularity_score"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_FALSE(create_op.ok());+ ASSERT_EQ("Bad or missing events.", create_op.error());++ analytics_rule = R"({+ "name": "books_popularity",+ "type": "counter",+ "params": {+ "source": {+ "collections": ["books"],+ "events": "query_click"+ },+ "destination": {+ "collection": "books",+ "counter_field": "popularity_score"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_FALSE(create_op.ok());+ ASSERT_EQ("Bad or missing events.", create_op.error());}TEST_F(AnalyticsManagerTest, PurchaseEventsStoreRetrieval) {