Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
initial special events commit
krunal4b6990c7c38d
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex e3c21dbe..28a45293 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -154,6 +154,9 @@ private://query collection => click eventsstd::unordered_map<std::string, std::vector<click_event_t>> query_collection_click_events;+ //query collection=>purchase events+ std::unordered_map<std::string, std::vector<click_event_t>> query_collection_special_events;+//query collection => query hits countstd::unordered_map<std::string, std::set<query_hits_count_t, query_hits_count_comp>> query_collection_hits_count;@@ -175,6 +178,7 @@ public:static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";static constexpr const char* CLICK_EVENT = "$CE";static constexpr const char* QUERY_HITS_COUNT = "$QH";+ static constexpr const char* SPECIAL_EVENT = "$SE";static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";static constexpr const char* NOHITS_QUERIES_TYPE = "nohits_queries";static constexpr const char* POPULAR_CLICKS_TYPE = "popular_clicks";@@ -213,14 +217,14 @@ public:std::unordered_map<std::string, QueryAnalytics*> get_popular_queries();- Option<bool> add_click_event(const std::string& query_collection, const std::string& query, const std::string& user_id,+ Option<bool> add_click_special_event(const std::string& event_type, const std::string& query_collection, const std::string& query, const std::string& user_id,std::string doc_id, uint64_t position, const std::string& client_ip);- void persist_query_hits_click_events(ReplicationState *raft_server, uint64_t prev_persistence_s);+ void persist_other_events(ReplicationState *raft_server, uint64_t prev_persistence_s);void persist_popular_clicks(ReplicationState *raft_server, uint64_t prev_persistence_s);- nlohmann::json get_click_events();+ nlohmann::json get_other_events(const std::string& event_type);std::unordered_map<std::string, popular_clicks_t> get_popular_clicks();diff --git a/include/core_api.h b/include/core_api.hindex 158c975b..39508922 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -169,7 +169,7 @@ bool put_upsert_analytics_rules(const std::shared_ptr<http_req>& req, const std:bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);-bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool get_analytics_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);bool post_replicate_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex f76dbd60..a705cd66 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -279,11 +279,12 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection,}}-Option<bool> AnalyticsManager::add_click_event(const std::string &query_collection, const std::string &query, const std::string &user_id,+Option<bool> AnalyticsManager::add_click_special_event(const std::string& event_type, const std::string &query_collection, const std::string &query, const std::string &user_id,std::string doc_id, uint64_t position, const std::string& client_ip) {std::unique_lock lock(mutex);if(analytics_store) {- auto &click_events_vec = query_collection_click_events[query_collection];+ auto &events_vec = event_type == "query_click" ?+ query_collection_click_events[query_collection] : query_collection_special_events[query_collection];auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();@@ -310,7 +311,7 @@ Option<bool> AnalyticsManager::add_click_event(const std::string &query_collectistd::chrono::system_clock::now().time_since_epoch()).count();click_event_t click_event(query, now_ts_useconds, user_id, doc_id, position);- click_events_vec.emplace_back(click_event);+ events_vec.emplace_back(click_event);auto popular_clicks_it = popular_clicks.find(query_collection);if(popular_clicks_it != popular_clicks.end()) {@@ -386,7 +387,7 @@ void AnalyticsManager::run(ReplicationState* raft_server) {checkEventsExpiry();persist_query_events(raft_server, prev_persistence_s);- persist_query_hits_click_events(raft_server, prev_persistence_s);+ persist_other_events(raft_server, prev_persistence_s);persist_popular_clicks(raft_server, prev_persistence_s);prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(@@ -482,7 +483,7 @@ void AnalyticsManager::persist_query_events(ReplicationState *raft_server, uint6}}-void AnalyticsManager::persist_query_hits_click_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {+void AnalyticsManager::persist_other_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {// lock is held by callernlohmann::json payload_json = nlohmann::json::array();@@ -498,7 +499,7 @@ void AnalyticsManager::persist_query_hits_click_events(ReplicationState *raft_seconst std::string &base_url = leader_url + "analytics";std::string res;- const std::string &update_url = base_url + "/" + event_type +"/replicate";+ const std::string &update_url = base_url + "/events/replicate";std::map<std::string, std::string> res_headers;long status_code = HttpClient::post_response(update_url, import_payload,res, res_headers, {}, 10 * 1000, true);@@ -548,6 +549,26 @@ void AnalyticsManager::persist_query_hits_click_events(ReplicationState *raft_seif(send_http_response("query_hits_counts")) {query_collection_hits_count.clear();}++ payload_json.clear();+++ for (const auto &special_events_collection_it: query_collection_special_events) {+ auto collection_id = CollectionManager::get_instance().get_collection(+ special_events_collection_it.first)->get_collection_id();+ for (const auto &special_event: special_events_collection_it.second) {+ // send http request+ nlohmann::json special_event_json;+ special_event.to_json(special_event_json);+ special_event_json["collection_id"] = std::to_string(collection_id);+ special_event_json["event_type"] = "special_events";+ payload_json.push_back(special_event_json);+ }+ }++ if(send_http_response("special_events")) {+ query_collection_special_events.clear();+ }}void AnalyticsManager::persist_popular_clicks(ReplicationState *raft_server, uint64_t prev_persistence_s) {@@ -626,18 +647,19 @@ std::unordered_map<std::string, popular_clicks_t> AnalyticsManager::get_popular_return popular_clicks;}-nlohmann::json AnalyticsManager::get_click_events() {+nlohmann::json AnalyticsManager::get_other_events(const std::string& event_type) {std::unique_lock lk(mutex);- std::vector<std::string> click_event_jsons;+ std::vector<std::string> event_jsons;nlohmann::json result_json = nlohmann::json::array();if (analytics_store) {- analytics_store->scan_fill(std::string(CLICK_EVENT) + "_", std::string(CLICK_EVENT) + "`",- click_event_jsons);+ auto event_prefix = event_type.find("click_events") != std::string::npos ? std::string(CLICK_EVENT) : std::string(SPECIAL_EVENT);+ analytics_store->scan_fill(event_prefix + "_", event_prefix + "`",+ event_jsons);- for (const auto &click_event_json: click_event_jsons) {- nlohmann::json click_event = nlohmann::json::parse(click_event_json);- result_json.push_back(click_event);+ for (const auto &event_json: event_jsons) {+ nlohmann::json event = nlohmann::json::parse(event_json);+ result_json.push_back(event);}}@@ -673,6 +695,8 @@ Option<bool> AnalyticsManager::write_events_to_store(nlohmann::json &event_jsonskey = std::string(CLICK_EVENT) + key;} else if(event_json["event_type"] == "query_hits_counts") {key = std::string(QUERY_HITS_COUNT) + key;+ } else if(event_json["event_type"] == "special_events") {+ key = std::string(SPECIAL_EVENT) + key;}if(analytics_store) {diff --git a/src/core_api.cpp b/src/core_api.cppindex ffa7ba6a..cdcfe63f 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -2792,22 +2792,32 @@ bool put_conversation_model(const std::shared_ptr<http_req>& req, const std::shareturn true;}-bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+bool get_analytics_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {auto analytics_store = AnalyticsManager::get_instance().get_analytics_store();if (!analytics_store) {LOG(ERROR) << "Analytics store not initialized.";return true;}+ auto event_type = req->params["name"];+ if((event_type.find("click_events") == std::string::npos)+ && (event_type.find("special_events") == std::string::npos)) {++ LOG(ERROR) << "Unknown event : " << event_type;+ LOG(ERROR) << "get_analytics_events supports only click_events and special_events.";+ return true;+ }+export_state_t *export_state = nullptr;- auto click_event_prefix = std::string(AnalyticsManager::CLICK_EVENT) + "_";+ auto event_prefix = event_type.find("click_events") != std::string::npos ? std::string(AnalyticsManager::CLICK_EVENT)+ : std::string(AnalyticsManager::SPECIAL_EVENT);if (req->data == nullptr) {export_state = new export_state_t();req->data = export_state;- export_state->iter_upper_bound_key = std::string(AnalyticsManager::CLICK_EVENT) + "`";+ export_state->iter_upper_bound_key = event_prefix + "`";export_state->iter_upper_bound = new rocksdb::Slice(export_state->iter_upper_bound_key);- export_state->it = analytics_store->scan(click_event_prefix, export_state->iter_upper_bound);+ export_state->it = analytics_store->scan(event_prefix + "_", export_state->iter_upper_bound);} else {export_state = dynamic_cast<export_state_t *>(req->data);}@@ -2826,13 +2836,13 @@ bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptreturn false;}- while (it->Valid() && it->key().ToString().compare(0, click_event_prefix.size(), click_event_prefix) == 0) {+ while (it->Valid() && it->key().ToString().compare(0, event_prefix.size(), event_prefix) == 0) {res->body += it->value().ToString();it->Next();// append a new line character if there is going to be one more record to sendif (it->Valid() &&- it->key().ToString().compare(0, click_event_prefix.size(), click_event_prefix) == 0) {+ it->key().ToString().compare(0, event_prefix.size(), event_prefix) == 0) {res->body += "\n";req->last_chunk_aggregate = false;res->final = false;diff --git a/src/event_manager.cpp b/src/event_manager.cppindex 03b512e0..9c097d3a 100644--- a/src/event_manager.cpp+++ b/src/event_manager.cpp@@ -47,7 +47,7 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::strstd::string query = event_data_query_it.get<std::string>();AnalyticsManager::get_instance().add_suggestion(coll.get<std::string>(), query, query, false, "");}- } else if(event_type == "query_click") {+ } else if(event_type == "query_click" || event_type == "query_special") {if (!event.contains("data")) {return Option<bool>(404, "key `data` not found.");}@@ -89,7 +89,7 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::struint64_t position = event_data_val["position"].get<uint64_t>();const std::string& collection = event_data_val["collection"].get<std::string>();- auto op = AnalyticsManager::get_instance().add_click_event(collection, query, user_id, doc_id, position, client_ip);+ auto op = AnalyticsManager::get_instance().add_click_special_event(event_type, collection, query, user_id, doc_id, position, client_ip);if(!op.ok()) {return Option<bool>(op.code(), op.error());}diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex a52db200..2c950fd7 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -79,14 +79,12 @@ void master_server_routes() {server->post("/analytics/rules", post_create_analytics_rules);server->put("/analytics/rules/:name", put_upsert_analytics_rules);server->del("/analytics/rules/:name", del_analytics_rules);- server->post("/analytics/events", post_create_event);- //collection based query click events- server->get("/analytics/click_events", get_click_events, false, true);- server->post("/analytics/click_events", post_create_event);- server->post("/analytics/click_events/replicate", post_replicate_events);+ //analytics events+ server->post("/analytics/events", post_create_event);+ server->post("/analytics/events/replicate", post_replicate_events);+ server->get("/analytics/events/:name", get_analytics_events, false, true);server->get("/analytics/query_hits_counts", get_query_hits_counts);- server->post("/analytics/query_hits_counts/replicate", post_replicate_events);// metaserver->get("/metrics.json", get_metrics_json);diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex fcd5ae21..f6832cbf 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -389,7 +389,7 @@ TEST_F(AnalyticsManagerTest, ClickEventsStoreRetrieveal) {req->body = click_events.dump();ASSERT_TRUE(post_replicate_events(req, res));- auto result = analyticsManager.get_click_events();+ auto result = analyticsManager.get_other_events("click_events");ASSERT_EQ("0", result[0]["collection_id"]);ASSERT_EQ("13", result[0]["data"]["user_id"]);@@ -663,7 +663,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());- nlohmann::json resp = analyticsManager.get_click_events();+ nlohmann::json resp = analyticsManager.get_other_events("click_events");ASSERT_EQ(2, resp.size());ASSERT_EQ("technology", resp[0]["q"]);ASSERT_EQ("21", resp[0]["doc_id"]);@@ -677,7 +677,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {analyticsManager.checkEventsExpiry();- resp = analyticsManager.get_click_events();+ resp = analyticsManager.get_other_events("click_events");ASSERT_EQ(0, resp.size());//add query hits events with click events@@ -703,7 +703,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {// assumming ttl is ts + 5 secanalyticsManager.checkEventsExpiry();- resp = analyticsManager.get_click_events();+ resp = analyticsManager.get_other_events("click_events");ASSERT_EQ(0, resp.size());resp = analyticsManager.get_query_hits_counts();@@ -756,7 +756,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryAll) {ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());- nlohmann::json resp = analyticsManager.get_click_events();+ nlohmann::json resp = analyticsManager.get_other_events("click_events");ASSERT_EQ(2, resp.size());ASSERT_EQ("technology", resp[0]["q"]);ASSERT_EQ("21", resp[0]["doc_id"]);@@ -778,7 +778,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryAll) {// assuming ttl is ts + 5 secanalyticsManager.checkEventsExpiry();- resp = analyticsManager.get_click_events();+ resp = analyticsManager.get_other_events("click_events");ASSERT_EQ(0, resp.size());resp = analyticsManager.get_query_hits_counts();@@ -844,7 +844,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryPartial) {ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());- nlohmann::json resp = analyticsManager.get_click_events();+ nlohmann::json resp = analyticsManager.get_other_events("click_events");ASSERT_EQ(3, resp.size());ASSERT_EQ("technology", resp[0]["q"]);ASSERT_EQ("21", resp[0]["doc_id"]);@@ -880,7 +880,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryPartial) {//only events in ttl interval will be removedanalyticsManager.checkEventsExpiry();- resp = analyticsManager.get_click_events();+ resp = analyticsManager.get_other_events("click_events");ASSERT_EQ(1, resp.size());ASSERT_EQ("technology", resp[0]["q"]);ASSERT_EQ("19", resp[0]["doc_id"]);@@ -1066,4 +1066,74 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {create_op = analyticsManager.create_rule(analytics_rule, false, true);ASSERT_FALSE(create_op.ok());ASSERT_EQ("counter_field `popularity_score` not found in destination collection.", create_op.error());+}++TEST_F(AnalyticsManagerTest, SpecialEventsStoreRetrieval) {+ nlohmann::json titles_schema = R"({+ "name": "titles",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection *titles_coll = collectionManager.create_collection(titles_schema).get();++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ nlohmann::json event1 = R"({+ "type": "query_special",+ "data": {+ "q": "technology",+ "collection": "titles",+ "doc_id": "21",+ "position": 2,+ "user_id": "13"+ }+ })"_json;++ req->body = event1.dump();+ ASSERT_TRUE(post_create_event(req, res));++ nlohmann::json event2 = R"({+ "type": "query_special",+ "data": {+ "q": "technology",+ "collection": "titles",+ "doc_id": "21",+ "position": 4,+ "user_id": "11"+ }+ })"_json;++ req->body = event2.dump();+ ASSERT_TRUE(post_create_event(req, res));++ event1["collection_id"] = "0";+ event1["timestamp"] = 1521512521;+ event1["event_type"] = "special_events";+ event2["collection_id"] = "0";+ event2["timestamp"] = 1521514354;+ event2["event_type"] = "special_events";++ nlohmann::json click_events = nlohmann::json::array();+ click_events.push_back(event1);+ click_events.push_back(event2);++ req->body = click_events.dump();+ ASSERT_TRUE(post_replicate_events(req, res));++ auto result = analyticsManager.get_other_events("special_events");++ ASSERT_EQ("0", result[0]["collection_id"]);+ ASSERT_EQ("13", result[0]["data"]["user_id"]);+ ASSERT_EQ("21", result[0]["data"]["doc_id"]);+ ASSERT_EQ(2, result[0]["data"]["position"]);+ ASSERT_EQ("technology", result[0]["data"]["q"]);++ ASSERT_EQ("0", result[1]["collection_id"]);+ ASSERT_EQ("11", result[1]["data"]["user_id"]);+ ASSERT_EQ("21", result[1]["data"]["doc_id"]);+ ASSERT_EQ(4, result[1]["data"]["position"]);+ ASSERT_EQ("technology", result[1]["data"]["q"]);}\ No newline at end of filediff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex 361e0aed..fd8be7b1 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -1520,7 +1520,8 @@ TEST_F(CoreAPIUtilsTest, GetClickEvents) {std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);// no events in db- get_click_events(req, res);+ req->params["name"] = "click_events";+ get_analytics_events(req, res);ASSERT_EQ("{\"message\": \"Not Found\"}", res->body);//add some events@@ -1585,7 +1586,8 @@ TEST_F(CoreAPIUtilsTest, GetClickEvents) {//get click eventsreq->data = nullptr;- get_click_events(req, res);+ req->params["name"] = "click_events";+ get_analytics_events(req, res);std::vector<std::string> res_strs;StringUtils::split(res->body, res_strs, "\n");