Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

initial special events commit

krunal4b6990c7c38d
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index e3c21dbe..28a45293 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -154,6 +154,9 @@ private:
//query collection => click events
std::unordered_map<std::string, std::vector<click_event_t>> query_collection_click_events;
+ //query collection=>purchase events
+ std::unordered_map<std::string, std::vector<click_event_t>> query_collection_special_events;
+
//query collection => query hits count
std::unordered_map<std::string, std::set<query_hits_count_t, query_hits_count_comp>> query_collection_hits_count;
@@ -175,6 +178,7 @@ public:
static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";
static constexpr const char* CLICK_EVENT = "$CE";
static constexpr const char* QUERY_HITS_COUNT = "$QH";
+ static constexpr const char* SPECIAL_EVENT = "$SE";
static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";
static constexpr const char* NOHITS_QUERIES_TYPE = "nohits_queries";
static constexpr const char* POPULAR_CLICKS_TYPE = "popular_clicks";
@@ -213,14 +217,14 @@ public:
std::unordered_map<std::string, QueryAnalytics*> get_popular_queries();
- Option<bool> add_click_event(const std::string& query_collection, const std::string& query, const std::string& user_id,
+ Option<bool> add_click_special_event(const std::string& event_type, const std::string& query_collection, const std::string& query, const std::string& user_id,
std::string doc_id, uint64_t position, const std::string& client_ip);
- void persist_query_hits_click_events(ReplicationState *raft_server, uint64_t prev_persistence_s);
+ void persist_other_events(ReplicationState *raft_server, uint64_t prev_persistence_s);
void persist_popular_clicks(ReplicationState *raft_server, uint64_t prev_persistence_s);
- nlohmann::json get_click_events();
+ nlohmann::json get_other_events(const std::string& event_type);
std::unordered_map<std::string, popular_clicks_t> get_popular_clicks();
diff --git a/include/core_api.h b/include/core_api.h
index 158c975b..39508922 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -169,7 +169,7 @@ bool put_upsert_analytics_rules(const std::shared_ptr<http_req>& req, const std:
bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
-bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+bool get_analytics_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
bool post_replicate_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index f76dbd60..a705cd66 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -279,11 +279,12 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection,
}
}
-Option<bool> AnalyticsManager::add_click_event(const std::string &query_collection, const std::string &query, const std::string &user_id,
+Option<bool> AnalyticsManager::add_click_special_event(const std::string& event_type, const std::string &query_collection, const std::string &query, const std::string &user_id,
std::string doc_id, uint64_t position, const std::string& client_ip) {
std::unique_lock lock(mutex);
if(analytics_store) {
- auto &click_events_vec = query_collection_click_events[query_collection];
+ auto &events_vec = event_type == "query_click" ?
+ query_collection_click_events[query_collection] : query_collection_special_events[query_collection];
auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
@@ -310,7 +311,7 @@ Option<bool> AnalyticsManager::add_click_event(const std::string &query_collecti
std::chrono::system_clock::now().time_since_epoch()).count();
click_event_t click_event(query, now_ts_useconds, user_id, doc_id, position);
- click_events_vec.emplace_back(click_event);
+ events_vec.emplace_back(click_event);
auto popular_clicks_it = popular_clicks.find(query_collection);
if(popular_clicks_it != popular_clicks.end()) {
@@ -386,7 +387,7 @@ void AnalyticsManager::run(ReplicationState* raft_server) {
checkEventsExpiry();
persist_query_events(raft_server, prev_persistence_s);
- persist_query_hits_click_events(raft_server, prev_persistence_s);
+ persist_other_events(raft_server, prev_persistence_s);
persist_popular_clicks(raft_server, prev_persistence_s);
prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(
@@ -482,7 +483,7 @@ void AnalyticsManager::persist_query_events(ReplicationState *raft_server, uint6
}
}
-void AnalyticsManager::persist_query_hits_click_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {
+void AnalyticsManager::persist_other_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {
// lock is held by caller
nlohmann::json payload_json = nlohmann::json::array();
@@ -498,7 +499,7 @@ void AnalyticsManager::persist_query_hits_click_events(ReplicationState *raft_se
const std::string &base_url = leader_url + "analytics";
std::string res;
- const std::string &update_url = base_url + "/" + event_type +"/replicate";
+ const std::string &update_url = base_url + "/events/replicate";
std::map<std::string, std::string> res_headers;
long status_code = HttpClient::post_response(update_url, import_payload,
res, res_headers, {}, 10 * 1000, true);
@@ -548,6 +549,26 @@ void AnalyticsManager::persist_query_hits_click_events(ReplicationState *raft_se
if(send_http_response("query_hits_counts")) {
query_collection_hits_count.clear();
}
+
+ payload_json.clear();
+
+
+ for (const auto &special_events_collection_it: query_collection_special_events) {
+ auto collection_id = CollectionManager::get_instance().get_collection(
+ special_events_collection_it.first)->get_collection_id();
+ for (const auto &special_event: special_events_collection_it.second) {
+ // send http request
+ nlohmann::json special_event_json;
+ special_event.to_json(special_event_json);
+ special_event_json["collection_id"] = std::to_string(collection_id);
+ special_event_json["event_type"] = "special_events";
+ payload_json.push_back(special_event_json);
+ }
+ }
+
+ if(send_http_response("special_events")) {
+ query_collection_special_events.clear();
+ }
}
void AnalyticsManager::persist_popular_clicks(ReplicationState *raft_server, uint64_t prev_persistence_s) {
@@ -626,18 +647,19 @@ std::unordered_map<std::string, popular_clicks_t> AnalyticsManager::get_popular_
return popular_clicks;
}
-nlohmann::json AnalyticsManager::get_click_events() {
+nlohmann::json AnalyticsManager::get_other_events(const std::string& event_type) {
std::unique_lock lk(mutex);
- std::vector<std::string> click_event_jsons;
+ std::vector<std::string> event_jsons;
nlohmann::json result_json = nlohmann::json::array();
if (analytics_store) {
- analytics_store->scan_fill(std::string(CLICK_EVENT) + "_", std::string(CLICK_EVENT) + "`",
- click_event_jsons);
+ auto event_prefix = event_type.find("click_events") != std::string::npos ? std::string(CLICK_EVENT) : std::string(SPECIAL_EVENT);
+ analytics_store->scan_fill(event_prefix + "_", event_prefix + "`",
+ event_jsons);
- for (const auto &click_event_json: click_event_jsons) {
- nlohmann::json click_event = nlohmann::json::parse(click_event_json);
- result_json.push_back(click_event);
+ for (const auto &event_json: event_jsons) {
+ nlohmann::json event = nlohmann::json::parse(event_json);
+ result_json.push_back(event);
}
}
@@ -673,6 +695,8 @@ Option<bool> AnalyticsManager::write_events_to_store(nlohmann::json &event_jsons
key = std::string(CLICK_EVENT) + key;
} else if(event_json["event_type"] == "query_hits_counts") {
key = std::string(QUERY_HITS_COUNT) + key;
+ } else if(event_json["event_type"] == "special_events") {
+ key = std::string(SPECIAL_EVENT) + key;
}
if(analytics_store) {
diff --git a/src/core_api.cpp b/src/core_api.cpp
index ffa7ba6a..cdcfe63f 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -2792,22 +2792,32 @@ bool put_conversation_model(const std::shared_ptr<http_req>& req, const std::sha
return true;
}
-bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+bool get_analytics_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
auto analytics_store = AnalyticsManager::get_instance().get_analytics_store();
if (!analytics_store) {
LOG(ERROR) << "Analytics store not initialized.";
return true;
}
+ auto event_type = req->params["name"];
+ if((event_type.find("click_events") == std::string::npos)
+ && (event_type.find("special_events") == std::string::npos)) {
+
+ LOG(ERROR) << "Unknown event : " << event_type;
+ LOG(ERROR) << "get_analytics_events supports only click_events and special_events.";
+ return true;
+ }
+
export_state_t *export_state = nullptr;
- auto click_event_prefix = std::string(AnalyticsManager::CLICK_EVENT) + "_";
+ auto event_prefix = event_type.find("click_events") != std::string::npos ? std::string(AnalyticsManager::CLICK_EVENT)
+ : std::string(AnalyticsManager::SPECIAL_EVENT);
if (req->data == nullptr) {
export_state = new export_state_t();
req->data = export_state;
- export_state->iter_upper_bound_key = std::string(AnalyticsManager::CLICK_EVENT) + "`";
+ export_state->iter_upper_bound_key = event_prefix + "`";
export_state->iter_upper_bound = new rocksdb::Slice(export_state->iter_upper_bound_key);
- export_state->it = analytics_store->scan(click_event_prefix, export_state->iter_upper_bound);
+ export_state->it = analytics_store->scan(event_prefix + "_", export_state->iter_upper_bound);
} else {
export_state = dynamic_cast<export_state_t *>(req->data);
}
@@ -2826,13 +2836,13 @@ bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_pt
return false;
}
- while (it->Valid() && it->key().ToString().compare(0, click_event_prefix.size(), click_event_prefix) == 0) {
+ while (it->Valid() && it->key().ToString().compare(0, event_prefix.size(), event_prefix) == 0) {
res->body += it->value().ToString();
it->Next();
// append a new line character if there is going to be one more record to send
if (it->Valid() &&
- it->key().ToString().compare(0, click_event_prefix.size(), click_event_prefix) == 0) {
+ it->key().ToString().compare(0, event_prefix.size(), event_prefix) == 0) {
res->body += "\n";
req->last_chunk_aggregate = false;
res->final = false;
diff --git a/src/event_manager.cpp b/src/event_manager.cpp
index 03b512e0..9c097d3a 100644
--- a/src/event_manager.cpp
+++ b/src/event_manager.cpp
@@ -47,7 +47,7 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::str
std::string query = event_data_query_it.get<std::string>();
AnalyticsManager::get_instance().add_suggestion(coll.get<std::string>(), query, query, false, "");
}
- } else if(event_type == "query_click") {
+ } else if(event_type == "query_click" || event_type == "query_special") {
if (!event.contains("data")) {
return Option<bool>(404, "key `data` not found.");
}
@@ -89,7 +89,7 @@ Option<bool> EventManager::add_event(const nlohmann::json& event, const std::str
uint64_t position = event_data_val["position"].get<uint64_t>();
const std::string& collection = event_data_val["collection"].get<std::string>();
- auto op = AnalyticsManager::get_instance().add_click_event(collection, query, user_id, doc_id, position, client_ip);
+ auto op = AnalyticsManager::get_instance().add_click_special_event(event_type, collection, query, user_id, doc_id, position, client_ip);
if(!op.ok()) {
return Option<bool>(op.code(), op.error());
}
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index a52db200..2c950fd7 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -79,14 +79,12 @@ void master_server_routes() {
server->post("/analytics/rules", post_create_analytics_rules);
server->put("/analytics/rules/:name", put_upsert_analytics_rules);
server->del("/analytics/rules/:name", del_analytics_rules);
- server->post("/analytics/events", post_create_event);
- //collection based query click events
- server->get("/analytics/click_events", get_click_events, false, true);
- server->post("/analytics/click_events", post_create_event);
- server->post("/analytics/click_events/replicate", post_replicate_events);
+ //analytics events
+ server->post("/analytics/events", post_create_event);
+ server->post("/analytics/events/replicate", post_replicate_events);
+ server->get("/analytics/events/:name", get_analytics_events, false, true);
server->get("/analytics/query_hits_counts", get_query_hits_counts);
- server->post("/analytics/query_hits_counts/replicate", post_replicate_events);
// meta
server->get("/metrics.json", get_metrics_json);
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index fcd5ae21..f6832cbf 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -389,7 +389,7 @@ TEST_F(AnalyticsManagerTest, ClickEventsStoreRetrieveal) {
req->body = click_events.dump();
ASSERT_TRUE(post_replicate_events(req, res));
- auto result = analyticsManager.get_click_events();
+ auto result = analyticsManager.get_other_events("click_events");
ASSERT_EQ("0", result[0]["collection_id"]);
ASSERT_EQ("13", result[0]["data"]["user_id"]);
@@ -663,7 +663,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {
ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());
- nlohmann::json resp = analyticsManager.get_click_events();
+ nlohmann::json resp = analyticsManager.get_other_events("click_events");
ASSERT_EQ(2, resp.size());
ASSERT_EQ("technology", resp[0]["q"]);
ASSERT_EQ("21", resp[0]["doc_id"]);
@@ -677,7 +677,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {
analyticsManager.checkEventsExpiry();
- resp = analyticsManager.get_click_events();
+ resp = analyticsManager.get_other_events("click_events");
ASSERT_EQ(0, resp.size());
//add query hits events with click events
@@ -703,7 +703,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {
// assumming ttl is ts + 5 sec
analyticsManager.checkEventsExpiry();
- resp = analyticsManager.get_click_events();
+ resp = analyticsManager.get_other_events("click_events");
ASSERT_EQ(0, resp.size());
resp = analyticsManager.get_query_hits_counts();
@@ -756,7 +756,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryAll) {
ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());
- nlohmann::json resp = analyticsManager.get_click_events();
+ nlohmann::json resp = analyticsManager.get_other_events("click_events");
ASSERT_EQ(2, resp.size());
ASSERT_EQ("technology", resp[0]["q"]);
ASSERT_EQ("21", resp[0]["doc_id"]);
@@ -778,7 +778,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryAll) {
// assuming ttl is ts + 5 sec
analyticsManager.checkEventsExpiry();
- resp = analyticsManager.get_click_events();
+ resp = analyticsManager.get_other_events("click_events");
ASSERT_EQ(0, resp.size());
resp = analyticsManager.get_query_hits_counts();
@@ -844,7 +844,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryPartial) {
ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());
- nlohmann::json resp = analyticsManager.get_click_events();
+ nlohmann::json resp = analyticsManager.get_other_events("click_events");
ASSERT_EQ(3, resp.size());
ASSERT_EQ("technology", resp[0]["q"]);
ASSERT_EQ("21", resp[0]["doc_id"]);
@@ -880,7 +880,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiryPartial) {
//only events in ttl interval will be removed
analyticsManager.checkEventsExpiry();
- resp = analyticsManager.get_click_events();
+ resp = analyticsManager.get_other_events("click_events");
ASSERT_EQ(1, resp.size());
ASSERT_EQ("technology", resp[0]["q"]);
ASSERT_EQ("19", resp[0]["doc_id"]);
@@ -1066,4 +1066,74 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
create_op = analyticsManager.create_rule(analytics_rule, false, true);
ASSERT_FALSE(create_op.ok());
ASSERT_EQ("counter_field `popularity_score` not found in destination collection.", create_op.error());
+}
+
+TEST_F(AnalyticsManagerTest, SpecialEventsStoreRetrieval) {
+ nlohmann::json titles_schema = R"({
+ "name": "titles",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection *titles_coll = collectionManager.create_collection(titles_schema).get();
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ nlohmann::json event1 = R"({
+ "type": "query_special",
+ "data": {
+ "q": "technology",
+ "collection": "titles",
+ "doc_id": "21",
+ "position": 2,
+ "user_id": "13"
+ }
+ })"_json;
+
+ req->body = event1.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+
+ nlohmann::json event2 = R"({
+ "type": "query_special",
+ "data": {
+ "q": "technology",
+ "collection": "titles",
+ "doc_id": "21",
+ "position": 4,
+ "user_id": "11"
+ }
+ })"_json;
+
+ req->body = event2.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+
+ event1["collection_id"] = "0";
+ event1["timestamp"] = 1521512521;
+ event1["event_type"] = "special_events";
+ event2["collection_id"] = "0";
+ event2["timestamp"] = 1521514354;
+ event2["event_type"] = "special_events";
+
+ nlohmann::json click_events = nlohmann::json::array();
+ click_events.push_back(event1);
+ click_events.push_back(event2);
+
+ req->body = click_events.dump();
+ ASSERT_TRUE(post_replicate_events(req, res));
+
+ auto result = analyticsManager.get_other_events("special_events");
+
+ ASSERT_EQ("0", result[0]["collection_id"]);
+ ASSERT_EQ("13", result[0]["data"]["user_id"]);
+ ASSERT_EQ("21", result[0]["data"]["doc_id"]);
+ ASSERT_EQ(2, result[0]["data"]["position"]);
+ ASSERT_EQ("technology", result[0]["data"]["q"]);
+
+ ASSERT_EQ("0", result[1]["collection_id"]);
+ ASSERT_EQ("11", result[1]["data"]["user_id"]);
+ ASSERT_EQ("21", result[1]["data"]["doc_id"]);
+ ASSERT_EQ(4, result[1]["data"]["position"]);
+ ASSERT_EQ("technology", result[1]["data"]["q"]);
}
\ No newline at end of file
diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cpp
index 361e0aed..fd8be7b1 100644
--- a/test/core_api_utils_test.cpp
+++ b/test/core_api_utils_test.cpp
@@ -1520,7 +1520,8 @@ TEST_F(CoreAPIUtilsTest, GetClickEvents) {
std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
// no events in db
- get_click_events(req, res);
+ req->params["name"] = "click_events";
+ get_analytics_events(req, res);
ASSERT_EQ("{\"message\": \"Not Found\"}", res->body);
//add some events
@@ -1585,7 +1586,8 @@ TEST_F(CoreAPIUtilsTest, GetClickEvents) {
//get click events
req->data = nullptr;
- get_click_events(req, res);
+ req->params["name"] = "click_events";
+ get_analytics_events(req, res);
std::vector<std::string> res_strs;
StringUtils::split(res->body, res_strs, "\n");