Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

remove analytics logging to tsv file (#1843)

Krunal Gandhi99a774d35999
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index 45c02832..84f7c751 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -142,7 +142,6 @@ private:
Store* store = nullptr;
Store* analytics_store = nullptr;
- std::ofstream analytics_logs;
bool isRateLimitEnabled = true;
@@ -176,7 +175,7 @@ public:
AnalyticsManager(AnalyticsManager const&) = delete;
void operator=(AnalyticsManager const&) = delete;
- void init(Store* store, Store* analytics_store, const std::string& analytics_dir="");
+ void init(Store* store, Store* analytics_store);
void run(ReplicationState* raft_server);
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index 738b9b7b..5e5d9395 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -223,9 +223,9 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsert
if(event.contains("log_to_file")) {
log_to_file = event["log_to_file"].get<bool>();
- if(log_to_file && !analytics_logs.is_open()) {
+ if(log_to_file && !analytics_store) {
remove_index(suggestion_config_name);
- return Option<bool>(400, "Event can't be logged when analytics-dir is not defined.");
+ return Option<bool>(400, "Event can't be logged when analytics-db is not defined.");
}
}
event_weight_map[event["name"]] = event["weight"];
@@ -658,21 +658,7 @@ void AnalyticsManager::persist_events(ReplicationState *raft_server, uint64_t pr
for (auto &events_collection_it: query_collection_events) {
const auto& collection = events_collection_it.first;
for (const auto &event: events_collection_it.second) {
- if (analytics_logs.is_open() && event.log_to_file) {
- //store events to log file
- std::stringstream ssbuf;
- ssbuf << event.timestamp << "\t" << event.name << "\t"
- << collection << "\t" << event.user_id << "\t" << event.doc_id << "\t"
- << event.query << "\t";
-
- for(const auto& kv : event.data) {
- ssbuf << kv.second << "\t";
- }
-
- analytics_logs << ssbuf.str();
- analytics_logs << "\n";
- analytics_logs << std::flush;
-
+ if (event.log_to_file) {
nlohmann::json event_data;
event.to_json(event_data, collection);
payload.push_back(event_data);
@@ -720,7 +706,6 @@ void AnalyticsManager::persist_popular_events(ReplicationState *raft_server, uin
void AnalyticsManager::stop() {
quit = true;
cv.notify_all();
- analytics_logs.close();
}
void AnalyticsManager::dispose() {
@@ -751,13 +736,11 @@ void AnalyticsManager::dispose() {
events_cache.clear();
}
-void AnalyticsManager::init(Store* store, Store* analytics_store, const std::string& analytics_dir) {
+void AnalyticsManager::init(Store* store, Store* analytics_store) {
this->store = store;
this->analytics_store = analytics_store;
- if(!analytics_dir.empty()) {
- const auto analytics_log_path = analytics_dir + "/analytics_events.tsv";
- analytics_logs.open(analytics_log_path, std::ofstream::out | std::ofstream::app);
+ if(analytics_store) {
events_cache.capacity(1024);
}
}
@@ -799,7 +782,9 @@ void counter_event_t::serialize_as_docs(std::string &docs) {
bool AnalyticsManager::write_to_db(const nlohmann::json& payload) {
if(analytics_store) {
for(const auto& event: payload) {
- std::string key = event["user_id"].get<std::string>() + "_" + StringUtils::serialize_uint64_t(event["timestamp"].get<uint64_t>());
+ std::string key = event["user_id"].get<std::string>() + "_" + event["type"].get<std::string>()
+ + "_" + StringUtils::serialize_uint64_t(event["timestamp"].get<uint64_t>());
+
bool inserted = analytics_store->insert(key, event.dump());
if(!inserted) {
LOG(ERROR) << "Error while dumping events to analytics db.";
@@ -814,8 +799,8 @@ bool AnalyticsManager::write_to_db(const nlohmann::json& payload) {
return true;
}
-void AnalyticsManager::get_last_N_events(const std::string& userid, uint32_t N, std::vector<std::string>& values) {
- const std::string userid_prefix = userid + "_";
+void AnalyticsManager::get_last_N_events(const std::string& prefix, uint32_t N, std::vector<std::string>& values) {
+ const std::string userid_prefix = prefix + "_";
analytics_store->get_last_N_values(userid_prefix, N, values);
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 6bd1970a..eaea8d58 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -424,7 +424,7 @@ int run_server(const Config & config, const std::string & version, void (*master
HttpClient & httpClient = HttpClient::get_instance();
httpClient.init(config.get_api_key());
- AnalyticsManager::get_instance().init(&store, &analytics_store, analytics_dir);
+ AnalyticsManager::get_instance().init(&store, &analytics_store);
server = new HttpServer(
version,
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index 4e9ba593..d970cff5 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -36,7 +36,7 @@ protected:
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
- analyticsManager.init(store, analytic_store, state_dir_path);
+ analyticsManager.init(store, analytic_store);
analyticsManager.resetToggleRateLimit(false);
}
@@ -524,18 +524,32 @@ TEST_F(AnalyticsManagerTest, EventsPersist) {
req->body = event.dump();
ASSERT_TRUE(post_create_event(req, res));
- analyticsManager.persist_events(nullptr, 0);
+ //get events
+ nlohmann::json payload = nlohmann::json::array();
+ nlohmann::json event_data;
+ auto collection_events_map = analyticsManager.get_log_events();
+ for (auto &events_collection_it: collection_events_map) {
+ const auto& collection = events_collection_it.first;
+ for(const auto& event: events_collection_it.second) {
+ event.to_json(event_data, collection);
+ payload.push_back(event_data);
+ }
+ }
+
+ //manually trigger write to db
+ ASSERT_TRUE(analyticsManager.write_to_db(payload));
+
+ std::vector<std::string> values;
+ analyticsManager.get_last_N_events("13", 5, values);
+ ASSERT_EQ(1, values.size());
- auto fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
+ auto parsed_json = nlohmann::json::parse(values[0]);
- std::stringstream strbuff(fileOutput.get());
- std::string docid, userid, q, collection, name, timestamp;
- strbuff >> timestamp >> name >> collection >> userid >> docid >> q;
- ASSERT_EQ("APC", name);
- ASSERT_EQ("titles", collection);
- ASSERT_EQ("13", userid);
- ASSERT_EQ("21", docid);
- ASSERT_EQ("technology", q);
+ ASSERT_EQ("APC", parsed_json["name"]);
+ ASSERT_EQ("titles", parsed_json["collection"]);
+ ASSERT_EQ("13", parsed_json["user_id"]);
+ ASSERT_EQ("21", parsed_json["doc_id"]);
+ ASSERT_EQ("technology", parsed_json["query"]);
event = R"({
"type": "click",
@@ -550,26 +564,40 @@ TEST_F(AnalyticsManagerTest, EventsPersist) {
req->body = event.dump();
ASSERT_TRUE(post_create_event(req, res));
- analyticsManager.persist_events(nullptr, 0);
-
- fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
-
- std::stringstream strbuff2(fileOutput.get());
- timestamp.clear();name.clear();collection.clear();userid.clear();q.clear();
- strbuff2 >> timestamp >> name >> collection >> userid >> docid >> q;
- ASSERT_EQ("APC", name);
- ASSERT_EQ("titles", collection);
- ASSERT_EQ("13", userid);
- ASSERT_EQ("21", docid);
- ASSERT_EQ("technology", q);
-
- timestamp.clear();name.clear();collection.clear();userid.clear();q.clear();
- strbuff2 >> timestamp >> name >> collection >> userid >> docid >> q;
- ASSERT_EQ("APC", name);
- ASSERT_EQ("titles", collection);
- ASSERT_EQ("13", userid);
- ASSERT_EQ("12", docid);
- ASSERT_EQ("technology", q);
+ //get events
+ payload.clear();
+ collection_events_map = analyticsManager.get_log_events();
+ for (auto &events_collection_it: collection_events_map) {
+ const auto& collection = events_collection_it.first;
+ for(const auto& event: events_collection_it.second) {
+ event.to_json(event_data, collection);
+ payload.push_back(event_data);
+ }
+ }
+
+ //manually trigger write to db
+ ASSERT_TRUE(analyticsManager.write_to_db(payload));
+
+ values.clear();
+ analyticsManager.get_last_N_events("13", 5, values);
+ ASSERT_EQ(2, values.size());
+
+ parsed_json = nlohmann::json::parse(values[0]);
+
+ //events will be fetched in LIFO order
+ ASSERT_EQ("APC", parsed_json["name"]);
+ ASSERT_EQ("titles", parsed_json["collection"]);
+ ASSERT_EQ("13", parsed_json["user_id"]);
+ ASSERT_EQ("12", parsed_json["doc_id"]);
+ ASSERT_EQ("technology", parsed_json["query"]);
+
+ parsed_json = nlohmann::json::parse(values[1]);
+
+ ASSERT_EQ("APC", parsed_json["name"]);
+ ASSERT_EQ("titles", parsed_json["collection"]);
+ ASSERT_EQ("13", parsed_json["user_id"]);
+ ASSERT_EQ("21", parsed_json["doc_id"]);
+ ASSERT_EQ("technology", parsed_json["query"]);
}
TEST_F(AnalyticsManagerTest, EventsRateLimitTest) {
@@ -1016,8 +1044,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
//restart analytics manager as fresh
analyticsManager.dispose();
analyticsManager.stop();
- system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
- analyticsManager.init(store, analytic_store, "/tmp/typesense_test/analytics_manager_test");
+ analyticsManager.init(store, analytic_store);
nlohmann::json products_schema = R"({
"name": "books",
@@ -1262,19 +1289,30 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
ASSERT_EQ(0, results["hits"][1]["document"]["popularity"]);
ASSERT_EQ("Cool trousers", results["hits"][1]["document"]["title"]);
- //verify log file
- analyticsManager.persist_events(nullptr, 0);
+ nlohmann::json payload = nlohmann::json::array();
+ nlohmann::json event_data;
+ auto collection_events_map = analyticsManager.get_log_events();
+ for (auto &events_collection_it: collection_events_map) {
+ const auto& collection = events_collection_it.first;
+ for(const auto& event: events_collection_it.second) {
+ event.to_json(event_data, collection);
+ payload.push_back(event_data);
+ }
+ }
+
+ //manually trigger write to db
+ ASSERT_TRUE(analyticsManager.write_to_db(payload));
- auto fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
+ std::vector<std::string> values;
+ analyticsManager.get_last_N_events("11", 5, values);
+ ASSERT_EQ(1, values.size());
- std::stringstream strbuff(fileOutput.get());
- std::string timestamp, collection, userid, name, q, docid;
- strbuff >> timestamp >> name >> collection >> userid >> docid >> q;
- ASSERT_EQ("CNV4", name);
- ASSERT_EQ("books", collection);
- ASSERT_EQ("11", userid);
- ASSERT_EQ("1", docid);
- ASSERT_EQ("shorts", q);
+ auto parsed_json = nlohmann::json::parse(values[0]);
+ ASSERT_EQ("CNV4", parsed_json["name"]);
+ ASSERT_EQ("books", parsed_json["collection"]);
+ ASSERT_EQ("11", parsed_json["user_id"]);
+ ASSERT_EQ("1", parsed_json["doc_id"]);
+ ASSERT_EQ("shorts", parsed_json["query"]);
//now add click event rule
analytics_rule = R"({
@@ -1330,59 +1368,54 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
ASSERT_EQ(1, popular_clicks["books"].docid_counts.size());
ASSERT_EQ(10, popular_clicks["books"].docid_counts["1"]);
- //check log file
- analyticsManager.persist_events(nullptr, 0);
-
- fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
-
- std::stringstream strbuff2(fileOutput.get());
- timestamp.clear(), collection.clear(), userid.clear(), name.clear(), q.clear(), docid.clear();
- strbuff2 >> timestamp >> name >> collection >> userid >> docid >> q;
- ASSERT_EQ("CNV4", name);
- ASSERT_EQ("books", collection);
- ASSERT_EQ("11", userid);
- ASSERT_EQ("1", docid);
- ASSERT_EQ("shorts", q);
-
- timestamp.clear(), collection.clear(), userid.clear(), name.clear(), q.clear(), docid.clear();
- strbuff2 >> timestamp >> name >> collection >> userid >> docid >> q;
- ASSERT_EQ("APC2", name);
- ASSERT_EQ("books", collection);
- ASSERT_EQ("13", userid);
- ASSERT_EQ("21", docid);
- ASSERT_EQ("technology", q);
-
- //counter event rule should not be allowed with logging if analytics-dir is not specified
- analyticsManager.dispose();
- analyticsManager.stop();
- system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
- analyticsManager.init(store, analytic_store, "");
-
- analytics_rule = R"({
- "name": "books_popularity3",
- "type": "counter",
- "params": {
- "source": {
- "collections": ["books"],
- "events": [{"type": "conversion", "weight": 5, "name": "CNV4", "log_to_file" : true} ]
- },
- "destination": {
- "collection": "books",
- "counter_field": "popularity"
- }
+ payload.clear();
+ event_data.clear();
+ collection_events_map = analyticsManager.get_log_events();
+ for (auto &events_collection_it: collection_events_map) {
+ const auto& collection = events_collection_it.first;
+ for(const auto& event: events_collection_it.second) {
+ event.to_json(event_data, collection);
+ payload.push_back(event_data);
}
- })"_json;
+ }
+
+ //manually trigger write to db
+ ASSERT_TRUE(analyticsManager.write_to_db(payload));
+
+ values.clear();
+ analyticsManager.get_last_N_events("11", 5, values);
+ ASSERT_EQ(2, values.size());
+
+ parsed_json = nlohmann::json::parse(values[0]);
+ ASSERT_EQ("CNV4", parsed_json["name"]);
+ ASSERT_EQ("books", parsed_json["collection"]);
+ ASSERT_EQ("11", parsed_json["user_id"]);
+ ASSERT_EQ("1", parsed_json["doc_id"]);
+ ASSERT_EQ("shorts", parsed_json["query"]);
+
+ parsed_json = nlohmann::json::parse(values[1]);
+ ASSERT_EQ("CNV4", parsed_json["name"]);
+ ASSERT_EQ("books", parsed_json["collection"]);
+ ASSERT_EQ("11", parsed_json["user_id"]);
+ ASSERT_EQ("1", parsed_json["doc_id"]);
+ ASSERT_EQ("shorts", parsed_json["query"]);
+
+ values.clear();
+ analyticsManager.get_last_N_events("13", 5, values);
+ ASSERT_EQ(1, values.size());
+
+ parsed_json = nlohmann::json::parse(values[0]);
+ ASSERT_EQ("APC2", parsed_json["name"]);
+ ASSERT_EQ("books", parsed_json["collection"]);
+ ASSERT_EQ("13", parsed_json["user_id"]);
+ ASSERT_EQ("21", parsed_json["doc_id"]);
+ ASSERT_EQ("technology", parsed_json["query"]);
- create_op = analyticsManager.create_rule(analytics_rule, false, true);
- ASSERT_FALSE(create_op.ok());
- ASSERT_EQ("Event can't be logged when analytics-dir is not defined.", create_op.error());
- //counter events should work without analytic-dir
analyticsManager.dispose();
analyticsManager.stop();
- system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
- analyticsManager.init(store, analytic_store, "");
+ analyticsManager.init(store, analytic_store);
analytics_rule = R"({
"name": "books_popularity3",
@@ -1431,7 +1464,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreTTL) {
system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());
analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);
- analyticsManager.init(store, analytic_store, "");
+ analyticsManager.init(store, analytic_store);
auto analytics_rule = R"({
"name": "product_events2",
@@ -1511,7 +1544,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());
analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);
- analyticsManager.init(store, analytic_store, "");
+ analyticsManager.init(store, analytic_store);
auto analytics_rule = R"({
"name": "product_events2",
@@ -1520,7 +1553,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
"name": "product_events_logging2",
"source": {
"collections": ["titles"],
- "events": [{"type": "click", "name": "AB"}]
+ "events": [{"type": "click", "name": "AB"}, {"type": "visit", "name": "AV"}]
}
}
})"_json;
@@ -1612,6 +1645,84 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
values.clear();
analyticsManager.get_last_N_events("16", 8, values);
ASSERT_EQ(0, values.size());
+
+ //get specific event type or user
+
+ //add different type events
+ event1["name"] = "AV";
+ event1["type"] = "visit";
+ event1["data"]["user_id"] = "14";
+
+ for(auto i = 0; i < 5; i++) {
+ event1["data"]["doc_id"] = std::to_string(i);
+ req->body = event1.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+ }
+
+ payload.clear();
+ event_data.clear();
+ collection_events_map = analyticsManager.get_log_events();
+ for (auto &events_collection_it: collection_events_map) {
+ const auto& collection = events_collection_it.first;
+ for(const auto& event: events_collection_it.second) {
+ event.to_json(event_data, collection);
+ payload.push_back(event_data);
+ }
+ }
+
+ //manually trigger write to db
+ ASSERT_TRUE(analyticsManager.write_to_db(payload));
+
+ //get last 5 visit events for user_id 14
+ values.clear();
+ analyticsManager.get_last_N_events("14_visit", 5, values);
+ ASSERT_EQ(5, values.size());
+ for(int i = 0; i < 5; ++i) {
+ parsed_json = nlohmann::json::parse(values[i]);
+ ASSERT_EQ("AV", parsed_json["name"]);
+ ASSERT_EQ(std::to_string(4-i), parsed_json["doc_id"]);
+ }
+
+ //get last 5 click events for user_id 14
+ values.clear();
+ analyticsManager.get_last_N_events("14_click", 5, values);
+ ASSERT_EQ(5, values.size());
+ for(int i = 0; i < 5; ++i) {
+ parsed_json = nlohmann::json::parse(values[i]);
+ ASSERT_EQ("AB", parsed_json["name"]);
+ ASSERT_EQ(std::to_string(6-i), parsed_json["doc_id"]);
+ }
+
+ event1["name"] = "AB";
+ event1["type"] = "click";
+ event1["data"]["user_id"] = "14";
+
+ for(auto i = 7; i < 10; i++) {
+ event1["data"]["doc_id"] = std::to_string(i);
+ req->body = event1.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+ }
+
+ payload.clear();
+ event_data.clear();
+ collection_events_map = analyticsManager.get_log_events();
+ for (auto &events_collection_it: collection_events_map) {
+ const auto& collection = events_collection_it.first;
+
+ for(const auto& event: events_collection_it.second) {
+ event.to_json(event_data, collection);
+ payload.push_back(event_data);
+ }
+ }
+
+ ASSERT_TRUE(analyticsManager.write_to_db(payload));
+
+ values.clear();
+ analyticsManager.get_last_N_events("14_click", 10, values);
+ for(auto value : values) {
+ parsed_json = nlohmann::json::parse(value);
+ }
+ ASSERT_EQ(10, values.size());
}
TEST_F(AnalyticsManagerTest, AnalyticsWithAliases) {