Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
remove analytics logging to tsv file (#1843)
Krunal Gandhi99a774d35999
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 45c02832..84f7c751 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -142,7 +142,6 @@ private:Store* store = nullptr;Store* analytics_store = nullptr;- std::ofstream analytics_logs;bool isRateLimitEnabled = true;@@ -176,7 +175,7 @@ public:AnalyticsManager(AnalyticsManager const&) = delete;void operator=(AnalyticsManager const&) = delete;- void init(Store* store, Store* analytics_store, const std::string& analytics_dir="");+ void init(Store* store, Store* analytics_store);void run(ReplicationState* raft_server);diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex 738b9b7b..5e5d9395 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -223,9 +223,9 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json &payload, bool upsertif(event.contains("log_to_file")) {log_to_file = event["log_to_file"].get<bool>();- if(log_to_file && !analytics_logs.is_open()) {+ if(log_to_file && !analytics_store) {remove_index(suggestion_config_name);- return Option<bool>(400, "Event can't be logged when analytics-dir is not defined.");+ return Option<bool>(400, "Event can't be logged when analytics-db is not defined.");}}event_weight_map[event["name"]] = event["weight"];@@ -658,21 +658,7 @@ void AnalyticsManager::persist_events(ReplicationState *raft_server, uint64_t prfor (auto &events_collection_it: query_collection_events) {const auto& collection = events_collection_it.first;for (const auto &event: events_collection_it.second) {- if (analytics_logs.is_open() && event.log_to_file) {- //store events to log file- std::stringstream ssbuf;- ssbuf << event.timestamp << "\t" << event.name << "\t"- << collection << "\t" << event.user_id << "\t" << event.doc_id << "\t"- << event.query << "\t";-- for(const auto& kv : event.data) {- ssbuf << kv.second << "\t";- }-- analytics_logs << ssbuf.str();- analytics_logs << "\n";- analytics_logs << std::flush;-+ if (event.log_to_file) {nlohmann::json event_data;event.to_json(event_data, collection);payload.push_back(event_data);@@ -720,7 +706,6 @@ void AnalyticsManager::persist_popular_events(ReplicationState *raft_server, uinvoid AnalyticsManager::stop() {quit = true;cv.notify_all();- analytics_logs.close();}void AnalyticsManager::dispose() {@@ -751,13 +736,11 @@ void AnalyticsManager::dispose() {events_cache.clear();}-void AnalyticsManager::init(Store* store, Store* analytics_store, const std::string& analytics_dir) {+void AnalyticsManager::init(Store* store, Store* analytics_store) {this->store = store;this->analytics_store = analytics_store;- if(!analytics_dir.empty()) {- const auto analytics_log_path = analytics_dir + "/analytics_events.tsv";- analytics_logs.open(analytics_log_path, std::ofstream::out | std::ofstream::app);+ if(analytics_store) {events_cache.capacity(1024);}}@@ -799,7 +782,9 @@ void counter_event_t::serialize_as_docs(std::string &docs) {bool AnalyticsManager::write_to_db(const nlohmann::json& payload) {if(analytics_store) {for(const auto& event: payload) {- std::string key = event["user_id"].get<std::string>() + "_" + StringUtils::serialize_uint64_t(event["timestamp"].get<uint64_t>());+ std::string key = event["user_id"].get<std::string>() + "_" + event["type"].get<std::string>()+ + "_" + StringUtils::serialize_uint64_t(event["timestamp"].get<uint64_t>());+bool inserted = analytics_store->insert(key, event.dump());if(!inserted) {LOG(ERROR) << "Error while dumping events to analytics db.";@@ -814,8 +799,8 @@ bool AnalyticsManager::write_to_db(const nlohmann::json& payload) {return true;}-void AnalyticsManager::get_last_N_events(const std::string& userid, uint32_t N, std::vector<std::string>& values) {- const std::string userid_prefix = userid + "_";+void AnalyticsManager::get_last_N_events(const std::string& prefix, uint32_t N, std::vector<std::string>& values) {+ const std::string userid_prefix = prefix + "_";analytics_store->get_last_N_values(userid_prefix, N, values);}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 6bd1970a..eaea8d58 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -424,7 +424,7 @@ int run_server(const Config & config, const std::string & version, void (*masterHttpClient & httpClient = HttpClient::get_instance();httpClient.init(config.get_api_key());- AnalyticsManager::get_instance().init(&store, &analytics_store, analytics_dir);+ AnalyticsManager::get_instance().init(&store, &analytics_store);server = new HttpServer(version,diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex 4e9ba593..d970cff5 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -36,7 +36,7 @@ protected:collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);- analyticsManager.init(store, analytic_store, state_dir_path);+ analyticsManager.init(store, analytic_store);analyticsManager.resetToggleRateLimit(false);}@@ -524,18 +524,32 @@ TEST_F(AnalyticsManagerTest, EventsPersist) {req->body = event.dump();ASSERT_TRUE(post_create_event(req, res));- analyticsManager.persist_events(nullptr, 0);+ //get events+ nlohmann::json payload = nlohmann::json::array();+ nlohmann::json event_data;+ auto collection_events_map = analyticsManager.get_log_events();+ for (auto &events_collection_it: collection_events_map) {+ const auto& collection = events_collection_it.first;+ for(const auto& event: events_collection_it.second) {+ event.to_json(event_data, collection);+ payload.push_back(event_data);+ }+ }++ //manually trigger write to db+ ASSERT_TRUE(analyticsManager.write_to_db(payload));++ std::vector<std::string> values;+ analyticsManager.get_last_N_events("13", 5, values);+ ASSERT_EQ(1, values.size());- auto fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");+ auto parsed_json = nlohmann::json::parse(values[0]);- std::stringstream strbuff(fileOutput.get());- std::string docid, userid, q, collection, name, timestamp;- strbuff >> timestamp >> name >> collection >> userid >> docid >> q;- ASSERT_EQ("APC", name);- ASSERT_EQ("titles", collection);- ASSERT_EQ("13", userid);- ASSERT_EQ("21", docid);- ASSERT_EQ("technology", q);+ ASSERT_EQ("APC", parsed_json["name"]);+ ASSERT_EQ("titles", parsed_json["collection"]);+ ASSERT_EQ("13", parsed_json["user_id"]);+ ASSERT_EQ("21", parsed_json["doc_id"]);+ ASSERT_EQ("technology", parsed_json["query"]);event = R"({"type": "click",@@ -550,26 +564,40 @@ TEST_F(AnalyticsManagerTest, EventsPersist) {req->body = event.dump();ASSERT_TRUE(post_create_event(req, res));- analyticsManager.persist_events(nullptr, 0);-- fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");-- std::stringstream strbuff2(fileOutput.get());- timestamp.clear();name.clear();collection.clear();userid.clear();q.clear();- strbuff2 >> timestamp >> name >> collection >> userid >> docid >> q;- ASSERT_EQ("APC", name);- ASSERT_EQ("titles", collection);- ASSERT_EQ("13", userid);- ASSERT_EQ("21", docid);- ASSERT_EQ("technology", q);-- timestamp.clear();name.clear();collection.clear();userid.clear();q.clear();- strbuff2 >> timestamp >> name >> collection >> userid >> docid >> q;- ASSERT_EQ("APC", name);- ASSERT_EQ("titles", collection);- ASSERT_EQ("13", userid);- ASSERT_EQ("12", docid);- ASSERT_EQ("technology", q);+ //get events+ payload.clear();+ collection_events_map = analyticsManager.get_log_events();+ for (auto &events_collection_it: collection_events_map) {+ const auto& collection = events_collection_it.first;+ for(const auto& event: events_collection_it.second) {+ event.to_json(event_data, collection);+ payload.push_back(event_data);+ }+ }++ //manually trigger write to db+ ASSERT_TRUE(analyticsManager.write_to_db(payload));++ values.clear();+ analyticsManager.get_last_N_events("13", 5, values);+ ASSERT_EQ(2, values.size());++ parsed_json = nlohmann::json::parse(values[0]);++ //events will be fetched in LIFO order+ ASSERT_EQ("APC", parsed_json["name"]);+ ASSERT_EQ("titles", parsed_json["collection"]);+ ASSERT_EQ("13", parsed_json["user_id"]);+ ASSERT_EQ("12", parsed_json["doc_id"]);+ ASSERT_EQ("technology", parsed_json["query"]);++ parsed_json = nlohmann::json::parse(values[1]);++ ASSERT_EQ("APC", parsed_json["name"]);+ ASSERT_EQ("titles", parsed_json["collection"]);+ ASSERT_EQ("13", parsed_json["user_id"]);+ ASSERT_EQ("21", parsed_json["doc_id"]);+ ASSERT_EQ("technology", parsed_json["query"]);}TEST_F(AnalyticsManagerTest, EventsRateLimitTest) {@@ -1016,8 +1044,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {//restart analytics manager as freshanalyticsManager.dispose();analyticsManager.stop();- system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");- analyticsManager.init(store, analytic_store, "/tmp/typesense_test/analytics_manager_test");+ analyticsManager.init(store, analytic_store);nlohmann::json products_schema = R"({"name": "books",@@ -1262,19 +1289,30 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {ASSERT_EQ(0, results["hits"][1]["document"]["popularity"]);ASSERT_EQ("Cool trousers", results["hits"][1]["document"]["title"]);- //verify log file- analyticsManager.persist_events(nullptr, 0);+ nlohmann::json payload = nlohmann::json::array();+ nlohmann::json event_data;+ auto collection_events_map = analyticsManager.get_log_events();+ for (auto &events_collection_it: collection_events_map) {+ const auto& collection = events_collection_it.first;+ for(const auto& event: events_collection_it.second) {+ event.to_json(event_data, collection);+ payload.push_back(event_data);+ }+ }++ //manually trigger write to db+ ASSERT_TRUE(analyticsManager.write_to_db(payload));- auto fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");+ std::vector<std::string> values;+ analyticsManager.get_last_N_events("11", 5, values);+ ASSERT_EQ(1, values.size());- std::stringstream strbuff(fileOutput.get());- std::string timestamp, collection, userid, name, q, docid;- strbuff >> timestamp >> name >> collection >> userid >> docid >> q;- ASSERT_EQ("CNV4", name);- ASSERT_EQ("books", collection);- ASSERT_EQ("11", userid);- ASSERT_EQ("1", docid);- ASSERT_EQ("shorts", q);+ auto parsed_json = nlohmann::json::parse(values[0]);+ ASSERT_EQ("CNV4", parsed_json["name"]);+ ASSERT_EQ("books", parsed_json["collection"]);+ ASSERT_EQ("11", parsed_json["user_id"]);+ ASSERT_EQ("1", parsed_json["doc_id"]);+ ASSERT_EQ("shorts", parsed_json["query"]);//now add click event ruleanalytics_rule = R"({@@ -1330,59 +1368,54 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {ASSERT_EQ(1, popular_clicks["books"].docid_counts.size());ASSERT_EQ(10, popular_clicks["books"].docid_counts["1"]);- //check log file- analyticsManager.persist_events(nullptr, 0);-- fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");-- std::stringstream strbuff2(fileOutput.get());- timestamp.clear(), collection.clear(), userid.clear(), name.clear(), q.clear(), docid.clear();- strbuff2 >> timestamp >> name >> collection >> userid >> docid >> q;- ASSERT_EQ("CNV4", name);- ASSERT_EQ("books", collection);- ASSERT_EQ("11", userid);- ASSERT_EQ("1", docid);- ASSERT_EQ("shorts", q);-- timestamp.clear(), collection.clear(), userid.clear(), name.clear(), q.clear(), docid.clear();- strbuff2 >> timestamp >> name >> collection >> userid >> docid >> q;- ASSERT_EQ("APC2", name);- ASSERT_EQ("books", collection);- ASSERT_EQ("13", userid);- ASSERT_EQ("21", docid);- ASSERT_EQ("technology", q);-- //counter event rule should not be allowed with logging if analytics-dir is not specified- analyticsManager.dispose();- analyticsManager.stop();- system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");- analyticsManager.init(store, analytic_store, "");-- analytics_rule = R"({- "name": "books_popularity3",- "type": "counter",- "params": {- "source": {- "collections": ["books"],- "events": [{"type": "conversion", "weight": 5, "name": "CNV4", "log_to_file" : true} ]- },- "destination": {- "collection": "books",- "counter_field": "popularity"- }+ payload.clear();+ event_data.clear();+ collection_events_map = analyticsManager.get_log_events();+ for (auto &events_collection_it: collection_events_map) {+ const auto& collection = events_collection_it.first;+ for(const auto& event: events_collection_it.second) {+ event.to_json(event_data, collection);+ payload.push_back(event_data);}- })"_json;+ }++ //manually trigger write to db+ ASSERT_TRUE(analyticsManager.write_to_db(payload));++ values.clear();+ analyticsManager.get_last_N_events("11", 5, values);+ ASSERT_EQ(2, values.size());++ parsed_json = nlohmann::json::parse(values[0]);+ ASSERT_EQ("CNV4", parsed_json["name"]);+ ASSERT_EQ("books", parsed_json["collection"]);+ ASSERT_EQ("11", parsed_json["user_id"]);+ ASSERT_EQ("1", parsed_json["doc_id"]);+ ASSERT_EQ("shorts", parsed_json["query"]);++ parsed_json = nlohmann::json::parse(values[1]);+ ASSERT_EQ("CNV4", parsed_json["name"]);+ ASSERT_EQ("books", parsed_json["collection"]);+ ASSERT_EQ("11", parsed_json["user_id"]);+ ASSERT_EQ("1", parsed_json["doc_id"]);+ ASSERT_EQ("shorts", parsed_json["query"]);++ values.clear();+ analyticsManager.get_last_N_events("13", 5, values);+ ASSERT_EQ(1, values.size());++ parsed_json = nlohmann::json::parse(values[0]);+ ASSERT_EQ("APC2", parsed_json["name"]);+ ASSERT_EQ("books", parsed_json["collection"]);+ ASSERT_EQ("13", parsed_json["user_id"]);+ ASSERT_EQ("21", parsed_json["doc_id"]);+ ASSERT_EQ("technology", parsed_json["query"]);- create_op = analyticsManager.create_rule(analytics_rule, false, true);- ASSERT_FALSE(create_op.ok());- ASSERT_EQ("Event can't be logged when analytics-dir is not defined.", create_op.error());- //counter events should work without analytic-diranalyticsManager.dispose();analyticsManager.stop();- system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");- analyticsManager.init(store, analytic_store, "");+ analyticsManager.init(store, analytic_store);analytics_rule = R"({"name": "books_popularity3",@@ -1431,7 +1464,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreTTL) {system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);- analyticsManager.init(store, analytic_store, "");+ analyticsManager.init(store, analytic_store);auto analytics_rule = R"({"name": "product_events2",@@ -1511,7 +1544,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);- analyticsManager.init(store, analytic_store, "");+ analyticsManager.init(store, analytic_store);auto analytics_rule = R"({"name": "product_events2",@@ -1520,7 +1553,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {"name": "product_events_logging2","source": {"collections": ["titles"],- "events": [{"type": "click", "name": "AB"}]+ "events": [{"type": "click", "name": "AB"}, {"type": "visit", "name": "AV"}]}}})"_json;@@ -1612,6 +1645,84 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {values.clear();analyticsManager.get_last_N_events("16", 8, values);ASSERT_EQ(0, values.size());++ //get specific event type or user++ //add different type events+ event1["name"] = "AV";+ event1["type"] = "visit";+ event1["data"]["user_id"] = "14";++ for(auto i = 0; i < 5; i++) {+ event1["data"]["doc_id"] = std::to_string(i);+ req->body = event1.dump();+ ASSERT_TRUE(post_create_event(req, res));+ }++ payload.clear();+ event_data.clear();+ collection_events_map = analyticsManager.get_log_events();+ for (auto &events_collection_it: collection_events_map) {+ const auto& collection = events_collection_it.first;+ for(const auto& event: events_collection_it.second) {+ event.to_json(event_data, collection);+ payload.push_back(event_data);+ }+ }++ //manually trigger write to db+ ASSERT_TRUE(analyticsManager.write_to_db(payload));++ //get last 5 visit events for user_id 14+ values.clear();+ analyticsManager.get_last_N_events("14_visit", 5, values);+ ASSERT_EQ(5, values.size());+ for(int i = 0; i < 5; ++i) {+ parsed_json = nlohmann::json::parse(values[i]);+ ASSERT_EQ("AV", parsed_json["name"]);+ ASSERT_EQ(std::to_string(4-i), parsed_json["doc_id"]);+ }++ //get last 5 click events for user_id 14+ values.clear();+ analyticsManager.get_last_N_events("14_click", 5, values);+ ASSERT_EQ(5, values.size());+ for(int i = 0; i < 5; ++i) {+ parsed_json = nlohmann::json::parse(values[i]);+ ASSERT_EQ("AB", parsed_json["name"]);+ ASSERT_EQ(std::to_string(6-i), parsed_json["doc_id"]);+ }++ event1["name"] = "AB";+ event1["type"] = "click";+ event1["data"]["user_id"] = "14";++ for(auto i = 7; i < 10; i++) {+ event1["data"]["doc_id"] = std::to_string(i);+ req->body = event1.dump();+ ASSERT_TRUE(post_create_event(req, res));+ }++ payload.clear();+ event_data.clear();+ collection_events_map = analyticsManager.get_log_events();+ for (auto &events_collection_it: collection_events_map) {+ const auto& collection = events_collection_it.first;++ for(const auto& event: events_collection_it.second) {+ event.to_json(event_data, collection);+ payload.push_back(event_data);+ }+ }++ ASSERT_TRUE(analyticsManager.write_to_db(payload));++ values.clear();+ analyticsManager.get_last_N_events("14_click", 10, values);+ for(auto value : values) {+ parsed_json = nlohmann::json::parse(value);+ }+ ASSERT_EQ(10, values.size());}TEST_F(AnalyticsManagerTest, AnalyticsWithAliases) {