Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
remove sleep from tests
krunal4fc0482875bb
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex f0931937..9d9e9fd3 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -216,5 +216,9 @@ public:nlohmann::json get_query_hits_counts();- void checkEventsExpiry(uint64_t events_ttl_interval=2592000000000); //30days default+ void checkEventsExpiry();++ uint64_t get_current_time_us();++ void resetAnalyticsStore();};diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex 6e2ff532..74c3b1df 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -10,6 +10,7 @@LRU::Cache<std::string, event_cache_t> events_cache;#define CLICK_EVENTS_RATE_LIMIT_SEC 60#define CLICK_EVENTS_RATE_LIMIT_COUNT 5+#define EVENTS_TTL_INTERVAL_US 2592000000000 //30daysOption<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk) {/*@@ -609,21 +610,49 @@ void AnalyticsManager::resetRateLimit() {events_cache.clear();}-void AnalyticsManager::checkEventsExpiry(uint64_t events_ttl_interval) {+void AnalyticsManager::resetAnalyticsStore() {+ const std::string click_events_prefix = std::string(CLICK_EVENT) + "_";+ const std::string query_hits_prefix = std::string(QUERY_HITS_COUNT) + "_";++ //delete click events+ auto delete_prefix_begin = click_events_prefix;+ auto delete_prefix_end = click_events_prefix + "`";++ analytics_store->delete_range(delete_prefix_begin, delete_prefix_end);++ //delete query hits counts+ delete_prefix_begin = query_hits_prefix;+ delete_prefix_end = query_hits_prefix + "`";++ analytics_store->delete_range(delete_prefix_begin, delete_prefix_end);+}++#ifdef TEST_BUILD+uint64_t AnalyticsManager::get_current_time_us() {+ uint64_t now_ts_useconds = 1701851345000000 + EVENTS_TTL_INTERVAL_US;++ return now_ts_useconds;+}+#else+uint64_t AnalyticsManager::get_current_time_us() {+ uint64_t now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ return now_ts_useconds;+}+#endif++void AnalyticsManager::checkEventsExpiry() {if (analytics_store) {//LOG(INFO) << "checking for events expiry";//we check for 30days events validity, events older than 30 days will be removed from db- auto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(- std::chrono::system_clock::now().time_since_epoch()).count();-- auto ts_ttl_useconds = now_ts_useconds - events_ttl_interval;+ auto ts_ttl_useconds = get_current_time_us() - EVENTS_TTL_INTERVAL_US;const std::string click_events_prefix = std::string(CLICK_EVENT) + "_";const std::string query_hits_prefix = std::string(QUERY_HITS_COUNT) + "_";//first remove click events-auto delete_prefix_begin = click_events_prefix;auto delete_prefix_end = delete_prefix_begin + StringUtils::serialize_uint64_t(ts_ttl_useconds);diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex 397ef78b..457d9c75 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -502,12 +502,9 @@ TEST_F(AnalyticsManagerTest, QueryHitsCount) {ASSERT_EQ(1625365616, result[1]["timestamp"]);}-TEST_F(AnalyticsManagerTest, EventsExpiry) {- //delete the analytic store db and reinit- delete analytics_store;- std::string analytics_db_path = "/tmp/typesense_test/analytics_db2";- analytics_store = new Store(analytics_db_path);- analyticsManager.init(store, analytics_store);+TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {+ //flush all events from analytics store+ analyticsManager.resetAnalyticsStore();nlohmann::json titles_schema = R"({"name": "titles",@@ -520,6 +517,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {nlohmann::json events = nlohmann::json::array();+ auto ts = 1701851341000000;nlohmann::json event;event["event_type"] = "click_events";event["q"] = "technology";@@ -527,14 +525,12 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {event["doc_id"] = "21";event["position"] = 2;event["user_id"] = 13;- event["timestamp"] = std::chrono::duration_cast<std::chrono::microseconds>(- std::chrono::system_clock::now().time_since_epoch()).count();+ event["timestamp"] = ts;events.push_back(event);event["doc_id"] = "12";event["position"] = 3;- event["timestamp"] = std::chrono::duration_cast<std::chrono::microseconds>(- std::chrono::system_clock::now().time_since_epoch()).count();+ event["timestamp"] = ts+1000000; //1 sec laterevents.push_back(event);ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());@@ -551,10 +547,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {ASSERT_EQ(3, resp[1]["position"]);ASSERT_EQ(13, resp[1]["user_id"]);- LOG(INFO) << "waiting for TTL to pass";- sleep(1);//wait till ttl passes-- analyticsManager.checkEventsExpiry(1000000); //1 second+ analyticsManager.checkEventsExpiry();resp = analyticsManager.get_click_events();ASSERT_EQ(0, resp.size());@@ -566,8 +559,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {event2["collection_id"] = "0";event2["user_id"] = 13;event2["hits_count"] = 124;- event2["timestamp"] = std::chrono::duration_cast<std::chrono::microseconds>(- std::chrono::system_clock::now().time_since_epoch()).count();+ event2["timestamp"] = ts + 10000000; //after 10sevents.push_back(event2);@@ -580,7 +572,8 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {ASSERT_EQ(124, resp[0]["hits_count"]);//now old click events will be deleted on checking expiry but query hits events will be remaining- analyticsManager.checkEventsExpiry(1000000); //1 second+ // assumming ttl is ts + 5 sec+ analyticsManager.checkEventsExpiry();resp = analyticsManager.get_click_events();ASSERT_EQ(0, resp.size());@@ -590,4 +583,76 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {ASSERT_EQ("technology", resp[0]["q"]);ASSERT_EQ(13, resp[0]["user_id"]);ASSERT_EQ(124, resp[0]["hits_count"]);+}++TEST_F(AnalyticsManagerTest, EventsExpiryAll) {+ //flush all events from analytics store+ analyticsManager.resetAnalyticsStore();++ nlohmann::json titles_schema = R"({+ "name": "titles",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection *titles_coll = collectionManager.create_collection(titles_schema).get();++ nlohmann::json events = nlohmann::json::array();++ auto ts = 1701851341000000;+ nlohmann::json event;+ event["event_type"] = "click_events";+ event["q"] = "technology";+ event["collection_id"] = "0";+ event["doc_id"] = "21";+ event["position"] = 2;+ event["user_id"] = 13;+ event["timestamp"] = ts;+ events.push_back(event);++ event["doc_id"] = "12";+ event["position"] = 3;+ event["timestamp"] = ts + 1000000; //after 1s+ events.push_back(event);++ nlohmann::json event2;+ event2["event_type"] = "query_hits_counts";+ event2["q"] = "technology";+ event2["collection_id"] = "0";+ event2["user_id"] = 13;+ event2["hits_count"] = 124;+ event2["timestamp"] = ts + 2000000; //after 2s++ events.push_back(event2);++ ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());++ nlohmann::json resp = analyticsManager.get_click_events();+ ASSERT_EQ(2, resp.size());+ ASSERT_EQ("technology", resp[0]["q"]);+ ASSERT_EQ("21", resp[0]["doc_id"]);+ ASSERT_EQ(2, resp[0]["position"]);+ ASSERT_EQ(13, resp[0]["user_id"]);++ ASSERT_EQ("technology", resp[1]["q"]);+ ASSERT_EQ("12", resp[1]["doc_id"]);+ ASSERT_EQ(3, resp[1]["position"]);+ ASSERT_EQ(13, resp[1]["user_id"]);++ resp = analyticsManager.get_query_hits_counts();+ ASSERT_EQ(1, resp.size());+ ASSERT_EQ("technology", resp[0]["q"]);+ ASSERT_EQ(13, resp[0]["user_id"]);+ ASSERT_EQ(124, resp[0]["hits_count"]);++ //check for events expiry+ // assuming ttl is ts + 5 sec+ analyticsManager.checkEventsExpiry();++ resp = analyticsManager.get_click_events();+ ASSERT_EQ(0, resp.size());++ resp = analyticsManager.get_query_hits_counts();+ ASSERT_EQ(0, resp.size());}\ No newline at end of file