Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

remove sleep from tests

krunal4fc0482875bb
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index f0931937..9d9e9fd3 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -216,5 +216,9 @@ public:
nlohmann::json get_query_hits_counts();
- void checkEventsExpiry(uint64_t events_ttl_interval=2592000000000); //30days default
+ void checkEventsExpiry();
+
+ uint64_t get_current_time_us();
+
+ void resetAnalyticsStore();
};
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index 6e2ff532..74c3b1df 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -10,6 +10,7 @@
LRU::Cache<std::string, event_cache_t> events_cache;
#define CLICK_EVENTS_RATE_LIMIT_SEC 60
#define CLICK_EVENTS_RATE_LIMIT_COUNT 5
+#define EVENTS_TTL_INTERVAL_US 2592000000000 //30days
Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk) {
/*
@@ -609,21 +610,49 @@ void AnalyticsManager::resetRateLimit() {
events_cache.clear();
}
-void AnalyticsManager::checkEventsExpiry(uint64_t events_ttl_interval) {
+void AnalyticsManager::resetAnalyticsStore() {
+ const std::string click_events_prefix = std::string(CLICK_EVENT) + "_";
+ const std::string query_hits_prefix = std::string(QUERY_HITS_COUNT) + "_";
+
+ //delete click events
+ auto delete_prefix_begin = click_events_prefix;
+ auto delete_prefix_end = click_events_prefix + "`";
+
+ analytics_store->delete_range(delete_prefix_begin, delete_prefix_end);
+
+ //delete query hits counts
+ delete_prefix_begin = query_hits_prefix;
+ delete_prefix_end = query_hits_prefix + "`";
+
+ analytics_store->delete_range(delete_prefix_begin, delete_prefix_end);
+}
+
+#ifdef TEST_BUILD
+uint64_t AnalyticsManager::get_current_time_us() {
+ uint64_t now_ts_useconds = 1701851345000000 + EVENTS_TTL_INTERVAL_US;
+
+ return now_ts_useconds;
+}
+#else
+uint64_t AnalyticsManager::get_current_time_us() {
+ uint64_t now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ return now_ts_useconds;
+}
+#endif
+
+void AnalyticsManager::checkEventsExpiry() {
if (analytics_store) {
//LOG(INFO) << "checking for events expiry";
//we check for 30days events validity, events older than 30 days will be removed from db
- auto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(
- std::chrono::system_clock::now().time_since_epoch()).count();
-
- auto ts_ttl_useconds = now_ts_useconds - events_ttl_interval;
+ auto ts_ttl_useconds = get_current_time_us() - EVENTS_TTL_INTERVAL_US;
const std::string click_events_prefix = std::string(CLICK_EVENT) + "_";
const std::string query_hits_prefix = std::string(QUERY_HITS_COUNT) + "_";
//first remove click events
-
auto delete_prefix_begin = click_events_prefix;
auto delete_prefix_end = delete_prefix_begin + StringUtils::serialize_uint64_t(ts_ttl_useconds);
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index 397ef78b..457d9c75 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -502,12 +502,9 @@ TEST_F(AnalyticsManagerTest, QueryHitsCount) {
ASSERT_EQ(1625365616, result[1]["timestamp"]);
}
-TEST_F(AnalyticsManagerTest, EventsExpiry) {
- //delete the analytic store db and reinit
- delete analytics_store;
- std::string analytics_db_path = "/tmp/typesense_test/analytics_db2";
- analytics_store = new Store(analytics_db_path);
- analyticsManager.init(store, analytics_store);
+TEST_F(AnalyticsManagerTest, EventsExpiryBasic) {
+ //flush all events from analytics store
+ analyticsManager.resetAnalyticsStore();
nlohmann::json titles_schema = R"({
"name": "titles",
@@ -520,6 +517,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {
nlohmann::json events = nlohmann::json::array();
+ auto ts = 1701851341000000;
nlohmann::json event;
event["event_type"] = "click_events";
event["q"] = "technology";
@@ -527,14 +525,12 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {
event["doc_id"] = "21";
event["position"] = 2;
event["user_id"] = 13;
- event["timestamp"] = std::chrono::duration_cast<std::chrono::microseconds>(
- std::chrono::system_clock::now().time_since_epoch()).count();
+ event["timestamp"] = ts;
events.push_back(event);
event["doc_id"] = "12";
event["position"] = 3;
- event["timestamp"] = std::chrono::duration_cast<std::chrono::microseconds>(
- std::chrono::system_clock::now().time_since_epoch()).count();
+ event["timestamp"] = ts+1000000; //1 sec later
events.push_back(event);
ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());
@@ -551,10 +547,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {
ASSERT_EQ(3, resp[1]["position"]);
ASSERT_EQ(13, resp[1]["user_id"]);
- LOG(INFO) << "waiting for TTL to pass";
- sleep(1);//wait till ttl passes
-
- analyticsManager.checkEventsExpiry(1000000); //1 second
+ analyticsManager.checkEventsExpiry();
resp = analyticsManager.get_click_events();
ASSERT_EQ(0, resp.size());
@@ -566,8 +559,7 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {
event2["collection_id"] = "0";
event2["user_id"] = 13;
event2["hits_count"] = 124;
- event2["timestamp"] = std::chrono::duration_cast<std::chrono::microseconds>(
- std::chrono::system_clock::now().time_since_epoch()).count();
+ event2["timestamp"] = ts + 10000000; //after 10s
events.push_back(event2);
@@ -580,7 +572,8 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {
ASSERT_EQ(124, resp[0]["hits_count"]);
//now old click events will be deleted on checking expiry but query hits events will be remaining
- analyticsManager.checkEventsExpiry(1000000); //1 second
+ // assumming ttl is ts + 5 sec
+ analyticsManager.checkEventsExpiry();
resp = analyticsManager.get_click_events();
ASSERT_EQ(0, resp.size());
@@ -590,4 +583,76 @@ TEST_F(AnalyticsManagerTest, EventsExpiry) {
ASSERT_EQ("technology", resp[0]["q"]);
ASSERT_EQ(13, resp[0]["user_id"]);
ASSERT_EQ(124, resp[0]["hits_count"]);
+}
+
+TEST_F(AnalyticsManagerTest, EventsExpiryAll) {
+ //flush all events from analytics store
+ analyticsManager.resetAnalyticsStore();
+
+ nlohmann::json titles_schema = R"({
+ "name": "titles",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection *titles_coll = collectionManager.create_collection(titles_schema).get();
+
+ nlohmann::json events = nlohmann::json::array();
+
+ auto ts = 1701851341000000;
+ nlohmann::json event;
+ event["event_type"] = "click_events";
+ event["q"] = "technology";
+ event["collection_id"] = "0";
+ event["doc_id"] = "21";
+ event["position"] = 2;
+ event["user_id"] = 13;
+ event["timestamp"] = ts;
+ events.push_back(event);
+
+ event["doc_id"] = "12";
+ event["position"] = 3;
+ event["timestamp"] = ts + 1000000; //after 1s
+ events.push_back(event);
+
+ nlohmann::json event2;
+ event2["event_type"] = "query_hits_counts";
+ event2["q"] = "technology";
+ event2["collection_id"] = "0";
+ event2["user_id"] = 13;
+ event2["hits_count"] = 124;
+ event2["timestamp"] = ts + 2000000; //after 2s
+
+ events.push_back(event2);
+
+ ASSERT_TRUE(analyticsManager.write_events_to_store(events).ok());
+
+ nlohmann::json resp = analyticsManager.get_click_events();
+ ASSERT_EQ(2, resp.size());
+ ASSERT_EQ("technology", resp[0]["q"]);
+ ASSERT_EQ("21", resp[0]["doc_id"]);
+ ASSERT_EQ(2, resp[0]["position"]);
+ ASSERT_EQ(13, resp[0]["user_id"]);
+
+ ASSERT_EQ("technology", resp[1]["q"]);
+ ASSERT_EQ("12", resp[1]["doc_id"]);
+ ASSERT_EQ(3, resp[1]["position"]);
+ ASSERT_EQ(13, resp[1]["user_id"]);
+
+ resp = analyticsManager.get_query_hits_counts();
+ ASSERT_EQ(1, resp.size());
+ ASSERT_EQ("technology", resp[0]["q"]);
+ ASSERT_EQ(13, resp[0]["user_id"]);
+ ASSERT_EQ(124, resp[0]["hits_count"]);
+
+ //check for events expiry
+ // assuming ttl is ts + 5 sec
+ analyticsManager.checkEventsExpiry();
+
+ resp = analyticsManager.get_click_events();
+ ASSERT_EQ(0, resp.size());
+
+ resp = analyticsManager.get_query_hits_counts();
+ ASSERT_EQ(0, resp.size());
}
\ No newline at end of file