Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

add events expiry and minor bug fix

krunal81bb569b8f6f
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index 37ddcbf1..4593c823 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -215,4 +215,6 @@ public:
uint64_t hits_count);
nlohmann::json get_query_hits_counts();
+
+ void checkEventsExpiry();
};
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index 75a403e6..1de66eac 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -5,10 +5,12 @@
#include "http_client.h"
#include "collection_manager.h"
#include "lru/lru.hpp"
+#include "string_utils.h"
LRU::Cache<std::string, event_cache_t> events_cache;
#define CLICK_EVENTS_RATE_LIMIT_SEC 60
#define CLICK_EVENTS_RATE_LIMIT_COUNT 5
+#define EVENTS_TTL_INTERVAL_US 2592000000000 //30days
Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk) {
/*
@@ -341,6 +343,7 @@ void AnalyticsManager::run(ReplicationState* raft_server) {
continue;
}
+ checkEventsExpiry();
persist_query_events(raft_server, prev_persistence_s);
persist_query_hits_click_events(raft_server, prev_persistence_s);
@@ -441,9 +444,9 @@ void AnalyticsManager::persist_query_hits_click_events(ReplicationState *raft_se
// lock is held by caller
nlohmann::json payload_json = nlohmann::json::array();
- auto send_http_response = [&](const std::string& event_type) {
+ auto send_http_response = [&](const std::string& event_type)->bool {
if(payload_json.empty()) {
- return;
+ return false;
}
const std::string import_payload = payload_json.dump();
@@ -461,10 +464,11 @@ void AnalyticsManager::persist_query_hits_click_events(ReplicationState *raft_se
if (status_code != 200) {
LOG(ERROR) << "Error while sending " << event_type <<" to leader. "
<< "Status code: " << status_code << ", response: " << res;
- } else {
- query_collection_click_events.clear();
+ return false;
}
+ return true;
}
+ return false;
};
for (const auto &click_events_collection_it: query_collection_click_events) {
@@ -480,8 +484,12 @@ void AnalyticsManager::persist_query_hits_click_events(ReplicationState *raft_se
}
}
- send_http_response("click_events");
+ if(send_http_response("click_events")) {
+ query_collection_click_events.clear();
+ }
+
+ payload_json.clear();
for (const auto &query_collection_hits_count_it: query_collection_hits_count) {
auto collection_id = CollectionManager::get_instance().get_collection(
@@ -495,7 +503,9 @@ void AnalyticsManager::persist_query_hits_click_events(ReplicationState *raft_se
payload_json.push_back(query_hits_count_json);
}
}
- send_http_response("query_hits_counts");
+ if(send_http_response("query_hits_counts")) {
+ query_collection_hits_count.clear();
+ }
}
void AnalyticsManager::stop() {
@@ -571,17 +581,17 @@ nlohmann::json AnalyticsManager::get_query_hits_counts() {
}
Option<bool> AnalyticsManager::write_events_to_store(nlohmann::json &event_jsons) {
+ //LOG(INFO) << "writing events to analytics db";
for(const auto& event_json : event_jsons) {
auto collection_id = event_json["collection_id"].get<std::string>();
- auto timestamp = event_json["timestamp"].get<uint64_t>();
+ auto timestamp = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
- std::string key = "";
+ std::string key = "_" + StringUtils::serialize_uint64_t(timestamp) + "_" + collection_id;
if(event_json["event_type"] == "click_events") {
- key = std::string(CLICK_EVENT) + "_" + collection_id + "_" +
- std::to_string(timestamp);
+ key = std::string(CLICK_EVENT) + key;
} else if(event_json["event_type"] == "query_hits_counts") {
- key = std::string(QUERY_HITS_COUNT) + "_" + collection_id + "_" +
- std::to_string(timestamp);
+ key = std::string(QUERY_HITS_COUNT) + key;
}
if(analytics_store) {
@@ -599,4 +609,42 @@ Option<bool> AnalyticsManager::write_events_to_store(nlohmann::json &event_jsons
void AnalyticsManager::resetRateLimit() {
events_cache.clear();
+}
+
+void AnalyticsManager::checkEventsExpiry() {
+ if (analytics_store) {
+ //LOG(INFO) << "checking for events expiry";
+
+ //we check for 30days events validity, events older than 30 days will be removed from db
+ auto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(
+ std::chrono::system_clock::now().time_since_epoch()).count();
+
+ auto ts_ttl_useconds = now_ts_useconds - EVENTS_TTL_INTERVAL_US;
+
+ const std::string click_events_prefix = std::string(CLICK_EVENT) + "_";
+ const std::string query_hits_prefix = std::string(QUERY_HITS_COUNT) + "_";
+
+ //first remove click events
+
+ auto delete_prefix_begin = click_events_prefix;
+ auto delete_prefix_end = delete_prefix_begin + StringUtils::serialize_uint64_t(ts_ttl_useconds);
+
+ auto iter = analytics_store->get_iterator();
+ iter->Seek(delete_prefix_end);
+ if (!iter->Valid()) { //exact key or key greater than not found
+ delete_prefix_end = query_hits_prefix;
+ }
+
+ analytics_store->delete_range(delete_prefix_begin, delete_prefix_end);
+
+ //now remove query hits counts
+ delete_prefix_begin = query_hits_prefix;
+ delete_prefix_end = std::string(QUERY_HITS_COUNT) + "_" +
+ StringUtils::serialize_uint64_t(ts_ttl_useconds);
+
+ iter->SeekToFirst();
+ iter->Seek(delete_prefix_end);
+
+ analytics_store->delete_range(delete_prefix_begin, delete_prefix_end);
+ }
}
\ No newline at end of file