Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

add analytics rate limit config (#1879)

Krunal Gandhifea777f41e5b
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index 726486fe..565a5359 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -146,6 +146,8 @@ private:
bool isRateLimitEnabled = true;
+ uint32_t analytics_minute_rate_limit;
+
AnalyticsManager() {}
~AnalyticsManager();
@@ -177,7 +179,7 @@ public:
AnalyticsManager(AnalyticsManager const&) = delete;
void operator=(AnalyticsManager const&) = delete;
- void init(Store* store, Store* analytics_store);
+ void init(Store* store, Store* analytics_store, uint32_t analytics_minute_rate_limit);
void run(ReplicationState* raft_server);
diff --git a/include/tsconfig.h b/include/tsconfig.h
index 089d7f8f..0e94aa6e 100644
--- a/include/tsconfig.h
+++ b/include/tsconfig.h
@@ -13,6 +13,7 @@ private:
std::string log_dir;
std::string analytics_dir;
int32_t analytics_db_ttl = 2419200; //four weeks in secs
+ uint32_t analytics_minute_rate_limit = 5;
std::string api_key;
// @deprecated
@@ -156,6 +157,10 @@ public:
this->analytics_db_ttl = analytics_db_ttl;
}
+ void set_analytics_minute_rate_limit(int32_t analytics_minute_rate_limit) {
+ this->analytics_minute_rate_limit = analytics_minute_rate_limit;
+ }
+
void set_api_key(const std::string & api_key) {
this->api_key = api_key;
}
@@ -238,6 +243,9 @@ public:
int32_t get_analytics_db_ttl() const {
return this->analytics_db_ttl;
}
+ int32_t get_analytics_minute_rate_limit() const {
+ return this->analytics_minute_rate_limit;
+ }
std::string get_api_key() const {
return this->api_key;
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index 0a51c859..c985cd49 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -7,7 +7,6 @@
#include "string_utils.h"
#define EVENTS_RATE_LIMIT_SEC 60
-#define EVENTS_RATE_LIMIT_COUNT 5
Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk) {
@@ -406,7 +405,7 @@ Option<bool> AnalyticsManager::add_event(const std::string& client_ip, const std
if (events_cache_it != events_cache.end()) {
// event found in events cache
if ((now_ts_seconds - events_cache_it->second.last_update_time) < EVENTS_RATE_LIMIT_SEC) {
- if (events_cache_it->second.count >= EVENTS_RATE_LIMIT_COUNT) {
+ if (events_cache_it->second.count >= analytics_minute_rate_limit) {
return Option<bool>(500, "event rate limit reached.");
} else {
events_cache_it->second.count++;
@@ -733,9 +732,10 @@ void AnalyticsManager::dispose() {
events_cache.clear();
}
-void AnalyticsManager::init(Store* store, Store* analytics_store) {
+void AnalyticsManager::init(Store* store, Store* analytics_store, uint32_t analytics_minute_rate_limit) {
this->store = store;
this->analytics_store = analytics_store;
+ this->analytics_minute_rate_limit = analytics_minute_rate_limit;
if(analytics_store) {
events_cache.capacity(1024);
diff --git a/src/tsconfig.cpp b/src/tsconfig.cpp
index e214396c..1714fc87 100644
--- a/src/tsconfig.cpp
+++ b/src/tsconfig.cpp
@@ -268,6 +268,10 @@ void Config::load_config_env() {
if(!get_env("TYPESENSE_ANALYTICS_DB_TTL").empty()) {
this->analytics_db_ttl = std::stoi(get_env("TYPESENSE_ANALYTICS_DB_TTL"));
}
+
+ if(!get_env("TYPESENSE_ANALYTICS_MINUTE_RATE_LIMIT").empty()) {
+ this->analytics_minute_rate_limit = std::stoi(get_env("TYPESENSE_ANALYTICS_MINUTE_RATE_LIMIT"));
+ }
}
void Config::load_config_file(cmdline::parser& options) {
@@ -303,7 +307,11 @@ void Config::load_config_file(cmdline::parser& options) {
}
if(reader.Exists("server", "analytics-db-ttl")) {
- this->analytics_db_ttl = std::stoi(reader.Get("server", "analytics-db-ttl", "0"));
+ this->analytics_db_ttl = reader.GetInteger("server", "analytics-db-ttl", 0);
+ }
+
+ if(reader.Exists("server", "analytics-minute-rate-limit")) {
+ this->analytics_minute_rate_limit = reader.GetInteger("server", "analytics-minute-rate-limit", 5);
}
if(reader.Exists("server", "api-key")) {
@@ -498,6 +506,10 @@ void Config::load_config_cmd_args(cmdline::parser& options) {
this->analytics_db_ttl = options.get<uint32_t>("analytics-db-ttl");
}
+ if(options.exist("analytics-minute-rate-limit")) {
+ this->analytics_minute_rate_limit = options.get<uint32_t>("analytics-minute-rate-limit");
+ }
+
if(options.exist("api-key")) {
this->api_key = options.get<std::string>("api-key");
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index ba13dca8..60a84956 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -67,6 +67,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("health-rusage-api-key", '\0', "API key that allows access to health end-point with resource usage.", false);
options.add<std::string>("analytics-dir", '\0', "Directory where Analytics will be stored.", false);
options.add<uint32_t>("analytics-db-ttl", '\0', "TTL in seconds for events stored in analytics db", false);
+ options.add<uint32_t>("analytics-minute-rate-limit", '\0', "per minute rate limit for /events endpoint", false);
+
options.add<std::string>("api-address", '\0', "Address to which Typesense API service binds.", false, "0.0.0.0");
options.add<uint32_t>("api-port", '\0', "Port on which Typesense API service listens.", false, 8108);
@@ -386,6 +388,7 @@ int run_server(const Config & config, const std::string & version, void (*master
std::string meta_dir = config.get_data_dir() + "/meta";
std::string analytics_dir = config.get_analytics_dir();
int32_t analytics_db_ttl = config.get_analytics_db_ttl();
+ uint32_t analytics_minute_rate_limit = config.get_analytics_minute_rate_limit();
size_t thread_pool_size = config.get_thread_pool_size();
@@ -414,7 +417,7 @@ int run_server(const Config & config, const std::string & version, void (*master
HttpClient & httpClient = HttpClient::get_instance();
httpClient.init(config.get_api_key());
- AnalyticsManager::get_instance().init(&store, &analytics_store);
+ AnalyticsManager::get_instance().init(&store, &analytics_store, analytics_minute_rate_limit);
server = new HttpServer(
version,
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index 10c78c98..6895d29f 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -18,6 +18,7 @@ protected:
std::vector<sort_by> sort_fields;
AnalyticsManager& analyticsManager = AnalyticsManager::get_instance();
+ uint32_t analytics_minute_rate_limit = 5;
void setupCollection() {
state_dir_path = "/tmp/typesense_test/analytics_manager_test";
@@ -36,7 +37,7 @@ protected:
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
- analyticsManager.init(store, analytic_store);
+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);
analyticsManager.resetToggleRateLimit(false);
}
@@ -836,6 +837,53 @@ TEST_F(AnalyticsManagerTest, EventsRateLimitTest) {
ASSERT_EQ("{\"message\": \"event rate limit reached.\"}", res->body);
analyticsManager.resetToggleRateLimit(false);
+
+
+ //try with different limit
+ //restart analytics manager as fresh
+ analyticsManager.dispose();
+ analyticsManager.stop();
+
+ analytics_minute_rate_limit = 20;
+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);
+
+ analytics_rule = R"({
+ "name": "product_events2",
+ "type": "log",
+ "params": {
+ "source": {
+ "collections": ["titles"],
+ "events": [{"type": "click", "name": "AB"}]
+ }
+ }
+ })"_json;
+
+ create_op = analyticsManager.create_rule(analytics_rule, true, true);
+ ASSERT_TRUE(create_op.ok());
+
+ event1 = R"({
+ "type": "click",
+ "name": "AB",
+ "data": {
+ "q": "technology",
+ "doc_id": "21",
+ "user_id": "13"
+ }
+ })"_json;
+
+ //reset the LRU cache to test the rate limit
+ analyticsManager.resetToggleRateLimit(true);
+
+ for(auto i = 0; i < 20; ++i) {
+ req->body = event1.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+ }
+
+ //as rate limit is 20, adding one more event above that should trigger rate limit
+ ASSERT_FALSE(post_create_event(req, res));
+ ASSERT_EQ("{\"message\": \"event rate limit reached.\"}", res->body);
+
+ analyticsManager.resetToggleRateLimit(false);
}
TEST_F(AnalyticsManagerTest, NoresultsQueries) {
@@ -1244,7 +1292,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
//restart analytics manager as fresh
analyticsManager.dispose();
analyticsManager.stop();
- analyticsManager.init(store, analytic_store);
+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);
nlohmann::json products_schema = R"({
"name": "books",
@@ -1606,7 +1654,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
analyticsManager.dispose();
analyticsManager.stop();
- analyticsManager.init(store, analytic_store);
+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);
analytics_rule = R"({
"name": "books_popularity3",
@@ -1654,7 +1702,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreTTL) {
system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());
analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);
- analyticsManager.init(store, analytic_store);
+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);
auto analytics_rule = R"({
"name": "product_events2",
@@ -1733,7 +1781,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());
analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);
- analyticsManager.init(store, analytic_store);
+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);
auto analytics_rule = R"({
"name": "product_events2",
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 7e2720bd..8d0cef5e 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -28,7 +28,7 @@ protected:
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
- AnalyticsManager::get_instance().init(store, analytic_store);
+ AnalyticsManager::get_instance().init(store, analytic_store, 5);
schema = R"({
"name": "collection1",