Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
add analytics rate limit config (#1879)
Krunal Gandhifea777f41e5b
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 726486fe..565a5359 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -146,6 +146,8 @@ private:bool isRateLimitEnabled = true;+ uint32_t analytics_minute_rate_limit;+AnalyticsManager() {}~AnalyticsManager();@@ -177,7 +179,7 @@ public:AnalyticsManager(AnalyticsManager const&) = delete;void operator=(AnalyticsManager const&) = delete;- void init(Store* store, Store* analytics_store);+ void init(Store* store, Store* analytics_store, uint32_t analytics_minute_rate_limit);void run(ReplicationState* raft_server);diff --git a/include/tsconfig.h b/include/tsconfig.hindex 089d7f8f..0e94aa6e 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -13,6 +13,7 @@ private:std::string log_dir;std::string analytics_dir;int32_t analytics_db_ttl = 2419200; //four weeks in secs+ uint32_t analytics_minute_rate_limit = 5;std::string api_key;// @deprecated@@ -156,6 +157,10 @@ public:this->analytics_db_ttl = analytics_db_ttl;}+ void set_analytics_minute_rate_limit(int32_t analytics_minute_rate_limit) {+ this->analytics_minute_rate_limit = analytics_minute_rate_limit;+ }+void set_api_key(const std::string & api_key) {this->api_key = api_key;}@@ -238,6 +243,9 @@ public:int32_t get_analytics_db_ttl() const {return this->analytics_db_ttl;}+ int32_t get_analytics_minute_rate_limit() const {+ return this->analytics_minute_rate_limit;+ }std::string get_api_key() const {return this->api_key;diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex 0a51c859..c985cd49 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -7,7 +7,6 @@#include "string_utils.h"#define EVENTS_RATE_LIMIT_SEC 60-#define EVENTS_RATE_LIMIT_COUNT 5Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk) {@@ -406,7 +405,7 @@ Option<bool> AnalyticsManager::add_event(const std::string& client_ip, const stdif (events_cache_it != events_cache.end()) {// event found in events cacheif ((now_ts_seconds - events_cache_it->second.last_update_time) < EVENTS_RATE_LIMIT_SEC) {- if (events_cache_it->second.count >= EVENTS_RATE_LIMIT_COUNT) {+ if (events_cache_it->second.count >= analytics_minute_rate_limit) {return Option<bool>(500, "event rate limit reached.");} else {events_cache_it->second.count++;@@ -733,9 +732,10 @@ void AnalyticsManager::dispose() {events_cache.clear();}-void AnalyticsManager::init(Store* store, Store* analytics_store) {+void AnalyticsManager::init(Store* store, Store* analytics_store, uint32_t analytics_minute_rate_limit) {this->store = store;this->analytics_store = analytics_store;+ this->analytics_minute_rate_limit = analytics_minute_rate_limit;if(analytics_store) {events_cache.capacity(1024);diff --git a/src/tsconfig.cpp b/src/tsconfig.cppindex e214396c..1714fc87 100644--- a/src/tsconfig.cpp+++ b/src/tsconfig.cpp@@ -268,6 +268,10 @@ void Config::load_config_env() {if(!get_env("TYPESENSE_ANALYTICS_DB_TTL").empty()) {this->analytics_db_ttl = std::stoi(get_env("TYPESENSE_ANALYTICS_DB_TTL"));}++ if(!get_env("TYPESENSE_ANALYTICS_MINUTE_RATE_LIMIT").empty()) {+ this->analytics_minute_rate_limit = std::stoi(get_env("TYPESENSE_ANALYTICS_MINUTE_RATE_LIMIT"));+ }}void Config::load_config_file(cmdline::parser& options) {@@ -303,7 +307,11 @@ void Config::load_config_file(cmdline::parser& options) {}if(reader.Exists("server", "analytics-db-ttl")) {- this->analytics_db_ttl = std::stoi(reader.Get("server", "analytics-db-ttl", "0"));+ this->analytics_db_ttl = reader.GetInteger("server", "analytics-db-ttl", 0);+ }++ if(reader.Exists("server", "analytics-minute-rate-limit")) {+ this->analytics_minute_rate_limit = reader.GetInteger("server", "analytics-minute-rate-limit", 5);}if(reader.Exists("server", "api-key")) {@@ -498,6 +506,10 @@ void Config::load_config_cmd_args(cmdline::parser& options) {this->analytics_db_ttl = options.get<uint32_t>("analytics-db-ttl");}+ if(options.exist("analytics-minute-rate-limit")) {+ this->analytics_minute_rate_limit = options.get<uint32_t>("analytics-minute-rate-limit");+ }+if(options.exist("api-key")) {this->api_key = options.get<std::string>("api-key");}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex ba13dca8..60a84956 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -67,6 +67,8 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<std::string>("health-rusage-api-key", '\0', "API key that allows access to health end-point with resource usage.", false);options.add<std::string>("analytics-dir", '\0', "Directory where Analytics will be stored.", false);options.add<uint32_t>("analytics-db-ttl", '\0', "TTL in seconds for events stored in analytics db", false);+ options.add<uint32_t>("analytics-minute-rate-limit", '\0', "per minute rate limit for /events endpoint", false);+options.add<std::string>("api-address", '\0', "Address to which Typesense API service binds.", false, "0.0.0.0");options.add<uint32_t>("api-port", '\0', "Port on which Typesense API service listens.", false, 8108);@@ -386,6 +388,7 @@ int run_server(const Config & config, const std::string & version, void (*masterstd::string meta_dir = config.get_data_dir() + "/meta";std::string analytics_dir = config.get_analytics_dir();int32_t analytics_db_ttl = config.get_analytics_db_ttl();+ uint32_t analytics_minute_rate_limit = config.get_analytics_minute_rate_limit();size_t thread_pool_size = config.get_thread_pool_size();@@ -414,7 +417,7 @@ int run_server(const Config & config, const std::string & version, void (*masterHttpClient & httpClient = HttpClient::get_instance();httpClient.init(config.get_api_key());- AnalyticsManager::get_instance().init(&store, &analytics_store);+ AnalyticsManager::get_instance().init(&store, &analytics_store, analytics_minute_rate_limit);server = new HttpServer(version,diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex 10c78c98..6895d29f 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -18,6 +18,7 @@ protected:std::vector<sort_by> sort_fields;AnalyticsManager& analyticsManager = AnalyticsManager::get_instance();+ uint32_t analytics_minute_rate_limit = 5;void setupCollection() {state_dir_path = "/tmp/typesense_test/analytics_manager_test";@@ -36,7 +37,7 @@ protected:collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);- analyticsManager.init(store, analytic_store);+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);analyticsManager.resetToggleRateLimit(false);}@@ -836,6 +837,53 @@ TEST_F(AnalyticsManagerTest, EventsRateLimitTest) {ASSERT_EQ("{\"message\": \"event rate limit reached.\"}", res->body);analyticsManager.resetToggleRateLimit(false);+++ //try with different limit+ //restart analytics manager as fresh+ analyticsManager.dispose();+ analyticsManager.stop();++ analytics_minute_rate_limit = 20;+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);++ analytics_rule = R"({+ "name": "product_events2",+ "type": "log",+ "params": {+ "source": {+ "collections": ["titles"],+ "events": [{"type": "click", "name": "AB"}]+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_TRUE(create_op.ok());++ event1 = R"({+ "type": "click",+ "name": "AB",+ "data": {+ "q": "technology",+ "doc_id": "21",+ "user_id": "13"+ }+ })"_json;++ //reset the LRU cache to test the rate limit+ analyticsManager.resetToggleRateLimit(true);++ for(auto i = 0; i < 20; ++i) {+ req->body = event1.dump();+ ASSERT_TRUE(post_create_event(req, res));+ }++ //as rate limit is 20, adding one more event above that should trigger rate limit+ ASSERT_FALSE(post_create_event(req, res));+ ASSERT_EQ("{\"message\": \"event rate limit reached.\"}", res->body);++ analyticsManager.resetToggleRateLimit(false);}TEST_F(AnalyticsManagerTest, NoresultsQueries) {@@ -1244,7 +1292,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {//restart analytics manager as freshanalyticsManager.dispose();analyticsManager.stop();- analyticsManager.init(store, analytic_store);+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);nlohmann::json products_schema = R"({"name": "books",@@ -1606,7 +1654,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {analyticsManager.dispose();analyticsManager.stop();- analyticsManager.init(store, analytic_store);+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);analytics_rule = R"({"name": "books_popularity3",@@ -1654,7 +1702,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreTTL) {system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);- analyticsManager.init(store, analytic_store);+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);auto analytics_rule = R"({"name": "product_events2",@@ -1733,7 +1781,7 @@ TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);- analyticsManager.init(store, analytic_store);+ analyticsManager.init(store, analytic_store, analytics_minute_rate_limit);auto analytics_rule = R"({"name": "product_events2",diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 7e2720bd..8d0cef5e 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -28,7 +28,7 @@ protected:collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);- AnalyticsManager::get_instance().init(store, analytic_store);+ AnalyticsManager::get_instance().init(store, analytic_store, 5);schema = R"({"name": "collection1",