Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
analytics db with ttl (#1789)
Krunal Gandhia1f8d7257a63
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 1d160fd8..45c02832 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -54,20 +54,7 @@ struct event_t {}}- void to_json(nlohmann::json& obj) const {- obj["query"] = query;- obj["type"] = event_type;- obj["timestamp"] = timestamp;- obj["user_id"] = user_id;- obj["doc_id"] = doc_id;- obj["name"] = name;-- if(event_type == "custom") {- for(const auto& kv : data) {- obj[kv.first] = kv.second;- }- }- }+ void to_json(nlohmann::json& obj, const std::string& coll) const;};struct counter_event_t {@@ -154,6 +141,7 @@ private:LRU::Cache<std::string, event_cache_t> events_cache;Store* store = nullptr;+ Store* analytics_store = nullptr;std::ofstream analytics_logs;bool isRateLimitEnabled = true;@@ -188,7 +176,7 @@ public:AnalyticsManager(AnalyticsManager const&) = delete;void operator=(AnalyticsManager const&) = delete;- void init(Store* store, const std::string& analytics_dir="");+ void init(Store* store, Store* analytics_store, const std::string& analytics_dir="");void run(ReplicationState* raft_server);@@ -217,7 +205,7 @@ public:Option<bool> add_event(const std::string& client_ip, const std::string& event_type,const std::string& event_name, const nlohmann::json& event_data);- void persist_events();+ void persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s);void persist_popular_events(ReplicationState *raft_server, uint64_t prev_persistence_s);@@ -229,4 +217,14 @@ public:std::unordered_map<std::string, QueryAnalytics*> get_nohits_queries();void resetToggleRateLimit(bool toggle);++ bool write_to_db(const nlohmann::json& payload);++ void get_last_N_events(const std::string& userid, uint32_t N, std::vector<std::string>& values);++#ifdef TEST_BUILD+ std::unordered_map<std::string, std::vector<event_t>> get_log_events() {+ return query_collection_events;+ }+#endif};diff --git a/include/core_api.h b/include/core_api.hindex 2c40e723..ace6c0dd 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -171,6 +171,8 @@ bool put_upsert_analytics_rules(const std::shared_ptr<http_req>& req, const std:bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool post_write_analytics_to_db(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+// Misc helpersvoid get_collections_for_auth(std::map<std::string, std::string>& req_params, const std::string& body,diff --git a/include/store.h b/include/store.hindex 68b7f635..c19d81ad 100644--- a/include/store.h+++ b/include/store.h@@ -21,6 +21,9 @@#include "string_utils.h"#include "logger.h"#include "file_utils.h"+#include <rocksdb/utilities/db_ttl.h>++#define FOURWEEKS_SECS 2419200class UInt64AddOperator : public rocksdb::AssociativeMergeOperator {public:@@ -60,23 +63,7 @@ private:// So we use unique lock only for assignment, but shared locks for all other operations on DBmutable std::shared_mutex mutex;- rocksdb::Status init_db() {- LOG(INFO) << "Initializing DB by opening state dir: " << state_dir_path;-- rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);- if(!s.ok()) {- LOG(ERROR) << "Error while initializing store: " << s.ToString();- if(s.code() == rocksdb::Status::Code::kIOError) {- LOG(ERROR) << "It seems like the data directory " << state_dir_path << " is already being used by "- << "another Typesense server. ";- LOG(ERROR) << "If you are SURE that this is not the case, delete the LOCK file "- << "in the data db directory and try again.";- }- }-- assert(s.ok());- return s;- }+ rocksdb::Status init_db(int32_t ttl);public:@@ -84,306 +71,56 @@ public:Store(const std::string & state_dir_path,const size_t wal_ttl_secs = 24*60*60,- const size_t wal_size_mb = 1024, bool disable_wal = true): state_dir_path(state_dir_path) {- // Optimize RocksDB- options.IncreaseParallelism();- options.OptimizeLevelStyleCompaction();- // create the DB if it's not already present- options.create_if_missing = true;- options.write_buffer_size = 4*1048576;- options.max_write_buffer_number = 2;- options.merge_operator.reset(new UInt64AddOperator);- options.compression = rocksdb::CompressionType::kSnappyCompression;-- options.max_log_file_size = 4*1048576;- options.keep_log_file_num = 5;-- /*options.table_properties_collector_factories.emplace_back(- rocksdb::NewCompactOnDeletionCollectorFactory(10000, 7500, 0.5));*/-- // these need to be high for replication scenarios- options.WAL_ttl_seconds = wal_ttl_secs;- options.WAL_size_limit_MB = wal_size_mb;-- // Disable WAL for master writes (Raft's WAL is used)- // The replica uses native WAL, though.- write_options.disableWAL = disable_wal;-- // open DB- init_db();- }-- ~Store() {- close();- }-- bool insert(const std::string& key, const std::string& value) {- std::shared_lock lock(mutex);- rocksdb::Status status = db->Put(write_options, key, value);- return status.ok();- }-- bool batch_write(rocksdb::WriteBatch& batch) {- std::shared_lock lock(mutex);- rocksdb::Status status = db->Write(write_options, &batch);- return status.ok();- }-- bool contains(const std::string& key) const {- std::shared_lock lock(mutex);-- std::string value;- bool value_found;- bool key_may_exist = db->KeyMayExist(rocksdb::ReadOptions(), key, &value, &value_found);-- // returns false when key definitely does not exist- if(!key_may_exist) {- return false;- }-- if(value_found) {- return true;- }-- // otherwise, we have try getting the value- rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);- return status.ok() && !status.IsNotFound();- }-- StoreStatus get(const std::string& key, std::string& value) const {- std::shared_lock lock(mutex);- rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);-- if(status.ok()) {- return StoreStatus::FOUND;- }-- if(status.IsNotFound()) {- return StoreStatus::NOT_FOUND;- }-- LOG(ERROR) << "Error while fetching the key: " << key << " - status is: " << status.ToString();- return StoreStatus::ERROR;- }-- bool remove(const std::string& key) {- std::shared_lock lock(mutex);- rocksdb::Status status = db->Delete(write_options, key);- return status.ok();- }-- rocksdb::Iterator* scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound) {- std::shared_lock lock(mutex);- rocksdb::ReadOptions read_opts;- if(iterate_upper_bound) {- read_opts.iterate_upper_bound = iterate_upper_bound;- }- rocksdb::Iterator *iter = db->NewIterator(read_opts);- iter->Seek(prefix);- return iter;- }-- rocksdb::Iterator* get_iterator() {- std::shared_lock lock(mutex);- rocksdb::Iterator* it = db->NewIterator(rocksdb::ReadOptions());- return it;- };-- void scan_fill(const std::string& prefix_start, const std::string& prefix_end, std::vector<std::string> & values) {- rocksdb::ReadOptions read_opts;- rocksdb::Slice upper_bound(prefix_end);- read_opts.iterate_upper_bound = &upper_bound;-- std::shared_lock lock(mutex);- rocksdb::Iterator *iter = db->NewIterator(read_opts);- for (iter->Seek(prefix_start); iter->Valid() && iter->key().starts_with(prefix_start); iter->Next()) {- values.push_back(iter->value().ToString());- }-- delete iter;- }-- void increment(const std::string & key, uint32_t value) {- std::shared_lock lock(mutex);- db->Merge(write_options, key, StringUtils::serialize_uint32_t(value));- }-- uint64_t get_latest_seq_number() const {- std::shared_lock lock(mutex);- return db->GetLatestSequenceNumber();- }-- Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number_org, const uint64_t max_updates) const {- std::shared_lock lock(mutex);- const uint64_t local_latest_seq_num = db->GetLatestSequenceNumber();-- // Since GetUpdatesSince(0) == GetUpdatesSince(1)- const uint64_t seq_number = (seq_number_org == 0) ? 1 : seq_number_org;-- if(seq_number == local_latest_seq_num+1) {- // replica has caught up, send an empty list as result- std::vector<std::string>* updates = new std::vector<std::string>();- return Option<std::vector<std::string>*>(updates);- }-- std::unique_ptr<rocksdb::TransactionLogIterator> iter;- rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);-- if(!status.ok()) {- LOG(ERROR) << "Error while fetching updates for replication: " << status.ToString();-- std::ostringstream error;- error << "Unable to fetch updates. " << "Master's latest sequence number is " << local_latest_seq_num- << " but requested sequence number is " << seq_number;- LOG(ERROR) << error.str();-- return Option<std::vector<std::string>*>(400, error.str());- }-- if(!iter->Valid()) {- std::ostringstream error;- error << "Invalid iterator. Master's latest sequence number is " << local_latest_seq_num << " but "- << "updates are requested from sequence number " << seq_number << ". "- << "The master's WAL entries might have expired (they are kept only for 24 hours).";- LOG(ERROR) << error.str();- return Option<std::vector<std::string>*>(400, error.str());- }-- uint64_t num_updates = 0;- std::vector<std::string>* updates = new std::vector<std::string>();-- bool first_iteration = true;-- while(iter->Valid() && num_updates < max_updates) {- const rocksdb::BatchResult & batch = iter->GetBatch();- if(first_iteration) {- first_iteration = false;- if(batch.sequence != seq_number) {- std::ostringstream error;- error << "Invalid iterator. Requested sequence number is " << seq_number << " but "- << "updates are available only from sequence number " << batch.sequence << ". "- << "The master's WAL entries might have expired (they are kept only for 24 hours).";- LOG(ERROR) << error.str();- return Option<std::vector<std::string>*>(400, error.str());- }- }-- const std::string & write_batch_serialized = batch.writeBatchPtr->Data();- updates->push_back(write_batch_serialized);- num_updates += 1;- iter->Next();- }-- return Option<std::vector<std::string>*>(updates);- }+ const size_t wal_size_mb = 1024,+ bool disable_wal = true,+ int32_t ttl=0);- void close() {- std::unique_lock lock(mutex);- delete db;- db = nullptr;- }+ ~Store();- int reload(bool clear_state_dir, const std::string& snapshot_path) {- std::unique_lock lock(mutex);+ bool insert(const std::string& key, const std::string& value);- // we don't use close() to avoid nested lock and because lock is required until db is re-initialized- delete db;- db = nullptr;+ bool batch_write(rocksdb::WriteBatch& batch);- if(clear_state_dir) {- if (!delete_path(state_dir_path, true)) {- LOG(WARNING) << "rm " << state_dir_path << " failed";- return -1;- }+ bool contains(const std::string& key) const;- LOG(INFO) << "rm " << state_dir_path << " success";- }+ StoreStatus get(const std::string& key, std::string& value) const;- if(!snapshot_path.empty()) {- // tries to use link if possible, or else copies- if (!copy_dir(snapshot_path, state_dir_path)) {- LOG(WARNING) << "copy snapshot " << snapshot_path << " to " << state_dir_path << " failed";- return -1;- }+ bool remove(const std::string& key);- LOG(INFO) << "copy snapshot " << snapshot_path << " to " << state_dir_path << " success";- }+ rocksdb::Iterator* scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound);- if (!create_directory(state_dir_path)) {- LOG(WARNING) << "CreateDirectory " << state_dir_path << " failed";- return -1;- }+ rocksdb::Iterator* get_iterator();- const rocksdb::Status& status = init_db();- if (!status.ok()) {- LOG(WARNING) << "Open DB " << state_dir_path << " failed, msg: " << status.ToString();- return -1;- }+ void scan_fill(const std::string& prefix_start, const std::string& prefix_end, std::vector<std::string> & values);- LOG(INFO) << "DB open success!";+ void increment(const std::string & key, uint32_t value);- return 0;- }+ uint64_t get_latest_seq_number() const;- void flush() {- std::shared_lock lock(mutex);- rocksdb::FlushOptions options;- db->Flush(options);- }+ Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number_org, const uint64_t max_updates) const;- rocksdb::Status compact_all() {- std::shared_lock lock(mutex);- return db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr);- }+ void close();- rocksdb::Status create_check_point(rocksdb::Checkpoint** checkpoint_ptr, const std::string& db_snapshot_path) {- std::shared_lock lock(mutex);- rocksdb::Status status = rocksdb::Checkpoint::Create(db, checkpoint_ptr);- if(!status.ok()) {- LOG(ERROR) << "Checkpoint Create failed, msg:" << status.ToString();- return status;- }+ int reload(bool clear_state_dir, const std::string& snapshot_path, int32_t ttl = 0);- status = (*checkpoint_ptr)->CreateCheckpoint(db_snapshot_path);+ void flush();- if(!status.ok()) {- LOG(WARNING) << "Checkpoint CreateCheckpoint failed at snapshot path: "- << db_snapshot_path << ", msg:" << status.ToString();- }+ rocksdb::Status compact_all();- return status;- }+ rocksdb::Status create_check_point(rocksdb::Checkpoint** checkpoint_ptr, const std::string& db_snapshot_path);- rocksdb::Status delete_range(const std::string& begin_key, const std::string& end_key) {- std::shared_lock lock(mutex);- return db->DeleteRange(rocksdb::WriteOptions(), db->DefaultColumnFamily(), begin_key, end_key);- }+ rocksdb::Status delete_range(const std::string& begin_key, const std::string& end_key);- rocksdb::Status compact_range(const rocksdb::Slice& begin_key, const rocksdb::Slice& end_key) {- std::shared_lock lock(mutex);- return db->CompactRange(rocksdb::CompactRangeOptions(), &begin_key, &end_key);- }+ rocksdb::Status compact_range(const rocksdb::Slice& begin_key, const rocksdb::Slice& end_key);// Only for internal tests- rocksdb::DB* _get_db_unsafe() const {- return db;- }+ rocksdb::DB* _get_db_unsafe() const;- const std::string& get_state_dir_path() const {- return state_dir_path;- }+ const std::string& get_state_dir_path() const;- const rocksdb::Options &get_db_options() const {- return options;- }+ const rocksdb::Options &get_db_options() const;- void print_memory_usage() {- std::string index_usage;- db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);- LOG(INFO) << "rocksdb.estimate-table-readers-mem: " << index_usage;+ void print_memory_usage();- std::string memtable_usage;- db->GetProperty("rocksdb.cur-size-all-mem-tables", &memtable_usage);- LOG(INFO) << "rocksdb.cur-size-all-mem-tables: " << memtable_usage;- }-};+ void get_last_N_values(const std::string& userid_prefix, uint32_t N, std::vector<std::string>& values);+};\ No newline at end of filediff --git a/include/tsconfig.h b/include/tsconfig.hindex 720232cb..089d7f8f 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -12,7 +12,7 @@ private:std::string data_dir;std::string log_dir;std::string analytics_dir;-+ int32_t analytics_db_ttl = 2419200; //four weeks in secsstd::string api_key;// @deprecated@@ -152,6 +152,10 @@ public:this->analytics_dir = analytics_dir;}+ void set_analytics_db_ttl(int32_t analytics_db_ttl) {+ this->analytics_db_ttl = analytics_db_ttl;+ }+void set_api_key(const std::string & api_key) {this->api_key = api_key;}@@ -231,6 +235,10 @@ public:return this->analytics_dir;}+ int32_t get_analytics_db_ttl() const {+ return this->analytics_db_ttl;+ }+std::string get_api_key() const {return this->api_key;}diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex 0194019e..c39d5609 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -522,7 +522,7 @@ void AnalyticsManager::run(ReplicationState* raft_server) {}persist_query_events(raft_server, prev_persistence_s);- persist_events();+ persist_events(raft_server, prev_persistence_s);persist_popular_events(raft_server, prev_persistence_s);prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(@@ -618,25 +618,57 @@ void AnalyticsManager::persist_query_events(ReplicationState *raft_server, uint6}}-void AnalyticsManager::persist_events() {+void AnalyticsManager::persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {// lock is held by caller++ auto send_http_response = [&](const std::string& import_payload) {+ // send http request+ if(raft_server == nullptr) {+ return;+ }++ std::string leader_url = raft_server->get_leader_url();+ if(!leader_url.empty()) {+ const std::string& base_url = leader_url + "analytics/";+ std::string res;++ const std::string& update_url = base_url + "/write_to_db";+ std::map<std::string, std::string> res_headers;+ long status_code = HttpClient::post_response(update_url, import_payload,+ res, res_headers, {}, 10*1000, true);++ if(status_code != 200) {+ LOG(ERROR) << "Error while sending "<<" log events to leader. "+ << "Status code: " << status_code << ", response: " << res;+ }+ }+ };++ nlohmann::json payload = nlohmann::json::array();for (auto &events_collection_it: query_collection_events) {const auto& collection = events_collection_it.first;for (const auto &event: events_collection_it.second) {if (analytics_logs.is_open() && event.log_to_file) {//store events to log file- analytics_logs << event.timestamp << "\t" << event.name << "\t"+ std::stringstream ssbuf;+ ssbuf << event.timestamp << "\t" << event.name << "\t"<< collection << "\t" << event.user_id << "\t" << event.doc_id << "\t"<< event.query << "\t";for(const auto& kv : event.data) {- analytics_logs << kv.second << "\t";+ ssbuf << kv.second << "\t";}+ analytics_logs << ssbuf.str();analytics_logs << "\n";analytics_logs << std::flush;++ nlohmann::json event_data;+ event.to_json(event_data, collection);+ payload.push_back(event_data);}}+ send_http_response(payload.dump());events_collection_it.second.clear();}}@@ -707,8 +739,9 @@ void AnalyticsManager::dispose() {events_cache.clear();}-void AnalyticsManager::init(Store* store, const std::string& analytics_dir) {+void AnalyticsManager::init(Store* store, Store* analytics_store, const std::string& analytics_dir) {this->store = store;+ this->analytics_store = analytics_store;if(!analytics_dir.empty()) {const auto analytics_log_path = analytics_dir + "/analytics_events.tsv";@@ -749,4 +782,43 @@ void counter_event_t::serialize_as_docs(std::string &docs) {if (!docs.empty()) {docs.pop_back();}+}++bool AnalyticsManager::write_to_db(const nlohmann::json& payload) {+ if(analytics_store) {+ for(const auto& event: payload) {+ std::string key = event["user_id"].get<std::string>() + "_" + StringUtils::serialize_uint64_t(event["timestamp"].get<uint64_t>());+ bool inserted = analytics_store->insert(key, event.dump());+ if(!inserted) {+ LOG(ERROR) << "Error while dumping events to analytics db.";+ return false;+ }+ }+ } else {+ LOG(ERROR) << "Analytics DB not initialized!!";+ return false;+ }++ return true;+}++void AnalyticsManager::get_last_N_events(const std::string& userid, uint32_t N, std::vector<std::string>& values) {+ const std::string userid_prefix = userid + "_";+ analytics_store->get_last_N_values(userid_prefix, N, values);+}++void event_t::to_json(nlohmann::json& obj, const std::string& coll) const {+ obj["query"] = query;+ obj["type"] = event_type;+ obj["timestamp"] = timestamp;+ obj["user_id"] = user_id;+ obj["doc_id"] = doc_id;+ obj["name"] = name;+ obj["collection"] = coll;++ if(event_type == "custom") {+ for(const auto& kv : data) {+ obj[kv.first] = kv.second;+ }+ }}\ No newline at end of filediff --git a/src/core_api.cpp b/src/core_api.cppindex 6afd3ea1..45776b4b 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -2844,6 +2844,22 @@ bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::sharedreturn true;}+bool post_write_analytics_to_db(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ nlohmann::json req_json;++ try {+ req_json = nlohmann::json::parse(req->body);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ res->set_400("Bad JSON.");+ return false;+ }++ AnalyticsManager::get_instance().write_to_db(req_json);++ return true;+}+bool post_proxy(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {HttpProxy& proxy = HttpProxy::get_instance();diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex fe13a54d..e95a01a2 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -80,6 +80,7 @@ void master_server_routes() {server->put("/analytics/rules/:name", put_upsert_analytics_rules);server->del("/analytics/rules/:name", del_analytics_rules);server->post("/analytics/events", post_create_event);+ server->post("analytics/aggregate_events", post_write_analytics_to_db);// metaserver->get("/metrics.json", get_metrics_json);diff --git a/src/raft_server.cpp b/src/raft_server.cppindex 6ce78567..096aaf16 100644--- a/src/raft_server.cpp+++ b/src/raft_server.cpp@@ -655,7 +655,7 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {if(analytics_store) {snapshot_path.append(std::string("/") + analytics_db_snapshot_name);- int reload_store = analytics_store->reload(true, snapshot_path);+ int reload_store = analytics_store->reload(true, snapshot_path, Config::get_instance().get_analytics_db_ttl());if (reload_store != 0) {LOG(ERROR) << "Failed to reload analytics db snapshot.";return reload_store;diff --git a/src/store.cpp b/src/store.cppnew file mode 100644index 00000000..902aa84c--- /dev/null+++ b/src/store.cpp@@ -0,0 +1,355 @@+#include "include/store.h"++Store::Store(const std::string & state_dir_path,+ const size_t wal_ttl_secs,+ const size_t wal_size_mb, bool disable_wal, int32_t ttl):+ state_dir_path(state_dir_path){+ // Optimize RocksDB+ options.IncreaseParallelism();+ options.OptimizeLevelStyleCompaction();+ // create the DB if it's not already present+ options.create_if_missing = true;+ options.write_buffer_size = 4*1048576;+ options.max_write_buffer_number = 2;+ options.merge_operator.reset(new UInt64AddOperator);+ options.compression = rocksdb::CompressionType::kSnappyCompression;++ options.max_log_file_size = 4*1048576;+ options.keep_log_file_num = 5;++ /*options.table_properties_collector_factories.emplace_back(+ rocksdb::NewCompactOnDeletionCollectorFactory(10000, 7500, 0.5));*/++ // these need to be high for replication scenarios+ options.WAL_ttl_seconds = wal_ttl_secs;+ options.WAL_size_limit_MB = wal_size_mb;++ // Disable WAL for master writes (Raft's WAL is used)+ // The replica uses native WAL, though.+ write_options.disableWAL = disable_wal;++ // open DB+ init_db(ttl);+}++Store::~Store() {+ close();+}++rocksdb::Status Store::init_db(int32_t ttl) {+ LOG(INFO) << "Initializing DB by opening state dir: " << state_dir_path;++ rocksdb::Status s;++ if(ttl > 0) {+ rocksdb::DBWithTTL* dbWithTtl;+ s = rocksdb::DBWithTTL::Open(options, state_dir_path,+ &dbWithTtl, ttl, false);+ db = dbWithTtl;+ } else {+ s = rocksdb::DB::Open(options, state_dir_path, &db);+ }++ if(!s.ok()) {+ LOG(ERROR) << "Error while initializing store: " << s.ToString();+ if(s.code() == rocksdb::Status::Code::kIOError) {+ LOG(ERROR) << "It seems like the data directory " << state_dir_path << " is already being used by "+ << "another Typesense server. ";+ LOG(ERROR) << "If you are SURE that this is not the case, delete the LOCK file "+ << "in the data db directory and try again.";+ }+ }++ assert(s.ok());+ return s;+}++bool Store::insert(const std::string& key, const std::string& value) {+ std::shared_lock lock(mutex);+ rocksdb::Status status = db->Put(write_options, key, value);+ return status.ok();+}++bool Store::batch_write(rocksdb::WriteBatch& batch) {+ std::shared_lock lock(mutex);+ rocksdb::Status status = db->Write(write_options, &batch);+ return status.ok();+}++bool Store::contains(const std::string& key) const {+ std::shared_lock lock(mutex);++ std::string value;+ bool value_found;+ bool key_may_exist = db->KeyMayExist(rocksdb::ReadOptions(), key, &value, &value_found);++ // returns false when key definitely does not exist+ if(!key_may_exist) {+ return false;+ }++ if(value_found) {+ return true;+ }++ // otherwise, we have try getting the value+ rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);+ return status.ok() && !status.IsNotFound();+}++StoreStatus Store::get(const std::string& key, std::string& value) const {+ std::shared_lock lock(mutex);+ rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);++ if(status.ok()) {+ return StoreStatus::FOUND;+ }++ if(status.IsNotFound()) {+ return StoreStatus::NOT_FOUND;+ }++ LOG(ERROR) << "Error while fetching the key: " << key << " - status is: " << status.ToString();+ return StoreStatus::ERROR;+}++bool Store::remove(const std::string& key) {+ std::shared_lock lock(mutex);+ rocksdb::Status status = db->Delete(write_options, key);+ return status.ok();+}++rocksdb::Iterator* Store::scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound) {+ std::shared_lock lock(mutex);+ rocksdb::ReadOptions read_opts;+ if(iterate_upper_bound) {+ read_opts.iterate_upper_bound = iterate_upper_bound;+ }+ rocksdb::Iterator *iter = db->NewIterator(read_opts);+ iter->Seek(prefix);+ return iter;+}++rocksdb::Iterator* Store::get_iterator() {+ std::shared_lock lock(mutex);+ rocksdb::Iterator* it = db->NewIterator(rocksdb::ReadOptions());+ return it;+}++void Store::scan_fill(const std::string& prefix_start, const std::string& prefix_end, std::vector<std::string> & values) {+ rocksdb::ReadOptions read_opts;+ rocksdb::Slice upper_bound(prefix_end);+ read_opts.iterate_upper_bound = &upper_bound;++ std::shared_lock lock(mutex);+ rocksdb::Iterator *iter = db->NewIterator(read_opts);+ for (iter->Seek(prefix_start); iter->Valid() && iter->key().starts_with(prefix_start); iter->Next()) {+ values.push_back(iter->value().ToString());+ }++ delete iter;+}++void Store::increment(const std::string & key, uint32_t value) {+ std::shared_lock lock(mutex);+ db->Merge(write_options, key, StringUtils::serialize_uint32_t(value));+}++uint64_t Store::get_latest_seq_number() const {+ std::shared_lock lock(mutex);+ return db->GetLatestSequenceNumber();+}++Option<std::vector<std::string>*> Store::get_updates_since(const uint64_t seq_number_org, const uint64_t max_updates) const {+ std::shared_lock lock(mutex);+ const uint64_t local_latest_seq_num = db->GetLatestSequenceNumber();++ // Since GetUpdatesSince(0) == GetUpdatesSince(1)+ const uint64_t seq_number = (seq_number_org == 0) ? 1 : seq_number_org;++ if(seq_number == local_latest_seq_num+1) {+ // replica has caught up, send an empty list as result+ std::vector<std::string>* updates = new std::vector<std::string>();+ return Option<std::vector<std::string>*>(updates);+ }++ std::unique_ptr<rocksdb::TransactionLogIterator> iter;+ rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);++ if(!status.ok()) {+ LOG(ERROR) << "Error while fetching updates for replication: " << status.ToString();++ std::ostringstream error;+ error << "Unable to fetch updates. " << "Master's latest sequence number is " << local_latest_seq_num+ << " but requested sequence number is " << seq_number;+ LOG(ERROR) << error.str();++ return Option<std::vector<std::string>*>(400, error.str());+ }++ if(!iter->Valid()) {+ std::ostringstream error;+ error << "Invalid iterator. Master's latest sequence number is " << local_latest_seq_num << " but "+ << "updates are requested from sequence number " << seq_number << ". "+ << "The master's WAL entries might have expired (they are kept only for 24 hours).";+ LOG(ERROR) << error.str();+ return Option<std::vector<std::string>*>(400, error.str());+ }++ uint64_t num_updates = 0;+ std::vector<std::string>* updates = new std::vector<std::string>();++ bool first_iteration = true;++ while(iter->Valid() && num_updates < max_updates) {+ const rocksdb::BatchResult & batch = iter->GetBatch();+ if(first_iteration) {+ first_iteration = false;+ if(batch.sequence != seq_number) {+ std::ostringstream error;+ error << "Invalid iterator. Requested sequence number is " << seq_number << " but "+ << "updates are available only from sequence number " << batch.sequence << ". "+ << "The master's WAL entries might have expired (they are kept only for 24 hours).";+ LOG(ERROR) << error.str();+ return Option<std::vector<std::string>*>(400, error.str());+ }+ }++ const std::string & write_batch_serialized = batch.writeBatchPtr->Data();+ updates->push_back(write_batch_serialized);+ num_updates += 1;+ iter->Next();+ }++ return Option<std::vector<std::string>*>(updates);+}++void Store::close() {+ std::unique_lock lock(mutex);+ delete db;+ db = nullptr;+}++int Store::reload(bool clear_state_dir, const std::string& snapshot_path, int32_t ttl) {+ std::unique_lock lock(mutex);++ // we don't use close() to avoid nested lock and because lock is required until db is re-initialized+ delete db;+ db = nullptr;++ if(clear_state_dir) {+ if (!delete_path(state_dir_path, true)) {+ LOG(WARNING) << "rm " << state_dir_path << " failed";+ return -1;+ }++ LOG(INFO) << "rm " << state_dir_path << " success";+ }++ if(!snapshot_path.empty()) {+ // tries to use link if possible, or else copies+ if (!copy_dir(snapshot_path, state_dir_path)) {+ LOG(WARNING) << "copy snapshot " << snapshot_path << " to " << state_dir_path << " failed";+ return -1;+ }++ LOG(INFO) << "copy snapshot " << snapshot_path << " to " << state_dir_path << " success";+ }++ if (!create_directory(state_dir_path)) {+ LOG(WARNING) << "CreateDirectory " << state_dir_path << " failed";+ return -1;+ }++ const rocksdb::Status& status = init_db(ttl);+ if (!status.ok()) {+ LOG(WARNING) << "Open DB " << state_dir_path << " failed, msg: " << status.ToString();+ return -1;+ }++ LOG(INFO) << "DB open success!";++ return 0;+}++void Store::flush() {+ std::shared_lock lock(mutex);+ rocksdb::FlushOptions options;+ db->Flush(options);+}++rocksdb::Status Store::compact_all() {+ std::shared_lock lock(mutex);+ return db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr);+}++rocksdb::Status Store::create_check_point(rocksdb::Checkpoint** checkpoint_ptr, const std::string& db_snapshot_path) {+ std::shared_lock lock(mutex);+ rocksdb::Status status = rocksdb::Checkpoint::Create(db, checkpoint_ptr);+ if(!status.ok()) {+ LOG(ERROR) << "Checkpoint Create failed, msg:" << status.ToString();+ return status;+ }++ status = (*checkpoint_ptr)->CreateCheckpoint(db_snapshot_path);++ if(!status.ok()) {+ LOG(WARNING) << "Checkpoint CreateCheckpoint failed at snapshot path: "+ << db_snapshot_path << ", msg:" << status.ToString();+ }++ return status;+}++rocksdb::Status Store::delete_range(const std::string& begin_key, const std::string& end_key) {+ std::shared_lock lock(mutex);+ return db->DeleteRange(rocksdb::WriteOptions(), db->DefaultColumnFamily(), begin_key, end_key);+}++rocksdb::Status Store::compact_range(const rocksdb::Slice& begin_key, const rocksdb::Slice& end_key) {+ std::shared_lock lock(mutex);+ return db->CompactRange(rocksdb::CompactRangeOptions(), &begin_key, &end_key);+}++rocksdb::DB* Store::_get_db_unsafe() const {+ return db;+}++const std::string& Store::get_state_dir_path() const {+ return state_dir_path;+}++const rocksdb::Options& Store::get_db_options() const {+ return options;+}++void Store::print_memory_usage() {+ std::string index_usage;+ db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);+ LOG(INFO) << "rocksdb.estimate-table-readers-mem: " << index_usage;++ std::string memtable_usage;+ db->GetProperty("rocksdb.cur-size-all-mem-tables", &memtable_usage);+ LOG(INFO) << "rocksdb.cur-size-all-mem-tables: " << memtable_usage;+}++void Store::get_last_N_values(const std::string& userid_prefix, uint32_t N, std::vector<std::string>& values) {+ std::shared_lock lock(mutex);++ rocksdb::Iterator* iter = db->NewIterator(rocksdb::ReadOptions());+ auto prefix_key = userid_prefix + "~";+ iter->SeekForPrev(prefix_key);++ while(iter->Valid() && N) {+ auto key = iter->key().ToString();+ if(!StringUtils::begins_with(key, userid_prefix)) {+ break;+ }++ values.push_back(iter->value().ToString());+ N--;+ iter->Prev();+ }++ delete iter;+}\ No newline at end of filediff --git a/src/tsconfig.cpp b/src/tsconfig.cppindex 641e260b..e214396c 100644--- a/src/tsconfig.cpp+++ b/src/tsconfig.cpp@@ -129,7 +129,6 @@ Option<std::string> Config::fetch_nodes_config(const std::string& path_to_nodes)void Config::load_config_env() {this->data_dir = get_env("TYPESENSE_DATA_DIR");this->log_dir = get_env("TYPESENSE_LOG_DIR");- this->analytics_dir = get_env("TYPESENSE_ANALYTICS_DIR");this->api_key = get_env("TYPESENSE_API_KEY");// @deprecated@@ -261,6 +260,14 @@ void Config::load_config_env() {if(!get_env("TYPESENSE_MAX_PER_PAGE").empty()) {this->max_per_page = std::stoi(get_env("TYPESENSE_MAX_PER_PAGE"));}++ if(!get_env("TYPESENSE_ANALYTICS_DIR").empty()) {+ this->analytics_dir = get_env("TYPESENSE_ANALYTICS_DIR");+ }++ if(!get_env("TYPESENSE_ANALYTICS_DB_TTL").empty()) {+ this->analytics_db_ttl = std::stoi(get_env("TYPESENSE_ANALYTICS_DB_TTL"));+ }}void Config::load_config_file(cmdline::parser& options) {@@ -295,6 +302,10 @@ void Config::load_config_file(cmdline::parser& options) {this->analytics_dir = reader.Get("server", "analytics-dir", "");}+ if(reader.Exists("server", "analytics-db-ttl")) {+ this->analytics_db_ttl = std::stoi(reader.Get("server", "analytics-db-ttl", "0"));+ }+if(reader.Exists("server", "api-key")) {this->api_key = reader.Get("server", "api-key", "");}@@ -483,6 +494,10 @@ void Config::load_config_cmd_args(cmdline::parser& options) {this->analytics_dir = options.get<std::string>("analytics-dir");}+ if(options.exist("analytics-db-ttl")) {+ this->analytics_db_ttl = options.get<uint32_t>("analytics-db-ttl");+ }+if(options.exist("api-key")) {this->api_key = options.get<std::string>("api-key");}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 2f66963c..94f0395e 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -68,6 +68,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<std::string>("search-only-api-key", 's', "[DEPRECATED: use API key management end-point] API key that allows only searches.", false);options.add<std::string>("health-rusage-api-key", '\0', "API key that allows access to health end-point with resource usage.", false);options.add<std::string>("analytics-dir", '\0', "Directory where Analytics will be stored.", false);+ options.add<uint32_t>("analytics-db-ttl", '\0', "TTL in seconds for events stored in analytics db", false);options.add<std::string>("api-address", '\0', "Address to which Typesense API service binds.", false, "0.0.0.0");options.add<uint32_t>("api-port", '\0', "Port on which Typesense API service listens.", false, 8108);@@ -385,6 +386,7 @@ int run_server(const Config & config, const std::string & version, void (*masterstd::string state_dir = config.get_data_dir() + "/state";std::string meta_dir = config.get_data_dir() + "/meta";std::string analytics_dir = config.get_analytics_dir();+ int32_t analytics_db_ttl = config.get_analytics_db_ttl();size_t thread_pool_size = config.get_thread_pool_size();@@ -406,14 +408,14 @@ int run_server(const Config & config, const std::string & version, void (*master// meta DB for storing house keeping thingsStore meta_store(meta_dir, 24*60*60, 1024, false);- //analytics DB for storing query click events- std::unique_ptr<Store> analytics_store = nullptr;+ //analytics DB for storing analytics events+ Store analytics_store(analytics_dir, 24*60*60, 1024, true, analytics_db_ttl);curl_global_init(CURL_GLOBAL_SSL);HttpClient & httpClient = HttpClient::get_instance();httpClient.init(config.get_api_key());- AnalyticsManager::get_instance().init(&store, analytics_dir);+ AnalyticsManager::get_instance().init(&store, &analytics_store, analytics_dir);server = new HttpServer(version,@@ -464,7 +466,7 @@ int run_server(const Config & config, const std::string & version, void (*master// first we start the peering service- ReplicationState replication_state(server, batch_indexer, &store, analytics_store.get(),+ ReplicationState replication_state(server, batch_indexer, &store, &analytics_store,&replication_thread_pool, server->get_message_dispatcher(),ssl_enabled,&config,diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex dfcf8f27..9324ab52 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -9,8 +9,10 @@class AnalyticsManagerTest : public ::testing::Test {protected:Store *store;+ Store *analytic_store;CollectionManager& collectionManager = CollectionManager::get_instance();std::atomic<bool> quit = false;+ std::string state_dir_path, analytics_dir_path;std::vector<std::string> query_fields;std::vector<sort_by> sort_fields;@@ -18,16 +20,23 @@ protected:AnalyticsManager& analyticsManager = AnalyticsManager::get_instance();void setupCollection() {- std::string state_dir_path = "/tmp/typesense_test/analytics_manager_test";+ state_dir_path = "/tmp/typesense_test/analytics_manager_test";+ analytics_dir_path = "/tmp/typesense-test/analytics";+LOG(INFO) << "Truncating and creating: " << state_dir_path;system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());system("mkdir -p /tmp/typesense_test/models");store = new Store(state_dir_path);++ LOG(INFO) << "Truncating and creating: " << analytics_dir_path;+ system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());+ analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);+collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);- analyticsManager.init(store, state_dir_path);+ analyticsManager.init(store, analytic_store, state_dir_path);analyticsManager.resetToggleRateLimit(false);}@@ -38,6 +47,7 @@ protected:virtual void TearDown() {collectionManager.dispose();delete store;+ delete analytic_store;analyticsManager.stop();}};@@ -514,7 +524,7 @@ TEST_F(AnalyticsManagerTest, EventsPersist) {req->body = event.dump();ASSERT_TRUE(post_create_event(req, res));- analyticsManager.persist_events();+ analyticsManager.persist_events(nullptr, 0);auto fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");@@ -540,7 +550,7 @@ TEST_F(AnalyticsManagerTest, EventsPersist) {req->body = event.dump();ASSERT_TRUE(post_create_event(req, res));- analyticsManager.persist_events();+ analyticsManager.persist_events(nullptr, 0);fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");@@ -1007,7 +1017,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {analyticsManager.dispose();analyticsManager.stop();system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");- analyticsManager.init(store, "/tmp/typesense_test/analytics_manager_test");+ analyticsManager.init(store, analytic_store, "/tmp/typesense_test/analytics_manager_test");nlohmann::json products_schema = R"({"name": "books",@@ -1235,7 +1245,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {ASSERT_EQ("Cool trousers", results["hits"][1]["document"]["title"]);//verify log file- analyticsManager.persist_events();+ analyticsManager.persist_events(nullptr, 0);auto fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");@@ -1303,7 +1313,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {ASSERT_EQ(10, popular_clicks["books"].docid_counts["1"]);//check log file- analyticsManager.persist_events();+ analyticsManager.persist_events(nullptr, 0);fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");@@ -1328,7 +1338,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {analyticsManager.dispose();analyticsManager.stop();system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");- analyticsManager.init(store, "");+ analyticsManager.init(store, analytic_store, "");analytics_rule = R"({"name": "books_popularity3",@@ -1351,11 +1361,10 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {//counter events should work without analytic-dir-analyticsManager.dispose();analyticsManager.stop();system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");- analyticsManager.init(store, "");+ analyticsManager.init(store, analytic_store, "");analytics_rule = R"({"name": "books_popularity3",@@ -1392,4 +1401,197 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {ASSERT_EQ("popularity", popular_clicks["books"].counter_field);ASSERT_EQ(1, popular_clicks["books"].docid_counts.size());ASSERT_EQ(5, popular_clicks["books"].docid_counts["1"]);+}++TEST_F(AnalyticsManagerTest, AnalyticsStoreTTL) {+ analyticsManager.dispose();+ analyticsManager.stop();+ delete analytic_store;++ //set TTL of an hour+ LOG(INFO) << "Truncating and creating: " << analytics_dir_path;+ system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());++ analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);+ analyticsManager.init(store, analytic_store, "");++ auto analytics_rule = R"({+ "name": "product_events2",+ "type": "log",+ "params": {+ "name": "product_events_logging2",+ "source": {+ "collections": ["titles"],+ "events": [{"type": "click", "name": "AB"}]+ }+ }+ })"_json;++ auto create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_TRUE(create_op.ok());++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ nlohmann::json event1 = R"({+ "type": "click",+ "name": "AB",+ "data": {+ "q": "technology",+ "doc_id": "21",+ "user_id": "13"+ }+ })"_json;++ req->body = event1.dump();+ ASSERT_TRUE(post_create_event(req, res));++ //get events+ nlohmann::json payload = nlohmann::json::array();+ nlohmann::json event_data;+ auto collection_events_map = analyticsManager.get_log_events();+ for (auto &events_collection_it: collection_events_map) {+ const auto& collection = events_collection_it.first;+ for(const auto& event: events_collection_it.second) {+ event.to_json(event_data, collection);+ }+ }+ payload.push_back(event_data);++ //manually trigger write to db+ ASSERT_TRUE(analyticsManager.write_to_db(payload));++ //try fetching from db+ const std::string prefix_start = "13_";+ const std::string prefix_end = "13`";+ std::vector<std::string> events;++ analytic_store->scan_fill(prefix_start, prefix_end, events);+ ASSERT_EQ(1, events.size());+ ASSERT_EQ(events[0].c_str(), event_data.dump());++ //now set TTL to 1s and open analytics db+ events.clear();+ delete analytic_store;++ analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, 1);++ sleep(2);+ analytic_store->compact_all();++ analytic_store->scan_fill(prefix_start, prefix_end, events);+ ASSERT_EQ(0, events.size());+}++TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {+ analyticsManager.dispose();+ analyticsManager.stop();+ delete analytic_store;++ //set TTL of an hour+ LOG(INFO) << "Truncating and creating: " << analytics_dir_path;+ system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());++ analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);+ analyticsManager.init(store, analytic_store, "");++ auto analytics_rule = R"({+ "name": "product_events2",+ "type": "log",+ "params": {+ "name": "product_events_logging2",+ "source": {+ "collections": ["titles"],+ "events": [{"type": "click", "name": "AB"}]+ }+ }+ })"_json;++ auto create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_TRUE(create_op.ok());++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ nlohmann::json event1;+ event1["type"] = "click";+ event1["name"] = "AB";+ event1["data"]["q"] = "technology";+ event1["data"]["user_id"] = "13";++ for(auto i = 0; i < 10; i++) {+ event1["data"]["doc_id"] = std::to_string(i);+ req->body = event1.dump();+ ASSERT_TRUE(post_create_event(req, res));+ }++ //add more user events+ for(auto i = 0; i < 7; i++) {+ event1["data"]["user_id"] = "14";+ event1["data"]["doc_id"] = std::to_string(i);+ req->body = event1.dump();+ ASSERT_TRUE(post_create_event(req, res));+ }++ for(auto i = 0; i < 5; i++) {+ event1["data"]["user_id"] = "15";+ event1["data"]["doc_id"] = std::to_string(i);+ req->body = event1.dump();+ ASSERT_TRUE(post_create_event(req, res));+ }++ //get events+ nlohmann::json payload = nlohmann::json::array();+ nlohmann::json event_data;+ auto collection_events_map = analyticsManager.get_log_events();+ for (auto &events_collection_it: collection_events_map) {+ const auto& collection = events_collection_it.first;+ for(const auto& event: events_collection_it.second) {+ event.to_json(event_data, collection);+ payload.push_back(event_data);+ }+ }++ //manually trigger write to db+ ASSERT_TRUE(analyticsManager.write_to_db(payload));++ //basic test+ std::vector<std::string> values;+ analyticsManager.get_last_N_events("13", 5, values);+ ASSERT_EQ(5, values.size());++ nlohmann::json parsed_json;+ uint32_t start_index = 9;+ for(auto i = 0; i < 5; i++) {+ parsed_json = nlohmann::json::parse(values[i]);+ ASSERT_EQ(std::to_string(start_index - i), parsed_json["doc_id"]);+ }++ //fetch events for middle user+ values.clear();+ analyticsManager.get_last_N_events("14", 5, values);+ ASSERT_EQ(5, values.size());++ start_index = 6;+ for(auto i = 0; i < 5; i++) {+ parsed_json = nlohmann::json::parse(values[i]);+ ASSERT_EQ(std::to_string(start_index - i), parsed_json["doc_id"]);+ }++ //fetch more events than stored in db+ values.clear();+ analyticsManager.get_last_N_events("15", 8, values);+ ASSERT_EQ(5, values.size());++ start_index = 4;+ for(auto i = 0; i < 5; i++) {+ parsed_json = nlohmann::json::parse(values[i]);+ ASSERT_EQ(std::to_string(start_index - i), parsed_json["doc_id"]);+ }+++ //fetch events for non-existing user+ values.clear();+ analyticsManager.get_last_N_events("16", 8, values);+ ASSERT_EQ(0, values.size());}\ No newline at end of filediff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 86b13819..ad985251 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -10,6 +10,7 @@class CollectionManagerTest : public ::testing::Test {protected:Store *store;+ Store* analytic_store;CollectionManager & collectionManager = CollectionManager::get_instance();std::atomic<bool> quit = false;Collection *collection1;@@ -22,10 +23,12 @@ protected:system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);+ analytic_store = new Store(state_dir_path + "/analytics");+collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);- AnalyticsManager::get_instance().init(store);+ AnalyticsManager::get_instance().init(store, analytic_store);schema = R"({"name": "collection1",@@ -62,6 +65,7 @@ protected:collectionManager.drop_collection("collection1");collectionManager.dispose();delete store;+ delete analytic_store;}}};