Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

analytics db with ttl (#1789)

Krunal Gandhia1f8d7257a63
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index 1d160fd8..45c02832 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -54,20 +54,7 @@ struct event_t {
}
}
- void to_json(nlohmann::json& obj) const {
- obj["query"] = query;
- obj["type"] = event_type;
- obj["timestamp"] = timestamp;
- obj["user_id"] = user_id;
- obj["doc_id"] = doc_id;
- obj["name"] = name;
-
- if(event_type == "custom") {
- for(const auto& kv : data) {
- obj[kv.first] = kv.second;
- }
- }
- }
+ void to_json(nlohmann::json& obj, const std::string& coll) const;
};
struct counter_event_t {
@@ -154,6 +141,7 @@ private:
LRU::Cache<std::string, event_cache_t> events_cache;
Store* store = nullptr;
+ Store* analytics_store = nullptr;
std::ofstream analytics_logs;
bool isRateLimitEnabled = true;
@@ -188,7 +176,7 @@ public:
AnalyticsManager(AnalyticsManager const&) = delete;
void operator=(AnalyticsManager const&) = delete;
- void init(Store* store, const std::string& analytics_dir="");
+ void init(Store* store, Store* analytics_store, const std::string& analytics_dir="");
void run(ReplicationState* raft_server);
@@ -217,7 +205,7 @@ public:
Option<bool> add_event(const std::string& client_ip, const std::string& event_type,
const std::string& event_name, const nlohmann::json& event_data);
- void persist_events();
+ void persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s);
void persist_popular_events(ReplicationState *raft_server, uint64_t prev_persistence_s);
@@ -229,4 +217,14 @@ public:
std::unordered_map<std::string, QueryAnalytics*> get_nohits_queries();
void resetToggleRateLimit(bool toggle);
+
+ bool write_to_db(const nlohmann::json& payload);
+
+ void get_last_N_events(const std::string& userid, uint32_t N, std::vector<std::string>& values);
+
+#ifdef TEST_BUILD
+ std::unordered_map<std::string, std::vector<event_t>> get_log_events() {
+ return query_collection_events;
+ }
+#endif
};
diff --git a/include/core_api.h b/include/core_api.h
index 2c40e723..ace6c0dd 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -171,6 +171,8 @@ bool put_upsert_analytics_rules(const std::shared_ptr<http_req>& req, const std:
bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+bool post_write_analytics_to_db(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+
// Misc helpers
void get_collections_for_auth(std::map<std::string, std::string>& req_params, const std::string& body,
diff --git a/include/store.h b/include/store.h
index 68b7f635..c19d81ad 100644
--- a/include/store.h
+++ b/include/store.h
@@ -21,6 +21,9 @@
#include "string_utils.h"
#include "logger.h"
#include "file_utils.h"
+#include <rocksdb/utilities/db_ttl.h>
+
+#define FOURWEEKS_SECS 2419200
class UInt64AddOperator : public rocksdb::AssociativeMergeOperator {
public:
@@ -60,23 +63,7 @@ private:
// So we use unique lock only for assignment, but shared locks for all other operations on DB
mutable std::shared_mutex mutex;
- rocksdb::Status init_db() {
- LOG(INFO) << "Initializing DB by opening state dir: " << state_dir_path;
-
- rocksdb::Status s = rocksdb::DB::Open(options, state_dir_path, &db);
- if(!s.ok()) {
- LOG(ERROR) << "Error while initializing store: " << s.ToString();
- if(s.code() == rocksdb::Status::Code::kIOError) {
- LOG(ERROR) << "It seems like the data directory " << state_dir_path << " is already being used by "
- << "another Typesense server. ";
- LOG(ERROR) << "If you are SURE that this is not the case, delete the LOCK file "
- << "in the data db directory and try again.";
- }
- }
-
- assert(s.ok());
- return s;
- }
+ rocksdb::Status init_db(int32_t ttl);
public:
@@ -84,306 +71,56 @@ public:
Store(const std::string & state_dir_path,
const size_t wal_ttl_secs = 24*60*60,
- const size_t wal_size_mb = 1024, bool disable_wal = true): state_dir_path(state_dir_path) {
- // Optimize RocksDB
- options.IncreaseParallelism();
- options.OptimizeLevelStyleCompaction();
- // create the DB if it's not already present
- options.create_if_missing = true;
- options.write_buffer_size = 4*1048576;
- options.max_write_buffer_number = 2;
- options.merge_operator.reset(new UInt64AddOperator);
- options.compression = rocksdb::CompressionType::kSnappyCompression;
-
- options.max_log_file_size = 4*1048576;
- options.keep_log_file_num = 5;
-
- /*options.table_properties_collector_factories.emplace_back(
- rocksdb::NewCompactOnDeletionCollectorFactory(10000, 7500, 0.5));*/
-
- // these need to be high for replication scenarios
- options.WAL_ttl_seconds = wal_ttl_secs;
- options.WAL_size_limit_MB = wal_size_mb;
-
- // Disable WAL for master writes (Raft's WAL is used)
- // The replica uses native WAL, though.
- write_options.disableWAL = disable_wal;
-
- // open DB
- init_db();
- }
-
- ~Store() {
- close();
- }
-
- bool insert(const std::string& key, const std::string& value) {
- std::shared_lock lock(mutex);
- rocksdb::Status status = db->Put(write_options, key, value);
- return status.ok();
- }
-
- bool batch_write(rocksdb::WriteBatch& batch) {
- std::shared_lock lock(mutex);
- rocksdb::Status status = db->Write(write_options, &batch);
- return status.ok();
- }
-
- bool contains(const std::string& key) const {
- std::shared_lock lock(mutex);
-
- std::string value;
- bool value_found;
- bool key_may_exist = db->KeyMayExist(rocksdb::ReadOptions(), key, &value, &value_found);
-
- // returns false when key definitely does not exist
- if(!key_may_exist) {
- return false;
- }
-
- if(value_found) {
- return true;
- }
-
- // otherwise, we have try getting the value
- rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);
- return status.ok() && !status.IsNotFound();
- }
-
- StoreStatus get(const std::string& key, std::string& value) const {
- std::shared_lock lock(mutex);
- rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);
-
- if(status.ok()) {
- return StoreStatus::FOUND;
- }
-
- if(status.IsNotFound()) {
- return StoreStatus::NOT_FOUND;
- }
-
- LOG(ERROR) << "Error while fetching the key: " << key << " - status is: " << status.ToString();
- return StoreStatus::ERROR;
- }
-
- bool remove(const std::string& key) {
- std::shared_lock lock(mutex);
- rocksdb::Status status = db->Delete(write_options, key);
- return status.ok();
- }
-
- rocksdb::Iterator* scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound) {
- std::shared_lock lock(mutex);
- rocksdb::ReadOptions read_opts;
- if(iterate_upper_bound) {
- read_opts.iterate_upper_bound = iterate_upper_bound;
- }
- rocksdb::Iterator *iter = db->NewIterator(read_opts);
- iter->Seek(prefix);
- return iter;
- }
-
- rocksdb::Iterator* get_iterator() {
- std::shared_lock lock(mutex);
- rocksdb::Iterator* it = db->NewIterator(rocksdb::ReadOptions());
- return it;
- };
-
- void scan_fill(const std::string& prefix_start, const std::string& prefix_end, std::vector<std::string> & values) {
- rocksdb::ReadOptions read_opts;
- rocksdb::Slice upper_bound(prefix_end);
- read_opts.iterate_upper_bound = &upper_bound;
-
- std::shared_lock lock(mutex);
- rocksdb::Iterator *iter = db->NewIterator(read_opts);
- for (iter->Seek(prefix_start); iter->Valid() && iter->key().starts_with(prefix_start); iter->Next()) {
- values.push_back(iter->value().ToString());
- }
-
- delete iter;
- }
-
- void increment(const std::string & key, uint32_t value) {
- std::shared_lock lock(mutex);
- db->Merge(write_options, key, StringUtils::serialize_uint32_t(value));
- }
-
- uint64_t get_latest_seq_number() const {
- std::shared_lock lock(mutex);
- return db->GetLatestSequenceNumber();
- }
-
- Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number_org, const uint64_t max_updates) const {
- std::shared_lock lock(mutex);
- const uint64_t local_latest_seq_num = db->GetLatestSequenceNumber();
-
- // Since GetUpdatesSince(0) == GetUpdatesSince(1)
- const uint64_t seq_number = (seq_number_org == 0) ? 1 : seq_number_org;
-
- if(seq_number == local_latest_seq_num+1) {
- // replica has caught up, send an empty list as result
- std::vector<std::string>* updates = new std::vector<std::string>();
- return Option<std::vector<std::string>*>(updates);
- }
-
- std::unique_ptr<rocksdb::TransactionLogIterator> iter;
- rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);
-
- if(!status.ok()) {
- LOG(ERROR) << "Error while fetching updates for replication: " << status.ToString();
-
- std::ostringstream error;
- error << "Unable to fetch updates. " << "Master's latest sequence number is " << local_latest_seq_num
- << " but requested sequence number is " << seq_number;
- LOG(ERROR) << error.str();
-
- return Option<std::vector<std::string>*>(400, error.str());
- }
-
- if(!iter->Valid()) {
- std::ostringstream error;
- error << "Invalid iterator. Master's latest sequence number is " << local_latest_seq_num << " but "
- << "updates are requested from sequence number " << seq_number << ". "
- << "The master's WAL entries might have expired (they are kept only for 24 hours).";
- LOG(ERROR) << error.str();
- return Option<std::vector<std::string>*>(400, error.str());
- }
-
- uint64_t num_updates = 0;
- std::vector<std::string>* updates = new std::vector<std::string>();
-
- bool first_iteration = true;
-
- while(iter->Valid() && num_updates < max_updates) {
- const rocksdb::BatchResult & batch = iter->GetBatch();
- if(first_iteration) {
- first_iteration = false;
- if(batch.sequence != seq_number) {
- std::ostringstream error;
- error << "Invalid iterator. Requested sequence number is " << seq_number << " but "
- << "updates are available only from sequence number " << batch.sequence << ". "
- << "The master's WAL entries might have expired (they are kept only for 24 hours).";
- LOG(ERROR) << error.str();
- return Option<std::vector<std::string>*>(400, error.str());
- }
- }
-
- const std::string & write_batch_serialized = batch.writeBatchPtr->Data();
- updates->push_back(write_batch_serialized);
- num_updates += 1;
- iter->Next();
- }
-
- return Option<std::vector<std::string>*>(updates);
- }
+ const size_t wal_size_mb = 1024,
+ bool disable_wal = true,
+ int32_t ttl=0);
- void close() {
- std::unique_lock lock(mutex);
- delete db;
- db = nullptr;
- }
+ ~Store();
- int reload(bool clear_state_dir, const std::string& snapshot_path) {
- std::unique_lock lock(mutex);
+ bool insert(const std::string& key, const std::string& value);
- // we don't use close() to avoid nested lock and because lock is required until db is re-initialized
- delete db;
- db = nullptr;
+ bool batch_write(rocksdb::WriteBatch& batch);
- if(clear_state_dir) {
- if (!delete_path(state_dir_path, true)) {
- LOG(WARNING) << "rm " << state_dir_path << " failed";
- return -1;
- }
+ bool contains(const std::string& key) const;
- LOG(INFO) << "rm " << state_dir_path << " success";
- }
+ StoreStatus get(const std::string& key, std::string& value) const;
- if(!snapshot_path.empty()) {
- // tries to use link if possible, or else copies
- if (!copy_dir(snapshot_path, state_dir_path)) {
- LOG(WARNING) << "copy snapshot " << snapshot_path << " to " << state_dir_path << " failed";
- return -1;
- }
+ bool remove(const std::string& key);
- LOG(INFO) << "copy snapshot " << snapshot_path << " to " << state_dir_path << " success";
- }
+ rocksdb::Iterator* scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound);
- if (!create_directory(state_dir_path)) {
- LOG(WARNING) << "CreateDirectory " << state_dir_path << " failed";
- return -1;
- }
+ rocksdb::Iterator* get_iterator();
- const rocksdb::Status& status = init_db();
- if (!status.ok()) {
- LOG(WARNING) << "Open DB " << state_dir_path << " failed, msg: " << status.ToString();
- return -1;
- }
+ void scan_fill(const std::string& prefix_start, const std::string& prefix_end, std::vector<std::string> & values);
- LOG(INFO) << "DB open success!";
+ void increment(const std::string & key, uint32_t value);
- return 0;
- }
+ uint64_t get_latest_seq_number() const;
- void flush() {
- std::shared_lock lock(mutex);
- rocksdb::FlushOptions options;
- db->Flush(options);
- }
+ Option<std::vector<std::string>*> get_updates_since(const uint64_t seq_number_org, const uint64_t max_updates) const;
- rocksdb::Status compact_all() {
- std::shared_lock lock(mutex);
- return db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr);
- }
+ void close();
- rocksdb::Status create_check_point(rocksdb::Checkpoint** checkpoint_ptr, const std::string& db_snapshot_path) {
- std::shared_lock lock(mutex);
- rocksdb::Status status = rocksdb::Checkpoint::Create(db, checkpoint_ptr);
- if(!status.ok()) {
- LOG(ERROR) << "Checkpoint Create failed, msg:" << status.ToString();
- return status;
- }
+ int reload(bool clear_state_dir, const std::string& snapshot_path, int32_t ttl = 0);
- status = (*checkpoint_ptr)->CreateCheckpoint(db_snapshot_path);
+ void flush();
- if(!status.ok()) {
- LOG(WARNING) << "Checkpoint CreateCheckpoint failed at snapshot path: "
- << db_snapshot_path << ", msg:" << status.ToString();
- }
+ rocksdb::Status compact_all();
- return status;
- }
+ rocksdb::Status create_check_point(rocksdb::Checkpoint** checkpoint_ptr, const std::string& db_snapshot_path);
- rocksdb::Status delete_range(const std::string& begin_key, const std::string& end_key) {
- std::shared_lock lock(mutex);
- return db->DeleteRange(rocksdb::WriteOptions(), db->DefaultColumnFamily(), begin_key, end_key);
- }
+ rocksdb::Status delete_range(const std::string& begin_key, const std::string& end_key);
- rocksdb::Status compact_range(const rocksdb::Slice& begin_key, const rocksdb::Slice& end_key) {
- std::shared_lock lock(mutex);
- return db->CompactRange(rocksdb::CompactRangeOptions(), &begin_key, &end_key);
- }
+ rocksdb::Status compact_range(const rocksdb::Slice& begin_key, const rocksdb::Slice& end_key);
// Only for internal tests
- rocksdb::DB* _get_db_unsafe() const {
- return db;
- }
+ rocksdb::DB* _get_db_unsafe() const;
- const std::string& get_state_dir_path() const {
- return state_dir_path;
- }
+ const std::string& get_state_dir_path() const;
- const rocksdb::Options &get_db_options() const {
- return options;
- }
+ const rocksdb::Options &get_db_options() const;
- void print_memory_usage() {
- std::string index_usage;
- db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);
- LOG(INFO) << "rocksdb.estimate-table-readers-mem: " << index_usage;
+ void print_memory_usage();
- std::string memtable_usage;
- db->GetProperty("rocksdb.cur-size-all-mem-tables", &memtable_usage);
- LOG(INFO) << "rocksdb.cur-size-all-mem-tables: " << memtable_usage;
- }
-};
+ void get_last_N_values(const std::string& userid_prefix, uint32_t N, std::vector<std::string>& values);
+};
\ No newline at end of file
diff --git a/include/tsconfig.h b/include/tsconfig.h
index 720232cb..089d7f8f 100644
--- a/include/tsconfig.h
+++ b/include/tsconfig.h
@@ -12,7 +12,7 @@ private:
std::string data_dir;
std::string log_dir;
std::string analytics_dir;
-
+ int32_t analytics_db_ttl = 2419200; //four weeks in secs
std::string api_key;
// @deprecated
@@ -152,6 +152,10 @@ public:
this->analytics_dir = analytics_dir;
}
+ void set_analytics_db_ttl(int32_t analytics_db_ttl) {
+ this->analytics_db_ttl = analytics_db_ttl;
+ }
+
void set_api_key(const std::string & api_key) {
this->api_key = api_key;
}
@@ -231,6 +235,10 @@ public:
return this->analytics_dir;
}
+ int32_t get_analytics_db_ttl() const {
+ return this->analytics_db_ttl;
+ }
+
std::string get_api_key() const {
return this->api_key;
}
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index 0194019e..c39d5609 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -522,7 +522,7 @@ void AnalyticsManager::run(ReplicationState* raft_server) {
}
persist_query_events(raft_server, prev_persistence_s);
- persist_events();
+ persist_events(raft_server, prev_persistence_s);
persist_popular_events(raft_server, prev_persistence_s);
prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(
@@ -618,25 +618,57 @@ void AnalyticsManager::persist_query_events(ReplicationState *raft_server, uint6
}
}
-void AnalyticsManager::persist_events() {
+void AnalyticsManager::persist_events(ReplicationState *raft_server, uint64_t prev_persistence_s) {
// lock is held by caller
+
+ auto send_http_response = [&](const std::string& import_payload) {
+ // send http request
+ if(raft_server == nullptr) {
+ return;
+ }
+
+ std::string leader_url = raft_server->get_leader_url();
+ if(!leader_url.empty()) {
+ const std::string& base_url = leader_url + "analytics/";
+ std::string res;
+
+ const std::string& update_url = base_url + "/write_to_db";
+ std::map<std::string, std::string> res_headers;
+ long status_code = HttpClient::post_response(update_url, import_payload,
+ res, res_headers, {}, 10*1000, true);
+
+ if(status_code != 200) {
+ LOG(ERROR) << "Error while sending "<<" log events to leader. "
+ << "Status code: " << status_code << ", response: " << res;
+ }
+ }
+ };
+
+ nlohmann::json payload = nlohmann::json::array();
for (auto &events_collection_it: query_collection_events) {
const auto& collection = events_collection_it.first;
for (const auto &event: events_collection_it.second) {
if (analytics_logs.is_open() && event.log_to_file) {
//store events to log file
- analytics_logs << event.timestamp << "\t" << event.name << "\t"
+ std::stringstream ssbuf;
+ ssbuf << event.timestamp << "\t" << event.name << "\t"
<< collection << "\t" << event.user_id << "\t" << event.doc_id << "\t"
<< event.query << "\t";
for(const auto& kv : event.data) {
- analytics_logs << kv.second << "\t";
+ ssbuf << kv.second << "\t";
}
+ analytics_logs << ssbuf.str();
analytics_logs << "\n";
analytics_logs << std::flush;
+
+ nlohmann::json event_data;
+ event.to_json(event_data, collection);
+ payload.push_back(event_data);
}
}
+ send_http_response(payload.dump());
events_collection_it.second.clear();
}
}
@@ -707,8 +739,9 @@ void AnalyticsManager::dispose() {
events_cache.clear();
}
-void AnalyticsManager::init(Store* store, const std::string& analytics_dir) {
+void AnalyticsManager::init(Store* store, Store* analytics_store, const std::string& analytics_dir) {
this->store = store;
+ this->analytics_store = analytics_store;
if(!analytics_dir.empty()) {
const auto analytics_log_path = analytics_dir + "/analytics_events.tsv";
@@ -749,4 +782,43 @@ void counter_event_t::serialize_as_docs(std::string &docs) {
if (!docs.empty()) {
docs.pop_back();
}
+}
+
+bool AnalyticsManager::write_to_db(const nlohmann::json& payload) {
+ if(analytics_store) {
+ for(const auto& event: payload) {
+ std::string key = event["user_id"].get<std::string>() + "_" + StringUtils::serialize_uint64_t(event["timestamp"].get<uint64_t>());
+ bool inserted = analytics_store->insert(key, event.dump());
+ if(!inserted) {
+ LOG(ERROR) << "Error while dumping events to analytics db.";
+ return false;
+ }
+ }
+ } else {
+ LOG(ERROR) << "Analytics DB not initialized!!";
+ return false;
+ }
+
+ return true;
+}
+
+void AnalyticsManager::get_last_N_events(const std::string& userid, uint32_t N, std::vector<std::string>& values) {
+ const std::string userid_prefix = userid + "_";
+ analytics_store->get_last_N_values(userid_prefix, N, values);
+}
+
+void event_t::to_json(nlohmann::json& obj, const std::string& coll) const {
+ obj["query"] = query;
+ obj["type"] = event_type;
+ obj["timestamp"] = timestamp;
+ obj["user_id"] = user_id;
+ obj["doc_id"] = doc_id;
+ obj["name"] = name;
+ obj["collection"] = coll;
+
+ if(event_type == "custom") {
+ for(const auto& kv : data) {
+ obj[kv.first] = kv.second;
+ }
+ }
}
\ No newline at end of file
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 6afd3ea1..45776b4b 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -2844,6 +2844,22 @@ bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared
return true;
}
+bool post_write_analytics_to_db(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ nlohmann::json req_json;
+
+ try {
+ req_json = nlohmann::json::parse(req->body);
+ } catch(const std::exception& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ AnalyticsManager::get_instance().write_to_db(req_json);
+
+ return true;
+}
+
bool post_proxy(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
HttpProxy& proxy = HttpProxy::get_instance();
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index fe13a54d..e95a01a2 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -80,6 +80,7 @@ void master_server_routes() {
server->put("/analytics/rules/:name", put_upsert_analytics_rules);
server->del("/analytics/rules/:name", del_analytics_rules);
server->post("/analytics/events", post_create_event);
+ server->post("analytics/aggregate_events", post_write_analytics_to_db);
// meta
server->get("/metrics.json", get_metrics_json);
diff --git a/src/raft_server.cpp b/src/raft_server.cpp
index 6ce78567..096aaf16 100644
--- a/src/raft_server.cpp
+++ b/src/raft_server.cpp
@@ -655,7 +655,7 @@ int ReplicationState::on_snapshot_load(braft::SnapshotReader* reader) {
if(analytics_store) {
snapshot_path.append(std::string("/") + analytics_db_snapshot_name);
- int reload_store = analytics_store->reload(true, snapshot_path);
+ int reload_store = analytics_store->reload(true, snapshot_path, Config::get_instance().get_analytics_db_ttl());
if (reload_store != 0) {
LOG(ERROR) << "Failed to reload analytics db snapshot.";
return reload_store;
diff --git a/src/store.cpp b/src/store.cpp
new file mode 100644
index 00000000..902aa84c
--- /dev/null
+++ b/src/store.cpp
@@ -0,0 +1,355 @@
+#include "include/store.h"
+
+Store::Store(const std::string & state_dir_path,
+ const size_t wal_ttl_secs,
+ const size_t wal_size_mb, bool disable_wal, int32_t ttl):
+ state_dir_path(state_dir_path){
+ // Optimize RocksDB
+ options.IncreaseParallelism();
+ options.OptimizeLevelStyleCompaction();
+ // create the DB if it's not already present
+ options.create_if_missing = true;
+ options.write_buffer_size = 4*1048576;
+ options.max_write_buffer_number = 2;
+ options.merge_operator.reset(new UInt64AddOperator);
+ options.compression = rocksdb::CompressionType::kSnappyCompression;
+
+ options.max_log_file_size = 4*1048576;
+ options.keep_log_file_num = 5;
+
+ /*options.table_properties_collector_factories.emplace_back(
+ rocksdb::NewCompactOnDeletionCollectorFactory(10000, 7500, 0.5));*/
+
+ // these need to be high for replication scenarios
+ options.WAL_ttl_seconds = wal_ttl_secs;
+ options.WAL_size_limit_MB = wal_size_mb;
+
+ // Disable WAL for master writes (Raft's WAL is used)
+ // The replica uses native WAL, though.
+ write_options.disableWAL = disable_wal;
+
+ // open DB
+ init_db(ttl);
+}
+
+Store::~Store() {
+ close();
+}
+
+rocksdb::Status Store::init_db(int32_t ttl) {
+ LOG(INFO) << "Initializing DB by opening state dir: " << state_dir_path;
+
+ rocksdb::Status s;
+
+ if(ttl > 0) {
+ rocksdb::DBWithTTL* dbWithTtl;
+ s = rocksdb::DBWithTTL::Open(options, state_dir_path,
+ &dbWithTtl, ttl, false);
+ db = dbWithTtl;
+ } else {
+ s = rocksdb::DB::Open(options, state_dir_path, &db);
+ }
+
+ if(!s.ok()) {
+ LOG(ERROR) << "Error while initializing store: " << s.ToString();
+ if(s.code() == rocksdb::Status::Code::kIOError) {
+ LOG(ERROR) << "It seems like the data directory " << state_dir_path << " is already being used by "
+ << "another Typesense server. ";
+ LOG(ERROR) << "If you are SURE that this is not the case, delete the LOCK file "
+ << "in the data db directory and try again.";
+ }
+ }
+
+ assert(s.ok());
+ return s;
+}
+
+bool Store::insert(const std::string& key, const std::string& value) {
+ std::shared_lock lock(mutex);
+ rocksdb::Status status = db->Put(write_options, key, value);
+ return status.ok();
+}
+
+bool Store::batch_write(rocksdb::WriteBatch& batch) {
+ std::shared_lock lock(mutex);
+ rocksdb::Status status = db->Write(write_options, &batch);
+ return status.ok();
+}
+
+bool Store::contains(const std::string& key) const {
+ std::shared_lock lock(mutex);
+
+ std::string value;
+ bool value_found;
+ bool key_may_exist = db->KeyMayExist(rocksdb::ReadOptions(), key, &value, &value_found);
+
+ // returns false when key definitely does not exist
+ if(!key_may_exist) {
+ return false;
+ }
+
+ if(value_found) {
+ return true;
+ }
+
+ // otherwise, we have try getting the value
+ rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);
+ return status.ok() && !status.IsNotFound();
+}
+
+StoreStatus Store::get(const std::string& key, std::string& value) const {
+ std::shared_lock lock(mutex);
+ rocksdb::Status status = db->Get(rocksdb::ReadOptions(), key, &value);
+
+ if(status.ok()) {
+ return StoreStatus::FOUND;
+ }
+
+ if(status.IsNotFound()) {
+ return StoreStatus::NOT_FOUND;
+ }
+
+ LOG(ERROR) << "Error while fetching the key: " << key << " - status is: " << status.ToString();
+ return StoreStatus::ERROR;
+}
+
+bool Store::remove(const std::string& key) {
+ std::shared_lock lock(mutex);
+ rocksdb::Status status = db->Delete(write_options, key);
+ return status.ok();
+}
+
+rocksdb::Iterator* Store::scan(const std::string & prefix, const rocksdb::Slice* iterate_upper_bound) {
+ std::shared_lock lock(mutex);
+ rocksdb::ReadOptions read_opts;
+ if(iterate_upper_bound) {
+ read_opts.iterate_upper_bound = iterate_upper_bound;
+ }
+ rocksdb::Iterator *iter = db->NewIterator(read_opts);
+ iter->Seek(prefix);
+ return iter;
+}
+
+rocksdb::Iterator* Store::get_iterator() {
+ std::shared_lock lock(mutex);
+ rocksdb::Iterator* it = db->NewIterator(rocksdb::ReadOptions());
+ return it;
+}
+
+void Store::scan_fill(const std::string& prefix_start, const std::string& prefix_end, std::vector<std::string> & values) {
+ rocksdb::ReadOptions read_opts;
+ rocksdb::Slice upper_bound(prefix_end);
+ read_opts.iterate_upper_bound = &upper_bound;
+
+ std::shared_lock lock(mutex);
+ rocksdb::Iterator *iter = db->NewIterator(read_opts);
+ for (iter->Seek(prefix_start); iter->Valid() && iter->key().starts_with(prefix_start); iter->Next()) {
+ values.push_back(iter->value().ToString());
+ }
+
+ delete iter;
+}
+
+void Store::increment(const std::string & key, uint32_t value) {
+ std::shared_lock lock(mutex);
+ db->Merge(write_options, key, StringUtils::serialize_uint32_t(value));
+}
+
+uint64_t Store::get_latest_seq_number() const {
+ std::shared_lock lock(mutex);
+ return db->GetLatestSequenceNumber();
+}
+
+Option<std::vector<std::string>*> Store::get_updates_since(const uint64_t seq_number_org, const uint64_t max_updates) const {
+ std::shared_lock lock(mutex);
+ const uint64_t local_latest_seq_num = db->GetLatestSequenceNumber();
+
+ // Since GetUpdatesSince(0) == GetUpdatesSince(1)
+ const uint64_t seq_number = (seq_number_org == 0) ? 1 : seq_number_org;
+
+ if(seq_number == local_latest_seq_num+1) {
+ // replica has caught up, send an empty list as result
+ std::vector<std::string>* updates = new std::vector<std::string>();
+ return Option<std::vector<std::string>*>(updates);
+ }
+
+ std::unique_ptr<rocksdb::TransactionLogIterator> iter;
+ rocksdb::Status status = db->GetUpdatesSince(seq_number, &iter);
+
+ if(!status.ok()) {
+ LOG(ERROR) << "Error while fetching updates for replication: " << status.ToString();
+
+ std::ostringstream error;
+ error << "Unable to fetch updates. " << "Master's latest sequence number is " << local_latest_seq_num
+ << " but requested sequence number is " << seq_number;
+ LOG(ERROR) << error.str();
+
+ return Option<std::vector<std::string>*>(400, error.str());
+ }
+
+ if(!iter->Valid()) {
+ std::ostringstream error;
+ error << "Invalid iterator. Master's latest sequence number is " << local_latest_seq_num << " but "
+ << "updates are requested from sequence number " << seq_number << ". "
+ << "The master's WAL entries might have expired (they are kept only for 24 hours).";
+ LOG(ERROR) << error.str();
+ return Option<std::vector<std::string>*>(400, error.str());
+ }
+
+ uint64_t num_updates = 0;
+ std::vector<std::string>* updates = new std::vector<std::string>();
+
+ bool first_iteration = true;
+
+ while(iter->Valid() && num_updates < max_updates) {
+ const rocksdb::BatchResult & batch = iter->GetBatch();
+ if(first_iteration) {
+ first_iteration = false;
+ if(batch.sequence != seq_number) {
+ std::ostringstream error;
+ error << "Invalid iterator. Requested sequence number is " << seq_number << " but "
+ << "updates are available only from sequence number " << batch.sequence << ". "
+ << "The master's WAL entries might have expired (they are kept only for 24 hours).";
+ LOG(ERROR) << error.str();
+ return Option<std::vector<std::string>*>(400, error.str());
+ }
+ }
+
+ const std::string & write_batch_serialized = batch.writeBatchPtr->Data();
+ updates->push_back(write_batch_serialized);
+ num_updates += 1;
+ iter->Next();
+ }
+
+ return Option<std::vector<std::string>*>(updates);
+}
+
+void Store::close() {
+ std::unique_lock lock(mutex);
+ delete db;
+ db = nullptr;
+}
+
+int Store::reload(bool clear_state_dir, const std::string& snapshot_path, int32_t ttl) {
+ std::unique_lock lock(mutex);
+
+ // we don't use close() to avoid nested lock and because lock is required until db is re-initialized
+ delete db;
+ db = nullptr;
+
+ if(clear_state_dir) {
+ if (!delete_path(state_dir_path, true)) {
+ LOG(WARNING) << "rm " << state_dir_path << " failed";
+ return -1;
+ }
+
+ LOG(INFO) << "rm " << state_dir_path << " success";
+ }
+
+ if(!snapshot_path.empty()) {
+ // tries to use link if possible, or else copies
+ if (!copy_dir(snapshot_path, state_dir_path)) {
+ LOG(WARNING) << "copy snapshot " << snapshot_path << " to " << state_dir_path << " failed";
+ return -1;
+ }
+
+ LOG(INFO) << "copy snapshot " << snapshot_path << " to " << state_dir_path << " success";
+ }
+
+ if (!create_directory(state_dir_path)) {
+ LOG(WARNING) << "CreateDirectory " << state_dir_path << " failed";
+ return -1;
+ }
+
+ const rocksdb::Status& status = init_db(ttl);
+ if (!status.ok()) {
+ LOG(WARNING) << "Open DB " << state_dir_path << " failed, msg: " << status.ToString();
+ return -1;
+ }
+
+ LOG(INFO) << "DB open success!";
+
+ return 0;
+}
+
+void Store::flush() {
+ std::shared_lock lock(mutex);
+ rocksdb::FlushOptions options;
+ db->Flush(options);
+}
+
+rocksdb::Status Store::compact_all() {
+ std::shared_lock lock(mutex);
+ return db->CompactRange(rocksdb::CompactRangeOptions(), nullptr, nullptr);
+}
+
+rocksdb::Status Store::create_check_point(rocksdb::Checkpoint** checkpoint_ptr, const std::string& db_snapshot_path) {
+ std::shared_lock lock(mutex);
+ rocksdb::Status status = rocksdb::Checkpoint::Create(db, checkpoint_ptr);
+ if(!status.ok()) {
+ LOG(ERROR) << "Checkpoint Create failed, msg:" << status.ToString();
+ return status;
+ }
+
+ status = (*checkpoint_ptr)->CreateCheckpoint(db_snapshot_path);
+
+ if(!status.ok()) {
+ LOG(WARNING) << "Checkpoint CreateCheckpoint failed at snapshot path: "
+ << db_snapshot_path << ", msg:" << status.ToString();
+ }
+
+ return status;
+}
+
+rocksdb::Status Store::delete_range(const std::string& begin_key, const std::string& end_key) {
+ std::shared_lock lock(mutex);
+ return db->DeleteRange(rocksdb::WriteOptions(), db->DefaultColumnFamily(), begin_key, end_key);
+}
+
+rocksdb::Status Store::compact_range(const rocksdb::Slice& begin_key, const rocksdb::Slice& end_key) {
+ std::shared_lock lock(mutex);
+ return db->CompactRange(rocksdb::CompactRangeOptions(), &begin_key, &end_key);
+}
+
+rocksdb::DB* Store::_get_db_unsafe() const {
+ return db;
+}
+
+const std::string& Store::get_state_dir_path() const {
+ return state_dir_path;
+}
+
+const rocksdb::Options& Store::get_db_options() const {
+ return options;
+}
+
+void Store::print_memory_usage() {
+ std::string index_usage;
+ db->GetProperty("rocksdb.estimate-table-readers-mem", &index_usage);
+ LOG(INFO) << "rocksdb.estimate-table-readers-mem: " << index_usage;
+
+ std::string memtable_usage;
+ db->GetProperty("rocksdb.cur-size-all-mem-tables", &memtable_usage);
+ LOG(INFO) << "rocksdb.cur-size-all-mem-tables: " << memtable_usage;
+}
+
+void Store::get_last_N_values(const std::string& userid_prefix, uint32_t N, std::vector<std::string>& values) {
+ std::shared_lock lock(mutex);
+
+ rocksdb::Iterator* iter = db->NewIterator(rocksdb::ReadOptions());
+ auto prefix_key = userid_prefix + "~";
+ iter->SeekForPrev(prefix_key);
+
+ while(iter->Valid() && N) {
+ auto key = iter->key().ToString();
+ if(!StringUtils::begins_with(key, userid_prefix)) {
+ break;
+ }
+
+ values.push_back(iter->value().ToString());
+ N--;
+ iter->Prev();
+ }
+
+ delete iter;
+}
\ No newline at end of file
diff --git a/src/tsconfig.cpp b/src/tsconfig.cpp
index 641e260b..e214396c 100644
--- a/src/tsconfig.cpp
+++ b/src/tsconfig.cpp
@@ -129,7 +129,6 @@ Option<std::string> Config::fetch_nodes_config(const std::string& path_to_nodes)
void Config::load_config_env() {
this->data_dir = get_env("TYPESENSE_DATA_DIR");
this->log_dir = get_env("TYPESENSE_LOG_DIR");
- this->analytics_dir = get_env("TYPESENSE_ANALYTICS_DIR");
this->api_key = get_env("TYPESENSE_API_KEY");
// @deprecated
@@ -261,6 +260,14 @@ void Config::load_config_env() {
if(!get_env("TYPESENSE_MAX_PER_PAGE").empty()) {
this->max_per_page = std::stoi(get_env("TYPESENSE_MAX_PER_PAGE"));
}
+
+ if(!get_env("TYPESENSE_ANALYTICS_DIR").empty()) {
+ this->analytics_dir = get_env("TYPESENSE_ANALYTICS_DIR");
+ }
+
+ if(!get_env("TYPESENSE_ANALYTICS_DB_TTL").empty()) {
+ this->analytics_db_ttl = std::stoi(get_env("TYPESENSE_ANALYTICS_DB_TTL"));
+ }
}
void Config::load_config_file(cmdline::parser& options) {
@@ -295,6 +302,10 @@ void Config::load_config_file(cmdline::parser& options) {
this->analytics_dir = reader.Get("server", "analytics-dir", "");
}
+ if(reader.Exists("server", "analytics-db-ttl")) {
+ this->analytics_db_ttl = std::stoi(reader.Get("server", "analytics-db-ttl", "0"));
+ }
+
if(reader.Exists("server", "api-key")) {
this->api_key = reader.Get("server", "api-key", "");
}
@@ -483,6 +494,10 @@ void Config::load_config_cmd_args(cmdline::parser& options) {
this->analytics_dir = options.get<std::string>("analytics-dir");
}
+ if(options.exist("analytics-db-ttl")) {
+ this->analytics_db_ttl = options.get<uint32_t>("analytics-db-ttl");
+ }
+
if(options.exist("api-key")) {
this->api_key = options.get<std::string>("api-key");
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 2f66963c..94f0395e 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -68,6 +68,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("search-only-api-key", 's', "[DEPRECATED: use API key management end-point] API key that allows only searches.", false);
options.add<std::string>("health-rusage-api-key", '\0', "API key that allows access to health end-point with resource usage.", false);
options.add<std::string>("analytics-dir", '\0', "Directory where Analytics will be stored.", false);
+ options.add<uint32_t>("analytics-db-ttl", '\0', "TTL in seconds for events stored in analytics db", false);
options.add<std::string>("api-address", '\0', "Address to which Typesense API service binds.", false, "0.0.0.0");
options.add<uint32_t>("api-port", '\0', "Port on which Typesense API service listens.", false, 8108);
@@ -385,6 +386,7 @@ int run_server(const Config & config, const std::string & version, void (*master
std::string state_dir = config.get_data_dir() + "/state";
std::string meta_dir = config.get_data_dir() + "/meta";
std::string analytics_dir = config.get_analytics_dir();
+ int32_t analytics_db_ttl = config.get_analytics_db_ttl();
size_t thread_pool_size = config.get_thread_pool_size();
@@ -406,14 +408,14 @@ int run_server(const Config & config, const std::string & version, void (*master
// meta DB for storing house keeping things
Store meta_store(meta_dir, 24*60*60, 1024, false);
- //analytics DB for storing query click events
- std::unique_ptr<Store> analytics_store = nullptr;
+ //analytics DB for storing analytics events
+ Store analytics_store(analytics_dir, 24*60*60, 1024, true, analytics_db_ttl);
curl_global_init(CURL_GLOBAL_SSL);
HttpClient & httpClient = HttpClient::get_instance();
httpClient.init(config.get_api_key());
- AnalyticsManager::get_instance().init(&store, analytics_dir);
+ AnalyticsManager::get_instance().init(&store, &analytics_store, analytics_dir);
server = new HttpServer(
version,
@@ -464,7 +466,7 @@ int run_server(const Config & config, const std::string & version, void (*master
// first we start the peering service
- ReplicationState replication_state(server, batch_indexer, &store, analytics_store.get(),
+ ReplicationState replication_state(server, batch_indexer, &store, &analytics_store,
&replication_thread_pool, server->get_message_dispatcher(),
ssl_enabled,
&config,
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index dfcf8f27..9324ab52 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -9,8 +9,10 @@
class AnalyticsManagerTest : public ::testing::Test {
protected:
Store *store;
+ Store *analytic_store;
CollectionManager& collectionManager = CollectionManager::get_instance();
std::atomic<bool> quit = false;
+ std::string state_dir_path, analytics_dir_path;
std::vector<std::string> query_fields;
std::vector<sort_by> sort_fields;
@@ -18,16 +20,23 @@ protected:
AnalyticsManager& analyticsManager = AnalyticsManager::get_instance();
void setupCollection() {
- std::string state_dir_path = "/tmp/typesense_test/analytics_manager_test";
+ state_dir_path = "/tmp/typesense_test/analytics_manager_test";
+ analytics_dir_path = "/tmp/typesense-test/analytics";
+
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
system("mkdir -p /tmp/typesense_test/models");
store = new Store(state_dir_path);
+
+ LOG(INFO) << "Truncating and creating: " << analytics_dir_path;
+ system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());
+ analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);
+
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
- analyticsManager.init(store, state_dir_path);
+ analyticsManager.init(store, analytic_store, state_dir_path);
analyticsManager.resetToggleRateLimit(false);
}
@@ -38,6 +47,7 @@ protected:
virtual void TearDown() {
collectionManager.dispose();
delete store;
+ delete analytic_store;
analyticsManager.stop();
}
};
@@ -514,7 +524,7 @@ TEST_F(AnalyticsManagerTest, EventsPersist) {
req->body = event.dump();
ASSERT_TRUE(post_create_event(req, res));
- analyticsManager.persist_events();
+ analyticsManager.persist_events(nullptr, 0);
auto fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
@@ -540,7 +550,7 @@ TEST_F(AnalyticsManagerTest, EventsPersist) {
req->body = event.dump();
ASSERT_TRUE(post_create_event(req, res));
- analyticsManager.persist_events();
+ analyticsManager.persist_events(nullptr, 0);
fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
@@ -1007,7 +1017,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
analyticsManager.dispose();
analyticsManager.stop();
system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
- analyticsManager.init(store, "/tmp/typesense_test/analytics_manager_test");
+ analyticsManager.init(store, analytic_store, "/tmp/typesense_test/analytics_manager_test");
nlohmann::json products_schema = R"({
"name": "books",
@@ -1235,7 +1245,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
ASSERT_EQ("Cool trousers", results["hits"][1]["document"]["title"]);
//verify log file
- analyticsManager.persist_events();
+ analyticsManager.persist_events(nullptr, 0);
auto fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
@@ -1303,7 +1313,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
ASSERT_EQ(10, popular_clicks["books"].docid_counts["1"]);
//check log file
- analyticsManager.persist_events();
+ analyticsManager.persist_events(nullptr, 0);
fileOutput = Config::fetch_file_contents("/tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
@@ -1328,7 +1338,7 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
analyticsManager.dispose();
analyticsManager.stop();
system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
- analyticsManager.init(store, "");
+ analyticsManager.init(store, analytic_store, "");
analytics_rule = R"({
"name": "books_popularity3",
@@ -1351,11 +1361,10 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
//counter events should work without analytic-dir
-
analyticsManager.dispose();
analyticsManager.stop();
system("rm -rf /tmp/typesense_test/analytics_manager_test/analytics_events.tsv");
- analyticsManager.init(store, "");
+ analyticsManager.init(store, analytic_store, "");
analytics_rule = R"({
"name": "books_popularity3",
@@ -1392,4 +1401,197 @@ TEST_F(AnalyticsManagerTest, PopularityScoreValidation) {
ASSERT_EQ("popularity", popular_clicks["books"].counter_field);
ASSERT_EQ(1, popular_clicks["books"].docid_counts.size());
ASSERT_EQ(5, popular_clicks["books"].docid_counts["1"]);
+}
+
+TEST_F(AnalyticsManagerTest, AnalyticsStoreTTL) {
+ analyticsManager.dispose();
+ analyticsManager.stop();
+ delete analytic_store;
+
+ //set TTL of an hour
+ LOG(INFO) << "Truncating and creating: " << analytics_dir_path;
+ system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());
+
+ analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);
+ analyticsManager.init(store, analytic_store, "");
+
+ auto analytics_rule = R"({
+ "name": "product_events2",
+ "type": "log",
+ "params": {
+ "name": "product_events_logging2",
+ "source": {
+ "collections": ["titles"],
+ "events": [{"type": "click", "name": "AB"}]
+ }
+ }
+ })"_json;
+
+ auto create_op = analyticsManager.create_rule(analytics_rule, true, true);
+ ASSERT_TRUE(create_op.ok());
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ nlohmann::json event1 = R"({
+ "type": "click",
+ "name": "AB",
+ "data": {
+ "q": "technology",
+ "doc_id": "21",
+ "user_id": "13"
+ }
+ })"_json;
+
+ req->body = event1.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+
+ //get events
+ nlohmann::json payload = nlohmann::json::array();
+ nlohmann::json event_data;
+ auto collection_events_map = analyticsManager.get_log_events();
+ for (auto &events_collection_it: collection_events_map) {
+ const auto& collection = events_collection_it.first;
+ for(const auto& event: events_collection_it.second) {
+ event.to_json(event_data, collection);
+ }
+ }
+ payload.push_back(event_data);
+
+ //manually trigger write to db
+ ASSERT_TRUE(analyticsManager.write_to_db(payload));
+
+ //try fetching from db
+ const std::string prefix_start = "13_";
+ const std::string prefix_end = "13`";
+ std::vector<std::string> events;
+
+ analytic_store->scan_fill(prefix_start, prefix_end, events);
+ ASSERT_EQ(1, events.size());
+ ASSERT_EQ(events[0].c_str(), event_data.dump());
+
+ //now set TTL to 1s and open analytics db
+ events.clear();
+ delete analytic_store;
+
+ analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, 1);
+
+ sleep(2);
+ analytic_store->compact_all();
+
+ analytic_store->scan_fill(prefix_start, prefix_end, events);
+ ASSERT_EQ(0, events.size());
+}
+
+TEST_F(AnalyticsManagerTest, AnalyticsStoreGetLastN) {
+ analyticsManager.dispose();
+ analyticsManager.stop();
+ delete analytic_store;
+
+ //set TTL of an hour
+ LOG(INFO) << "Truncating and creating: " << analytics_dir_path;
+ system(("rm -rf "+ analytics_dir_path +" && mkdir -p "+analytics_dir_path).c_str());
+
+ analytic_store = new Store(analytics_dir_path, 24*60*60, 1024, true, FOURWEEKS_SECS);
+ analyticsManager.init(store, analytic_store, "");
+
+ auto analytics_rule = R"({
+ "name": "product_events2",
+ "type": "log",
+ "params": {
+ "name": "product_events_logging2",
+ "source": {
+ "collections": ["titles"],
+ "events": [{"type": "click", "name": "AB"}]
+ }
+ }
+ })"_json;
+
+ auto create_op = analyticsManager.create_rule(analytics_rule, true, true);
+ ASSERT_TRUE(create_op.ok());
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ nlohmann::json event1;
+ event1["type"] = "click";
+ event1["name"] = "AB";
+ event1["data"]["q"] = "technology";
+ event1["data"]["user_id"] = "13";
+
+ for(auto i = 0; i < 10; i++) {
+ event1["data"]["doc_id"] = std::to_string(i);
+ req->body = event1.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+ }
+
+ //add more user events
+ for(auto i = 0; i < 7; i++) {
+ event1["data"]["user_id"] = "14";
+ event1["data"]["doc_id"] = std::to_string(i);
+ req->body = event1.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+ }
+
+ for(auto i = 0; i < 5; i++) {
+ event1["data"]["user_id"] = "15";
+ event1["data"]["doc_id"] = std::to_string(i);
+ req->body = event1.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+ }
+
+ //get events
+ nlohmann::json payload = nlohmann::json::array();
+ nlohmann::json event_data;
+ auto collection_events_map = analyticsManager.get_log_events();
+ for (auto &events_collection_it: collection_events_map) {
+ const auto& collection = events_collection_it.first;
+ for(const auto& event: events_collection_it.second) {
+ event.to_json(event_data, collection);
+ payload.push_back(event_data);
+ }
+ }
+
+ //manually trigger write to db
+ ASSERT_TRUE(analyticsManager.write_to_db(payload));
+
+ //basic test
+ std::vector<std::string> values;
+ analyticsManager.get_last_N_events("13", 5, values);
+ ASSERT_EQ(5, values.size());
+
+ nlohmann::json parsed_json;
+ uint32_t start_index = 9;
+ for(auto i = 0; i < 5; i++) {
+ parsed_json = nlohmann::json::parse(values[i]);
+ ASSERT_EQ(std::to_string(start_index - i), parsed_json["doc_id"]);
+ }
+
+ //fetch events for middle user
+ values.clear();
+ analyticsManager.get_last_N_events("14", 5, values);
+ ASSERT_EQ(5, values.size());
+
+ start_index = 6;
+ for(auto i = 0; i < 5; i++) {
+ parsed_json = nlohmann::json::parse(values[i]);
+ ASSERT_EQ(std::to_string(start_index - i), parsed_json["doc_id"]);
+ }
+
+ //fetch more events than stored in db
+ values.clear();
+ analyticsManager.get_last_N_events("15", 8, values);
+ ASSERT_EQ(5, values.size());
+
+ start_index = 4;
+ for(auto i = 0; i < 5; i++) {
+ parsed_json = nlohmann::json::parse(values[i]);
+ ASSERT_EQ(std::to_string(start_index - i), parsed_json["doc_id"]);
+ }
+
+
+ //fetch events for non-existing user
+ values.clear();
+ analyticsManager.get_last_N_events("16", 8, values);
+ ASSERT_EQ(0, values.size());
}
\ No newline at end of file
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 86b13819..ad985251 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -10,6 +10,7 @@
class CollectionManagerTest : public ::testing::Test {
protected:
Store *store;
+ Store* analytic_store;
CollectionManager & collectionManager = CollectionManager::get_instance();
std::atomic<bool> quit = false;
Collection *collection1;
@@ -22,10 +23,12 @@ protected:
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
+ analytic_store = new Store(state_dir_path + "/analytics");
+
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
- AnalyticsManager::get_instance().init(store);
+ AnalyticsManager::get_instance().init(store, analytic_store);
schema = R"({
"name": "collection1",
@@ -62,6 +65,7 @@ protected:
collectionManager.drop_collection("collection1");
collectionManager.dispose();
delete store;
+ delete analytic_store;
}
}
};