Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

adding field user_id and creating new store for analytics

krunale26c78219047
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index 4c9deae1..36efbd11 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -10,6 +10,7 @@
struct ClickEvent {
std::string query;
uint64_t timestamp;
+ std::string user_id;
std::string doc_id;
uint64_t position;
@@ -17,9 +18,10 @@ struct ClickEvent {
~ClickEvent() = default;
- ClickEvent(std::string q, uint64_t ts, std::string id, uint64_t pos) {
+ ClickEvent(std::string q, uint64_t ts, std::string uid, std::string id, uint64_t pos) {
query = q;
timestamp = ts;
+ user_id = uid;
doc_id = id;
position = pos;
}
@@ -28,6 +30,7 @@ struct ClickEvent {
if (this != &other) {
query = other.query;
timestamp = other.timestamp;
+ user_id = other.user_id;
doc_id = other.doc_id;
position = other.position;
return *this;
@@ -37,6 +40,7 @@ struct ClickEvent {
void to_json(nlohmann::json& obj) const {
obj["query"] = query;
obj["timestamp"] = timestamp;
+ obj["user_id"] = user_id;
obj["doc_id"] = doc_id;
obj["position"] = position;
}
@@ -80,6 +84,7 @@ private:
std::unordered_map<std::string, std::vector<ClickEvent>> query_collection_click_events;
Store* store = nullptr;
+ Store* analytics_store = nullptr;
AnalyticsManager() {}
@@ -105,7 +110,7 @@ public:
AnalyticsManager(AnalyticsManager const&) = delete;
void operator=(AnalyticsManager const&) = delete;
- void init(Store* store);
+ void init(Store* store, Store* analytics_store);
void run(ReplicationState* raft_server);
@@ -128,7 +133,7 @@ public:
std::unordered_map<std::string, PopularQueries*> get_popular_queries();
- void add_click_event(const std::string& query_collection, const std::string& query,
+ void add_click_event(const std::string& query_collection, const std::string& query, const std::string& user_id,
std::string doc_id, uint64_t position);
void persist_click_event(ReplicationState *raft_server, uint64_t prev_persistence_s);
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index 02f364d9..d1539812 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -218,7 +218,7 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection, const
}
}
-void AnalyticsManager::add_click_event(const std::string &query_collection, const std::string &query,
+void AnalyticsManager::add_click_event(const std::string &query_collection, const std::string &query, const std::string &user_id,
std::string doc_id, uint64_t position) {
std::unique_lock lock(mutex);
auto &click_events_vec = query_collection_click_events[query_collection];
@@ -226,7 +226,7 @@ void AnalyticsManager::add_click_event(const std::string &query_collection, cons
auto now_ts_seconds = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
- ClickEvent click_event(query, now_ts_seconds, doc_id, position);
+ ClickEvent click_event(query, now_ts_seconds, user_id, doc_id, position);
click_events_vec.emplace_back(click_event);
}
@@ -332,35 +332,40 @@ void AnalyticsManager::persist_suggestions(ReplicationState *raft_server, uint64
void AnalyticsManager::persist_click_event(ReplicationState *raft_server, uint64_t prev_persistence_s) {
// lock is held by caller
+ nlohmann::json payload_json = nlohmann::json::array();
+
for (const auto &click_events_collection_it: query_collection_click_events) {
+ auto collection_id = CollectionManager::get_instance().get_collection(
+ click_events_collection_it.first)->get_collection_id();
for (const auto &click_event: click_events_collection_it.second) {
// send http request
- auto collection_id = CollectionManager::get_instance().get_collection(click_events_collection_it.first)->get_collection_id();
nlohmann::json click_event_json;
click_event.to_json(click_event_json);
click_event_json["collection_id"] = std::to_string(collection_id);
+ payload_json.push_back(click_event_json);
+ }
+ }
- const std::string import_payload = click_event_json.dump();
- if(import_payload.empty()) {
- continue;
- }
+ if(payload_json.empty()) {
+ return;
+ }
- std::string leader_url = raft_server->get_leader_url();
- if (!leader_url.empty()) {
- const std::string &base_url = leader_url + "analytics";
- std::string res;
+ const std::string import_payload = payload_json.dump();
- const std::string &update_url = base_url + "/click_events/replicate";
- std::map<std::string, std::string> res_headers;
- long status_code = HttpClient::post_response(update_url, import_payload,
- res, res_headers, {}, 10 * 1000, true);
+ std::string leader_url = raft_server->get_leader_url();
+ if (!leader_url.empty()) {
+ const std::string &base_url = leader_url + "analytics";
+ std::string res;
- if (status_code != 200) {
- LOG(ERROR) << "Error while sending click events to leader. "
- << "Status code: " << status_code << ", response: " << res;
- }
- }
+ const std::string &update_url = base_url + "/click_events/replicate";
+ std::map<std::string, std::string> res_headers;
+ long status_code = HttpClient::post_response(update_url, import_payload,
+ res, res_headers, {}, 10 * 1000, true);
+ if (status_code != 200) {
+ LOG(ERROR) << "Error while sending click events to leader. "
+ << "Status code: " << status_code << ", response: " << res;
+ } else {
query_collection_click_events.clear();
}
}
@@ -381,8 +386,9 @@ void AnalyticsManager::dispose() {
popular_queries.clear();
}
-void AnalyticsManager::init(Store* store) {
+void AnalyticsManager::init(Store* store, Store* analytics_store) {
this->store = store;
+ this->analytics_store = analytics_store;
}
std::unordered_map<std::string, PopularQueries*> AnalyticsManager::get_popular_queries() {
@@ -393,7 +399,7 @@ std::unordered_map<std::string, PopularQueries*> AnalyticsManager::get_popular_q
nlohmann::json AnalyticsManager::get_click_events() {
std::unique_lock lk(mutex);
std::vector<std::string> click_event_jsons;
- store->scan_fill(std::string(CLICK_EVENT) + "_", std::string(CLICK_EVENT) + "`",
+ analytics_store->scan_fill(std::string(CLICK_EVENT) + "_", std::string(CLICK_EVENT) + "`",
click_event_jsons);
nlohmann::json result_json = nlohmann::json::array();
@@ -405,15 +411,16 @@ nlohmann::json AnalyticsManager::get_click_events() {
return result_json;
}
-Option<bool> AnalyticsManager::write_click_event_to_store(nlohmann::json &click_event_json) {
- auto collection_id = click_event_json["collection_id"].get<std::string>();
- auto timestamp = click_event_json["timestamp"].get<uint64_t>();
- const std::string key = std::string(CLICK_EVENT) + "_" + collection_id + "_" +
- std::to_string(timestamp);
- bool inserted = store->insert(key, click_event_json.dump());
- if (!inserted) {
- return Option<bool>(500, "Unable to insert clickevent into store.");
+Option<bool> AnalyticsManager::write_click_event_to_store(nlohmann::json &click_event_jsons) {
+ for(const auto& click_event_json : click_event_jsons) {
+ auto collection_id = click_event_json["collection_id"].get<std::string>();
+ auto timestamp = click_event_json["timestamp"].get<uint64_t>();
+ const std::string key = std::string(CLICK_EVENT) + "_" + collection_id + "_" +
+ std::to_string(timestamp);
+ bool inserted = analytics_store->insert(key, click_event_json.dump());
+ if (!inserted) {
+ return Option<bool>(500, "Unable to insert clickevent into store.");
+ }
}
-
return Option<bool>(true);
}
\ No newline at end of file
diff --git a/src/event_manager.cpp b/src/event_manager.cpp
index ca8f02e7..82b30d87 100644
--- a/src/event_manager.cpp
+++ b/src/event_manager.cpp
@@ -58,22 +58,23 @@ bool EventManager::add_event(const nlohmann::json& event) {
return false;
}
- if (!event_data_val.contains("q") || !event_data_val.contains("doc_id")
+ if (!event_data_val.contains("q") || !event_data_val.contains("doc_id") || !event_data_val.contains("user_id")
|| !event_data_val.contains("position") || !event_data_val.contains("collection")) {
return false;
}
- if (!event_data_val["q"].is_string() || !event_data_val["doc_id"].is_string()
+ if (!event_data_val["q"].is_string() || !event_data_val["doc_id"].is_string() || !event_data_val["user_id"].is_string()
|| !event_data_val["position"].is_number_unsigned() || !event_data_val["collection"].is_string()) {
return false;
}
const std::string query = event_data_val["q"].get<std::string>();
+ const std::string user_id = event_data_val["user_id"].get<std::string>();
const std::string doc_id = event_data_val["doc_id"].get<std::string>();
uint64_t position = event_data_val["position"].get<uint64_t>();
const std::string& collection = event_data_val["collection"].get<std::string>();
- AnalyticsManager::get_instance().add_click_event(collection, query, doc_id, position);
+ AnalyticsManager::get_instance().add_click_event(collection, query, user_id, doc_id, position);
}
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 97cfbd00..9d6c343e 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -374,6 +374,7 @@ int run_server(const Config & config, const std::string & version, void (*master
std::string db_dir = config.get_data_dir() + "/db";
std::string state_dir = config.get_data_dir() + "/state";
std::string meta_dir = config.get_data_dir() + "/meta";
+ std::string analytics_dir = config.get_data_dir() + "/analytics";
size_t thread_pool_size = config.get_thread_pool_size();
@@ -395,11 +396,14 @@ int run_server(const Config & config, const std::string & version, void (*master
// meta DB for storing house keeping things
Store meta_store(meta_dir, 24*60*60, 1024, false);
+ //analytics DB for storing query click events
+ Store analytics_store(analytics_dir, 24*60*60, 1024, false);
+
curl_global_init(CURL_GLOBAL_SSL);
HttpClient & httpClient = HttpClient::get_instance();
httpClient.init(config.get_api_key());
- AnalyticsManager::get_instance().init(&store);
+ AnalyticsManager::get_instance().init(&store, &analytics_store);
server = new HttpServer(
version,