Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
adding field user_id and creating new store for analytics
krunale26c78219047
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 4c9deae1..36efbd11 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -10,6 +10,7 @@struct ClickEvent {std::string query;uint64_t timestamp;+ std::string user_id;std::string doc_id;uint64_t position;@@ -17,9 +18,10 @@ struct ClickEvent {~ClickEvent() = default;- ClickEvent(std::string q, uint64_t ts, std::string id, uint64_t pos) {+ ClickEvent(std::string q, uint64_t ts, std::string uid, std::string id, uint64_t pos) {query = q;timestamp = ts;+ user_id = uid;doc_id = id;position = pos;}@@ -28,6 +30,7 @@ struct ClickEvent {if (this != &other) {query = other.query;timestamp = other.timestamp;+ user_id = other.user_id;doc_id = other.doc_id;position = other.position;return *this;@@ -37,6 +40,7 @@ struct ClickEvent {void to_json(nlohmann::json& obj) const {obj["query"] = query;obj["timestamp"] = timestamp;+ obj["user_id"] = user_id;obj["doc_id"] = doc_id;obj["position"] = position;}@@ -80,6 +84,7 @@ private:std::unordered_map<std::string, std::vector<ClickEvent>> query_collection_click_events;Store* store = nullptr;+ Store* analytics_store = nullptr;AnalyticsManager() {}@@ -105,7 +110,7 @@ public:AnalyticsManager(AnalyticsManager const&) = delete;void operator=(AnalyticsManager const&) = delete;- void init(Store* store);+ void init(Store* store, Store* analytics_store);void run(ReplicationState* raft_server);@@ -128,7 +133,7 @@ public:std::unordered_map<std::string, PopularQueries*> get_popular_queries();- void add_click_event(const std::string& query_collection, const std::string& query,+ void add_click_event(const std::string& query_collection, const std::string& query, const std::string& user_id,std::string doc_id, uint64_t position);void persist_click_event(ReplicationState *raft_server, uint64_t prev_persistence_s);diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex 02f364d9..d1539812 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -218,7 +218,7 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection, const}}-void AnalyticsManager::add_click_event(const std::string &query_collection, const std::string &query,+void AnalyticsManager::add_click_event(const std::string &query_collection, const std::string &query, const std::string &user_id,std::string doc_id, uint64_t position) {std::unique_lock lock(mutex);auto &click_events_vec = query_collection_click_events[query_collection];@@ -226,7 +226,7 @@ void AnalyticsManager::add_click_event(const std::string &query_collection, consauto now_ts_seconds = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();- ClickEvent click_event(query, now_ts_seconds, doc_id, position);+ ClickEvent click_event(query, now_ts_seconds, user_id, doc_id, position);click_events_vec.emplace_back(click_event);}@@ -332,35 +332,40 @@ void AnalyticsManager::persist_suggestions(ReplicationState *raft_server, uint64void AnalyticsManager::persist_click_event(ReplicationState *raft_server, uint64_t prev_persistence_s) {// lock is held by caller+ nlohmann::json payload_json = nlohmann::json::array();+for (const auto &click_events_collection_it: query_collection_click_events) {+ auto collection_id = CollectionManager::get_instance().get_collection(+ click_events_collection_it.first)->get_collection_id();for (const auto &click_event: click_events_collection_it.second) {// send http request- auto collection_id = CollectionManager::get_instance().get_collection(click_events_collection_it.first)->get_collection_id();nlohmann::json click_event_json;click_event.to_json(click_event_json);click_event_json["collection_id"] = std::to_string(collection_id);+ payload_json.push_back(click_event_json);+ }+ }- const std::string import_payload = click_event_json.dump();- if(import_payload.empty()) {- continue;- }+ if(payload_json.empty()) {+ return;+ }- std::string leader_url = raft_server->get_leader_url();- if (!leader_url.empty()) {- const std::string &base_url = leader_url + "analytics";- std::string res;+ const std::string import_payload = payload_json.dump();- const std::string &update_url = base_url + "/click_events/replicate";- std::map<std::string, std::string> res_headers;- long status_code = HttpClient::post_response(update_url, import_payload,- res, res_headers, {}, 10 * 1000, true);+ std::string leader_url = raft_server->get_leader_url();+ if (!leader_url.empty()) {+ const std::string &base_url = leader_url + "analytics";+ std::string res;- if (status_code != 200) {- LOG(ERROR) << "Error while sending click events to leader. "- << "Status code: " << status_code << ", response: " << res;- }- }+ const std::string &update_url = base_url + "/click_events/replicate";+ std::map<std::string, std::string> res_headers;+ long status_code = HttpClient::post_response(update_url, import_payload,+ res, res_headers, {}, 10 * 1000, true);+ if (status_code != 200) {+ LOG(ERROR) << "Error while sending click events to leader. "+ << "Status code: " << status_code << ", response: " << res;+ } else {query_collection_click_events.clear();}}@@ -381,8 +386,9 @@ void AnalyticsManager::dispose() {popular_queries.clear();}-void AnalyticsManager::init(Store* store) {+void AnalyticsManager::init(Store* store, Store* analytics_store) {this->store = store;+ this->analytics_store = analytics_store;}std::unordered_map<std::string, PopularQueries*> AnalyticsManager::get_popular_queries() {@@ -393,7 +399,7 @@ std::unordered_map<std::string, PopularQueries*> AnalyticsManager::get_popular_qnlohmann::json AnalyticsManager::get_click_events() {std::unique_lock lk(mutex);std::vector<std::string> click_event_jsons;- store->scan_fill(std::string(CLICK_EVENT) + "_", std::string(CLICK_EVENT) + "`",+ analytics_store->scan_fill(std::string(CLICK_EVENT) + "_", std::string(CLICK_EVENT) + "`",click_event_jsons);nlohmann::json result_json = nlohmann::json::array();@@ -405,15 +411,16 @@ nlohmann::json AnalyticsManager::get_click_events() {return result_json;}-Option<bool> AnalyticsManager::write_click_event_to_store(nlohmann::json &click_event_json) {- auto collection_id = click_event_json["collection_id"].get<std::string>();- auto timestamp = click_event_json["timestamp"].get<uint64_t>();- const std::string key = std::string(CLICK_EVENT) + "_" + collection_id + "_" +- std::to_string(timestamp);- bool inserted = store->insert(key, click_event_json.dump());- if (!inserted) {- return Option<bool>(500, "Unable to insert clickevent into store.");+Option<bool> AnalyticsManager::write_click_event_to_store(nlohmann::json &click_event_jsons) {+ for(const auto& click_event_json : click_event_jsons) {+ auto collection_id = click_event_json["collection_id"].get<std::string>();+ auto timestamp = click_event_json["timestamp"].get<uint64_t>();+ const std::string key = std::string(CLICK_EVENT) + "_" + collection_id + "_" ++ std::to_string(timestamp);+ bool inserted = analytics_store->insert(key, click_event_json.dump());+ if (!inserted) {+ return Option<bool>(500, "Unable to insert clickevent into store.");+ }}-return Option<bool>(true);}\ No newline at end of filediff --git a/src/event_manager.cpp b/src/event_manager.cppindex ca8f02e7..82b30d87 100644--- a/src/event_manager.cpp+++ b/src/event_manager.cpp@@ -58,22 +58,23 @@ bool EventManager::add_event(const nlohmann::json& event) {return false;}- if (!event_data_val.contains("q") || !event_data_val.contains("doc_id")+ if (!event_data_val.contains("q") || !event_data_val.contains("doc_id") || !event_data_val.contains("user_id")|| !event_data_val.contains("position") || !event_data_val.contains("collection")) {return false;}- if (!event_data_val["q"].is_string() || !event_data_val["doc_id"].is_string()+ if (!event_data_val["q"].is_string() || !event_data_val["doc_id"].is_string() || !event_data_val["user_id"].is_string()|| !event_data_val["position"].is_number_unsigned() || !event_data_val["collection"].is_string()) {return false;}const std::string query = event_data_val["q"].get<std::string>();+ const std::string user_id = event_data_val["user_id"].get<std::string>();const std::string doc_id = event_data_val["doc_id"].get<std::string>();uint64_t position = event_data_val["position"].get<uint64_t>();const std::string& collection = event_data_val["collection"].get<std::string>();- AnalyticsManager::get_instance().add_click_event(collection, query, doc_id, position);+ AnalyticsManager::get_instance().add_click_event(collection, query, user_id, doc_id, position);}}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 97cfbd00..9d6c343e 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -374,6 +374,7 @@ int run_server(const Config & config, const std::string & version, void (*masterstd::string db_dir = config.get_data_dir() + "/db";std::string state_dir = config.get_data_dir() + "/state";std::string meta_dir = config.get_data_dir() + "/meta";+ std::string analytics_dir = config.get_data_dir() + "/analytics";size_t thread_pool_size = config.get_thread_pool_size();@@ -395,11 +396,14 @@ int run_server(const Config & config, const std::string & version, void (*master// meta DB for storing house keeping thingsStore meta_store(meta_dir, 24*60*60, 1024, false);+ //analytics DB for storing query click events+ Store analytics_store(analytics_dir, 24*60*60, 1024, false);+curl_global_init(CURL_GLOBAL_SSL);HttpClient & httpClient = HttpClient::get_instance();httpClient.init(config.get_api_key());- AnalyticsManager::get_instance().init(&store);+ AnalyticsManager::get_instance().init(&store, &analytics_store);server = new HttpServer(version,