Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
query hits count analytics
krunala6b404e7ffac
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 02a94bdd..bd9b9b74 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -7,18 +7,18 @@#include <unordered_map>#include <shared_mutex>-struct ClickEvent {+struct click_event_t {std::string query;uint64_t timestamp;std::string user_id;std::string doc_id;uint64_t position;- ClickEvent() = delete;+ click_event_t() = delete;- ~ClickEvent() = default;+ ~click_event_t() = default;- ClickEvent(std::string q, uint64_t ts, std::string uid, std::string id, uint64_t pos) {+ click_event_t(std::string q, uint64_t ts, std::string uid, std::string id, uint64_t pos) {query = q;timestamp = ts;user_id = uid;@@ -26,7 +26,7 @@ struct ClickEvent {position = pos;}- ClickEvent& operator=(ClickEvent& other) {+ click_event_t& operator=(click_event_t& other) {if (this != &other) {query = other.query;timestamp = other.timestamp;@@ -46,6 +46,47 @@ struct ClickEvent {}};+struct query_hits_count_t {+ std::string query;+ uint64_t timestamp;+ std::string user_id;+ uint64_t hits_count;++ query_hits_count_t() = delete;++ ~query_hits_count_t() = default;++ query_hits_count_t(std::string q, uint64_t ts, std::string uid, uint64_t count) {+ query = q;+ timestamp = ts;+ user_id = uid;+ hits_count = count;+ }++ query_hits_count_t &operator=(query_hits_count_t &other) {+ if (this != &other) {+ query = other.query;+ timestamp = other.timestamp;+ user_id = other.user_id;+ hits_count = other.hits_count;+ return *this;+ }+ }++ void to_json(nlohmann::json &obj) const {+ obj["query"] = query;+ obj["timestamp"] = timestamp;+ obj["user_id"] = user_id;+ obj["hits_count"] = hits_count;+ }+};++struct query_hits_count_comp {+ bool operator()(const query_hits_count_t& a, const query_hits_count_t& b) const {+ return a.query < b.query;+ }+};+struct event_cache_t {uint64_t last_update_time;uint64_t count;@@ -97,7 +138,10 @@ private:std::unordered_map<std::string, QueryAnalytics*> nohits_queries;//query collection => click events- std::unordered_map<std::string, std::vector<ClickEvent>> query_collection_click_events;+ std::unordered_map<std::string, std::vector<click_event_t>> query_collection_click_events;++ //query collection => query hits count+ std::unordered_map<std::string, std::set<query_hits_count_t, query_hits_count_comp>> query_collection_hits_count;Store* store = nullptr;Store* analytics_store = nullptr;@@ -116,6 +160,7 @@ public:static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";static constexpr const char* CLICK_EVENT = "$CE";+ static constexpr const char* QUERY_HITS_COUNT = "$QH";static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";static constexpr const char* NOHITS_QUERIES_TYPE = "nohits_queries";@@ -165,4 +210,13 @@ public:std::unordered_map<std::string, QueryAnalytics*> get_nohits_queries();void resetRateLimit();++ void add_query_hits_count(const std::string& query_collection, const std::string& query, const std::string& user_id,+ uint64_t hits_count);++ void persist_query_hits_counts(ReplicationState *raft_server, uint64_t prev_persistence_s);++ nlohmann::json get_query_hits_counts();++ Option<bool> write_query_hits_counts_to_store(nlohmann::json& query_hits_counts_json);};diff --git a/include/core_api.h b/include/core_api.hindex 00f633ba..55186153 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -173,6 +173,10 @@ bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptbool post_replicate_click_event(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool get_query_hits_counts(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);++bool post_replicate_query_hits_counts(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+// Misc helpersvoid get_collections_for_auth(std::map<std::string, std::string>& req_params, const std::string& body,diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex 3b88e1be..351a89ca 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -273,7 +273,7 @@ Option<bool> AnalyticsManager::add_click_event(const std::string &query_collectiauto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();- ClickEvent click_event(query, now_ts_useconds, user_id, doc_id, position);+ click_event_t click_event(query, now_ts_useconds, user_id, doc_id, position);click_events_vec.emplace_back(click_event);return Option<bool>(true);@@ -298,6 +298,25 @@ void AnalyticsManager::add_nohits_query(const std::string &query_collection, con}}+void AnalyticsManager::add_query_hits_count(const std::string &query_collection, const std::string &query,+ const std::string &user_id, uint64_t hits_count) {+ std::unique_lock lock(mutex);+ if(analytics_store) {+ auto now_ts_useconds = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();++ auto &query_hits_count_set = query_collection_hits_count[query_collection];+ query_hits_count_t queryHitsCount(query, now_ts_useconds, user_id, hits_count);+ auto query_hits_count_set_it = query_hits_count_set.find(queryHitsCount);++ if(query_hits_count_set_it != query_hits_count_set.end()) {+ query_hits_count_set.erase(query_hits_count_set_it);+ }++ query_hits_count_set.emplace(queryHitsCount);+ }+}+void AnalyticsManager::run(ReplicationState* raft_server) {uint64_t prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();@@ -324,6 +343,8 @@ void AnalyticsManager::run(ReplicationState* raft_server) {persist_query_events(raft_server, prev_persistence_s);persist_click_events(raft_server, prev_persistence_s);+ persist_query_hits_counts(raft_server, prev_persistence_s);+prev_persistence_s = std::chrono::duration_cast<std::chrono::seconds>(std::chrono::system_clock::now().time_since_epoch()).count();@@ -458,6 +479,46 @@ void AnalyticsManager::persist_click_events(ReplicationState *raft_server, uint6}}+void AnalyticsManager::persist_query_hits_counts(ReplicationState *raft_server, uint64_t prev_persistence_s) {+ nlohmann::json payload_json = nlohmann::json::array();++ for (const auto &query_collection_hits_count_it: query_collection_hits_count) {+ auto collection_id = CollectionManager::get_instance().get_collection(+ query_collection_hits_count_it.first)->get_collection_id();+ for (const auto &query_hits_count: query_collection_hits_count_it.second) {+ // send http request+ nlohmann::json query_hits_count_json;+ query_hits_count.to_json(query_hits_count_json);+ query_hits_count_json["collection_id"] = std::to_string(collection_id);+ payload_json.push_back(query_hits_count_json);+ }+ }++ if (payload_json.empty()) {+ return;+ }++ const std::string import_payload = payload_json.dump();++ std::string leader_url = raft_server->get_leader_url();+ if (!leader_url.empty()) {+ const std::string &base_url = leader_url + "analytics";+ std::string res;++ const std::string &update_url = base_url + "/query_hits_counts/replicate";+ std::map<std::string, std::string> res_headers;+ long status_code = HttpClient::post_response(update_url, import_payload,+ res, res_headers, {}, 10 * 1000, true);++ if (status_code != 200) {+ LOG(ERROR) << "Error while sending click events to leader. "+ << "Status code: " << status_code << ", response: " << res;+ } else {+ query_collection_hits_count.clear();+ }+ }+}+void AnalyticsManager::stop() {quit = true;cv.notify_all();@@ -512,6 +573,24 @@ nlohmann::json AnalyticsManager::get_click_events() {return result_json;}+nlohmann::json AnalyticsManager::get_query_hits_counts() {+ std::unique_lock lk(mutex);+ std::vector<std::string> query_hits_counts_jsons;+ nlohmann::json result_json = nlohmann::json::array();++ if (analytics_store) {+ analytics_store->scan_fill(std::string(QUERY_HITS_COUNT) + "_", std::string(QUERY_HITS_COUNT) + "`",+ query_hits_counts_jsons);++ for (const auto &query_hits_count_json: query_hits_counts_jsons) {+ nlohmann::json query_hits_count = nlohmann::json::parse(query_hits_count_json);+ result_json.push_back(query_hits_count);+ }+ }++ return result_json;+}+Option<bool> AnalyticsManager::write_click_event_to_store(nlohmann::json &click_event_jsons) {for(const auto& click_event_json : click_event_jsons) {auto collection_id = click_event_json["collection_id"].get<std::string>();@@ -530,6 +609,24 @@ Option<bool> AnalyticsManager::write_click_event_to_store(nlohmann::json &click_return Option<bool>(true);}+Option<bool> AnalyticsManager::write_query_hits_counts_to_store(nlohmann::json &query_hits_counts_json) {+ for(const auto& query_hits_count : query_hits_counts_json) {+ auto collection_id = query_hits_count["collection_id"].get<std::string>();+ auto timestamp = query_hits_count["timestamp"].get<uint64_t>();+ const std::string key = std::string(QUERY_HITS_COUNT) + "_" + collection_id + "_" ++ std::to_string(timestamp);+ if(analytics_store) {+ bool inserted = analytics_store->insert(key, query_hits_count.dump());+ if (!inserted) {+ return Option<bool>(500, "Unable to insert query hits count into store.");+ }+ } else {+ return Option<bool>(500, "Analytics DB not initialized.");+ }+ }+ return Option<bool>(true);+}+void AnalyticsManager::resetRateLimit() {events_cache.clear();}\ No newline at end of filediff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex ca6fbeb0..0fcb0f52 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1534,6 +1534,9 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& restd::string analytics_query = Tokenizer::normalize_ascii_no_spaces(raw_query);AnalyticsManager::get_instance().add_suggestion(orig_coll_name, analytics_query,true, req_params["x-typesense-user-id"]);+ AnalyticsManager::get_instance().add_query_hits_count(orig_coll_name, analytics_query,+ req_params["x-typesense-user-id"],+ result["found"].get<size_t>());} else if(result.contains("found") == 0 && result["found"].get<size_t>() == 0) {std::string analytics_query = Tokenizer::normalize_ascii_no_spaces(raw_query);AnalyticsManager::get_instance().add_nohits_query(orig_coll_name, analytics_query,diff --git a/src/core_api.cpp b/src/core_api.cppindex e3afb1d2..8ca74f10 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -2823,6 +2823,34 @@ bool post_replicate_click_event(const std::shared_ptr<http_req>& req, const std:return false;}- res->set_200("ClickEvent wrote to DB.");+ res->set_200("click_event_t wrote to DB.");+ return true;+}++bool get_query_hits_counts(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ auto query_hits_counts = AnalyticsManager::get_instance().get_query_hits_counts();++ res->set_200(query_hits_counts.dump());+ return true;+}++bool post_replicate_query_hits_counts(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ nlohmann::json req_json;++ try {+ req_json = nlohmann::json::parse(req->body);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ res->set_400("Bad JSON.");+ return false;+ }++ auto op = AnalyticsManager::get_instance().write_query_hits_counts_to_store(req_json);+ if(!op.ok()) {+ res->set_body(op.code(), op.error());+ return false;+ }++ res->set_200("query hits counts wrote to DB.");return true;}\ No newline at end of filediff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex f9a1c3af..fc475d71 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -85,6 +85,8 @@ void master_server_routes() {server->get("/analytics/click_events", get_click_events);server->post("/analytics/click_events", post_create_event);server->post("/analytics/click_events/replicate", post_replicate_click_event);+ server->get("/analytics/query_hits_counts", get_query_hits_counts);+ server->post("/analytics/query_hits_counts/replicate", post_replicate_query_hits_counts);// metaserver->get("/metrics.json", get_metrics_json);diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex 4eea0a4f..e0181d7f 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -440,4 +440,59 @@ TEST_F(AnalyticsManagerTest, NoresultsQueries) {noresults_queries = analyticsManager.get_nohits_queries();ASSERT_EQ(0, noresults_queries.size());+}++TEST_F(AnalyticsManagerTest, QueryHitsCount) {+ nlohmann::json titles_schema = R"({+ "name": "titles",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection *titles_coll = collectionManager.create_collection(titles_schema).get();++ nlohmann::json doc;+ doc["title"] = "Cool trousers";+ ASSERT_TRUE(titles_coll->add(doc.dump()).ok());++ doc["title"] = "Cool pants";+ ASSERT_TRUE(titles_coll->add(doc.dump()).ok());++ doc["title"] = "Trendy sneakers";+ ASSERT_TRUE(titles_coll->add(doc.dump()).ok());++ doc["title"] = "Funky shorts";+ ASSERT_TRUE(titles_coll->add(doc.dump()).ok());++ nlohmann::json query_hits_array = nlohmann::json::array();+ nlohmann::json obj;++ obj["collection_id"] = "0";+ obj["query"] = "cool";+ obj["timestamp"] = 1625365612;+ obj["user_id"] = "1";+ obj["hits_count"] = 2;+ query_hits_array.push_back(obj);++ obj["query"] = "funky";+ obj["timestamp"] = 1625365616;+ obj["user_id"] = "1";+ obj["hits_count"] = 1;+ query_hits_array.push_back(obj);+++ auto op = analyticsManager.write_query_hits_counts_to_store(query_hits_array);+ ASSERT_TRUE(op.ok());++ auto result = analyticsManager.get_query_hits_counts();++ ASSERT_EQ(2, result.size());+ ASSERT_EQ("cool", result[0]["query"]);+ ASSERT_EQ(2, result[0]["hits_count"]);+ ASSERT_EQ(1625365612, result[0]["timestamp"]);++ ASSERT_EQ("funky", result[1]["query"]);+ ASSERT_EQ(1, result[1]["hits_count"]);+ ASSERT_EQ(1625365616, result[1]["timestamp"]);}\ No newline at end of file