Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
refactor replication and minor updates
krunalc51f44aacf3a
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 6fab4230..4c9deae1 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -10,17 +10,17 @@struct ClickEvent {std::string query;uint64_t timestamp;- uint64_t product_id;+ std::string doc_id;uint64_t position;ClickEvent() = delete;~ClickEvent() = default;- ClickEvent(std::string q, uint64_t ts, uint64_t pid, uint64_t pos) {+ ClickEvent(std::string q, uint64_t ts, std::string id, uint64_t pos) {query = q;timestamp = ts;- product_id = pid;+ doc_id = id;position = pos;}@@ -28,7 +28,7 @@ struct ClickEvent {if (this != &other) {query = other.query;timestamp = other.timestamp;- product_id = other.product_id;+ doc_id = other.doc_id;position = other.position;return *this;}@@ -37,13 +37,9 @@ struct ClickEvent {void to_json(nlohmann::json& obj) const {obj["query"] = query;obj["timestamp"] = timestamp;- obj["product_id"] = product_id;+ obj["doc_id"] = doc_id;obj["position"] = position;}-- bool operator < (const ClickEvent& rhs) const {- return this->timestamp < rhs.timestamp;- }};class AnalyticsManager {@@ -81,7 +77,7 @@ private:std::unordered_map<std::string, PopularQueries*> popular_queries;//query collection => click events- std::unordered_map<std::string, std::set<ClickEvent>> query_collection_click_events;+ std::unordered_map<std::string, std::vector<ClickEvent>> query_collection_click_events;Store* store = nullptr;@@ -133,9 +129,11 @@ public:std::unordered_map<std::string, PopularQueries*> get_popular_queries();void add_click_event(const std::string& query_collection, const std::string& query,- uint64_t product_id, uint64_t position);+ std::string doc_id, uint64_t position);void persist_click_event(ReplicationState *raft_server, uint64_t prev_persistence_s);- std::set<ClickEvent> get_click_events(const std::string& name);+ nlohmann::json get_click_events();++ Option<bool> write_click_event_to_store(nlohmann::json& click_event_json);};diff --git a/include/core_api.h b/include/core_api.hindex 17c7a802..d2048cf9 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -171,6 +171,8 @@ bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::sharedbool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool post_replicate_click_event(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+// Misc helpersvoid get_collections_for_auth(std::map<std::string, std::string>& req_params, const std::string& body,diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex fa9c2479..02f364d9 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -219,25 +219,15 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection, const}void AnalyticsManager::add_click_event(const std::string &query_collection, const std::string &query,- uint64_t product_id, uint64_t position) {+ std::string doc_id, uint64_t position) {std::unique_lock lock(mutex);- auto &click_events_set = query_collection_click_events[query_collection];+ auto &click_events_vec = query_collection_click_events[query_collection];auto now_ts_seconds = std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::system_clock::now().time_since_epoch()).count();- ClickEvent click_event(query, now_ts_seconds, product_id, position);- click_events_set.insert(click_event);-- if(store) {- const std::string key = std::string(CLICK_EVENT) + "_" + query_collection + "_" + std::to_string(now_ts_seconds);- nlohmann::json click_event_json;- click_event.to_json(click_event_json);- bool inserted = store->insert(key, click_event_json.dump());- if (!inserted) {- LOG(ERROR) << "Unable to insert clickevent into store.";- }- }+ ClickEvent click_event(query, now_ts_seconds, doc_id, position);+ click_events_vec.emplace_back(click_event);}void AnalyticsManager::run(ReplicationState* raft_server) {@@ -345,8 +335,10 @@ void AnalyticsManager::persist_click_event(ReplicationState *raft_server, uint64for (const auto &click_events_collection_it: query_collection_click_events) {for (const auto &click_event: click_events_collection_it.second) {// send http request+ auto collection_id = CollectionManager::get_instance().get_collection(click_events_collection_it.first)->get_collection_id();nlohmann::json click_event_json;click_event.to_json(click_event_json);+ click_event_json["collection_id"] = std::to_string(collection_id);const std::string import_payload = click_event_json.dump();if(import_payload.empty()) {@@ -355,19 +347,21 @@ void AnalyticsManager::persist_click_event(ReplicationState *raft_server, uint64std::string leader_url = raft_server->get_leader_url();if (!leader_url.empty()) {- const std::string &base_url = leader_url + "collections/" + click_events_collection_it.first;+ const std::string &base_url = leader_url + "analytics";std::string res;- const std::string &update_url = base_url + "/clickevents";+ const std::string &update_url = base_url + "/click_events/replicate";std::map<std::string, std::string> res_headers;long status_code = HttpClient::post_response(update_url, import_payload,res, res_headers, {}, 10 * 1000, true);if (status_code != 200) {- LOG(ERROR) << "Error while sending query suggestions events to leader. "+ LOG(ERROR) << "Error while sending click events to leader. "<< "Status code: " << status_code << ", response: " << res;}}++ query_collection_click_events.clear();}}}@@ -396,11 +390,30 @@ std::unordered_map<std::string, PopularQueries*> AnalyticsManager::get_popular_qreturn popular_queries;}-std::set<ClickEvent> AnalyticsManager::get_click_events(const std::string& coll) {+nlohmann::json AnalyticsManager::get_click_events() {std::unique_lock lk(mutex);- const auto it = query_collection_click_events.find(coll);- if(it == query_collection_click_events.end()) {- return {};+ std::vector<std::string> click_event_jsons;+ store->scan_fill(std::string(CLICK_EVENT) + "_", std::string(CLICK_EVENT) + "`",+ click_event_jsons);++ nlohmann::json result_json = nlohmann::json::array();+ for(const auto & click_event_json: click_event_jsons) {+ nlohmann::json click_event = nlohmann::json::parse(click_event_json);+ result_json.push_back(click_event);+ }++ return result_json;+}++Option<bool> AnalyticsManager::write_click_event_to_store(nlohmann::json &click_event_json) {+ auto collection_id = click_event_json["collection_id"].get<std::string>();+ auto timestamp = click_event_json["timestamp"].get<uint64_t>();+ const std::string key = std::string(CLICK_EVENT) + "_" + collection_id + "_" ++ std::to_string(timestamp);+ bool inserted = store->insert(key, click_event_json.dump());+ if (!inserted) {+ return Option<bool>(500, "Unable to insert clickevent into store.");}- return it->second;++ return Option<bool>(true);}\ No newline at end of filediff --git a/src/core_api.cpp b/src/core_api.cppindex e06127ce..55dcdea4 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -2381,21 +2381,29 @@ bool post_proxy(const std::shared_ptr<http_req>& req, const std::shared_ptr<http}bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {- auto click_events = AnalyticsManager::get_instance().get_click_events(req->params["collection"]);+ auto click_events = AnalyticsManager::get_instance().get_click_events();- if(click_events.empty()) {+ res->set_200(click_events.dump());+ return true;+}++bool post_replicate_click_event(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ nlohmann::json req_json;++ try {+ req_json = nlohmann::json::parse(req->body);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ res->set_400("Bad JSON.");return false;}-- nlohmann::json res_json, doc;- res_json["name"] = req->params["collection"];- res_json["click_events"] = nlohmann::json::array();- for(const auto& click_event : click_events) {- click_event.to_json(doc);- res_json["click_events"].push_back(doc);+ auto op = AnalyticsManager::get_instance().write_click_event_to_store(req_json);+ if(!op.ok()) {+ res->set_body(op.code(), op.error());+ return false;}- res->set_200(res_json.dump());+ res->set_200("ClickEvent wrote to DB.");return true;}\ No newline at end of filediff --git a/src/event_manager.cpp b/src/event_manager.cppindex 7f709e5e..ca8f02e7 100644--- a/src/event_manager.cpp+++ b/src/event_manager.cpp@@ -58,22 +58,22 @@ bool EventManager::add_event(const nlohmann::json& event) {return false;}- if (!event_data_val.contains("q") || !event_data_val.contains("product_id")+ if (!event_data_val.contains("q") || !event_data_val.contains("doc_id")|| !event_data_val.contains("position") || !event_data_val.contains("collection")) {return false;}- if (!event_data_val["q"].is_string() || !event_data_val["product_id"].is_number_unsigned()+ if (!event_data_val["q"].is_string() || !event_data_val["doc_id"].is_string()|| !event_data_val["position"].is_number_unsigned() || !event_data_val["collection"].is_string()) {return false;}const std::string query = event_data_val["q"].get<std::string>();- uint64_t product_id = event_data_val["product_id"].get<uint64_t>();+ const std::string doc_id = event_data_val["doc_id"].get<std::string>();uint64_t position = event_data_val["position"].get<uint64_t>();const std::string& collection = event_data_val["collection"].get<std::string>();- AnalyticsManager::get_instance().add_click_event(collection, query, product_id, position);+ AnalyticsManager::get_instance().add_click_event(collection, query, doc_id, position);}}diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex f08b404e..6cbc5d5a 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -82,8 +82,9 @@ void master_server_routes() {server->post("/analytics/events", post_create_event);//collection based query click events- server->get("/collections/:collection/click_events", get_click_events);- server->post("/collections/:collection/click_events", post_create_event);+ server->get("/analytics/click_events", get_click_events);+ server->post("/analytics/click_events", post_create_event);+ server->post("/analytics/click_events/replicate", post_replicate_click_event);// metaserver->get("/metrics.json", get_metrics_json);