Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

refactor replication and minor updates

krunalc51f44aacf3a
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index 6fab4230..4c9deae1 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -10,17 +10,17 @@
struct ClickEvent {
std::string query;
uint64_t timestamp;
- uint64_t product_id;
+ std::string doc_id;
uint64_t position;
ClickEvent() = delete;
~ClickEvent() = default;
- ClickEvent(std::string q, uint64_t ts, uint64_t pid, uint64_t pos) {
+ ClickEvent(std::string q, uint64_t ts, std::string id, uint64_t pos) {
query = q;
timestamp = ts;
- product_id = pid;
+ doc_id = id;
position = pos;
}
@@ -28,7 +28,7 @@ struct ClickEvent {
if (this != &other) {
query = other.query;
timestamp = other.timestamp;
- product_id = other.product_id;
+ doc_id = other.doc_id;
position = other.position;
return *this;
}
@@ -37,13 +37,9 @@ struct ClickEvent {
void to_json(nlohmann::json& obj) const {
obj["query"] = query;
obj["timestamp"] = timestamp;
- obj["product_id"] = product_id;
+ obj["doc_id"] = doc_id;
obj["position"] = position;
}
-
- bool operator < (const ClickEvent& rhs) const {
- return this->timestamp < rhs.timestamp;
- }
};
class AnalyticsManager {
@@ -81,7 +77,7 @@ private:
std::unordered_map<std::string, PopularQueries*> popular_queries;
//query collection => click events
- std::unordered_map<std::string, std::set<ClickEvent>> query_collection_click_events;
+ std::unordered_map<std::string, std::vector<ClickEvent>> query_collection_click_events;
Store* store = nullptr;
@@ -133,9 +129,11 @@ public:
std::unordered_map<std::string, PopularQueries*> get_popular_queries();
void add_click_event(const std::string& query_collection, const std::string& query,
- uint64_t product_id, uint64_t position);
+ std::string doc_id, uint64_t position);
void persist_click_event(ReplicationState *raft_server, uint64_t prev_persistence_s);
- std::set<ClickEvent> get_click_events(const std::string& name);
+ nlohmann::json get_click_events();
+
+ Option<bool> write_click_event_to_store(nlohmann::json& click_event_json);
};
diff --git a/include/core_api.h b/include/core_api.h
index 17c7a802..d2048cf9 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -171,6 +171,8 @@ bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared
bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+bool post_replicate_click_event(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+
// Misc helpers
void get_collections_for_auth(std::map<std::string, std::string>& req_params, const std::string& body,
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index fa9c2479..02f364d9 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -219,25 +219,15 @@ void AnalyticsManager::add_suggestion(const std::string &query_collection, const
}
void AnalyticsManager::add_click_event(const std::string &query_collection, const std::string &query,
- uint64_t product_id, uint64_t position) {
+ std::string doc_id, uint64_t position) {
std::unique_lock lock(mutex);
- auto &click_events_set = query_collection_click_events[query_collection];
+ auto &click_events_vec = query_collection_click_events[query_collection];
auto now_ts_seconds = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::system_clock::now().time_since_epoch()).count();
- ClickEvent click_event(query, now_ts_seconds, product_id, position);
- click_events_set.insert(click_event);
-
- if(store) {
- const std::string key = std::string(CLICK_EVENT) + "_" + query_collection + "_" + std::to_string(now_ts_seconds);
- nlohmann::json click_event_json;
- click_event.to_json(click_event_json);
- bool inserted = store->insert(key, click_event_json.dump());
- if (!inserted) {
- LOG(ERROR) << "Unable to insert clickevent into store.";
- }
- }
+ ClickEvent click_event(query, now_ts_seconds, doc_id, position);
+ click_events_vec.emplace_back(click_event);
}
void AnalyticsManager::run(ReplicationState* raft_server) {
@@ -345,8 +335,10 @@ void AnalyticsManager::persist_click_event(ReplicationState *raft_server, uint64
for (const auto &click_events_collection_it: query_collection_click_events) {
for (const auto &click_event: click_events_collection_it.second) {
// send http request
+ auto collection_id = CollectionManager::get_instance().get_collection(click_events_collection_it.first)->get_collection_id();
nlohmann::json click_event_json;
click_event.to_json(click_event_json);
+ click_event_json["collection_id"] = std::to_string(collection_id);
const std::string import_payload = click_event_json.dump();
if(import_payload.empty()) {
@@ -355,19 +347,21 @@ void AnalyticsManager::persist_click_event(ReplicationState *raft_server, uint64
std::string leader_url = raft_server->get_leader_url();
if (!leader_url.empty()) {
- const std::string &base_url = leader_url + "collections/" + click_events_collection_it.first;
+ const std::string &base_url = leader_url + "analytics";
std::string res;
- const std::string &update_url = base_url + "/clickevents";
+ const std::string &update_url = base_url + "/click_events/replicate";
std::map<std::string, std::string> res_headers;
long status_code = HttpClient::post_response(update_url, import_payload,
res, res_headers, {}, 10 * 1000, true);
if (status_code != 200) {
- LOG(ERROR) << "Error while sending query suggestions events to leader. "
+ LOG(ERROR) << "Error while sending click events to leader. "
<< "Status code: " << status_code << ", response: " << res;
}
}
+
+ query_collection_click_events.clear();
}
}
}
@@ -396,11 +390,30 @@ std::unordered_map<std::string, PopularQueries*> AnalyticsManager::get_popular_q
return popular_queries;
}
-std::set<ClickEvent> AnalyticsManager::get_click_events(const std::string& coll) {
+nlohmann::json AnalyticsManager::get_click_events() {
std::unique_lock lk(mutex);
- const auto it = query_collection_click_events.find(coll);
- if(it == query_collection_click_events.end()) {
- return {};
+ std::vector<std::string> click_event_jsons;
+ store->scan_fill(std::string(CLICK_EVENT) + "_", std::string(CLICK_EVENT) + "`",
+ click_event_jsons);
+
+ nlohmann::json result_json = nlohmann::json::array();
+ for(const auto & click_event_json: click_event_jsons) {
+ nlohmann::json click_event = nlohmann::json::parse(click_event_json);
+ result_json.push_back(click_event);
+ }
+
+ return result_json;
+}
+
+Option<bool> AnalyticsManager::write_click_event_to_store(nlohmann::json &click_event_json) {
+ auto collection_id = click_event_json["collection_id"].get<std::string>();
+ auto timestamp = click_event_json["timestamp"].get<uint64_t>();
+ const std::string key = std::string(CLICK_EVENT) + "_" + collection_id + "_" +
+ std::to_string(timestamp);
+ bool inserted = store->insert(key, click_event_json.dump());
+ if (!inserted) {
+ return Option<bool>(500, "Unable to insert clickevent into store.");
}
- return it->second;
+
+ return Option<bool>(true);
}
\ No newline at end of file
diff --git a/src/core_api.cpp b/src/core_api.cpp
index e06127ce..55dcdea4 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -2381,21 +2381,29 @@ bool post_proxy(const std::shared_ptr<http_req>& req, const std::shared_ptr<http
}
bool get_click_events(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
- auto click_events = AnalyticsManager::get_instance().get_click_events(req->params["collection"]);
+ auto click_events = AnalyticsManager::get_instance().get_click_events();
- if(click_events.empty()) {
+ res->set_200(click_events.dump());
+ return true;
+}
+
+bool post_replicate_click_event(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ nlohmann::json req_json;
+
+ try {
+ req_json = nlohmann::json::parse(req->body);
+ } catch(const std::exception& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ res->set_400("Bad JSON.");
return false;
}
-
- nlohmann::json res_json, doc;
- res_json["name"] = req->params["collection"];
- res_json["click_events"] = nlohmann::json::array();
- for(const auto& click_event : click_events) {
- click_event.to_json(doc);
- res_json["click_events"].push_back(doc);
+ auto op = AnalyticsManager::get_instance().write_click_event_to_store(req_json);
+ if(!op.ok()) {
+ res->set_body(op.code(), op.error());
+ return false;
}
- res->set_200(res_json.dump());
+ res->set_200("ClickEvent wrote to DB.");
return true;
}
\ No newline at end of file
diff --git a/src/event_manager.cpp b/src/event_manager.cpp
index 7f709e5e..ca8f02e7 100644
--- a/src/event_manager.cpp
+++ b/src/event_manager.cpp
@@ -58,22 +58,22 @@ bool EventManager::add_event(const nlohmann::json& event) {
return false;
}
- if (!event_data_val.contains("q") || !event_data_val.contains("product_id")
+ if (!event_data_val.contains("q") || !event_data_val.contains("doc_id")
|| !event_data_val.contains("position") || !event_data_val.contains("collection")) {
return false;
}
- if (!event_data_val["q"].is_string() || !event_data_val["product_id"].is_number_unsigned()
+ if (!event_data_val["q"].is_string() || !event_data_val["doc_id"].is_string()
|| !event_data_val["position"].is_number_unsigned() || !event_data_val["collection"].is_string()) {
return false;
}
const std::string query = event_data_val["q"].get<std::string>();
- uint64_t product_id = event_data_val["product_id"].get<uint64_t>();
+ const std::string doc_id = event_data_val["doc_id"].get<std::string>();
uint64_t position = event_data_val["position"].get<uint64_t>();
const std::string& collection = event_data_val["collection"].get<std::string>();
- AnalyticsManager::get_instance().add_click_event(collection, query, product_id, position);
+ AnalyticsManager::get_instance().add_click_event(collection, query, doc_id, position);
}
}
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index f08b404e..6cbc5d5a 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -82,8 +82,9 @@ void master_server_routes() {
server->post("/analytics/events", post_create_event);
//collection based query click events
- server->get("/collections/:collection/click_events", get_click_events);
- server->post("/collections/:collection/click_events", post_create_event);
+ server->get("/analytics/click_events", get_click_events);
+ server->post("/analytics/click_events", post_create_event);
+ server->post("/analytics/click_events/replicate", post_replicate_click_event);
// meta
server->get("/metrics.json", get_metrics_json);