Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Refactor analytics paths.

Kishore Nallan5abec3f6e05c
diff --git a/include/analytics_manager.h b/include/analytics_manager.h
index 239319fe..bbb8e2d5 100644
--- a/include/analytics_manager.h
+++ b/include/analytics_manager.h
@@ -20,7 +20,14 @@ private:
std::string name;
std::string suggestion_collection;
std::vector<std::string> query_collections;
- size_t max_suggestions;
+ size_t limit;
+
+ void to_json(nlohmann::json& obj) const {
+ obj["name"] = name;
+ obj["suggestion_collection"] = suggestion_collection;
+ obj["query_collections"] = query_collections;
+ obj["limit"] = limit;
+ }
};
// config name => config
@@ -38,10 +45,14 @@ private:
~AnalyticsManager();
+ Option<bool> remove_popular_queries_index(const std::string& name);
+
+ Option<bool> create_popular_queries_index(nlohmann::json &payload, bool write_to_disk);
+
public:
- static constexpr const char* ANALYTICS_CONFIG_PREFIX = "$AC";
- static constexpr const char* RESOURCE_TYPE = "popular_queries";
+ static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";
+ static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";
static AnalyticsManager& get_instance() {
static AnalyticsManager instance;
@@ -55,9 +66,11 @@ public:
void run(ReplicationState* raft_server);
- Option<bool> create_index(nlohmann::json& payload, bool write_to_disk = true);
+ Option<nlohmann::json> list_rules();
+
+ Option<bool> create_rule(nlohmann::json& payload, bool write_to_disk = true);
- Option<bool> remove_suggestion_index(const std::string& name);
+ Option<bool> remove_rule(const std::string& name);
void add_suggestion(const std::string& query_collection,
std::string& query, const bool live_query, const std::string& user_id);
diff --git a/include/core_api.h b/include/core_api.h
index 4a18a26f..9d31d65a 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -141,13 +141,15 @@ bool del_throttle(const std::shared_ptr<http_req>& req, const std::shared_ptr<ht
bool del_exceed(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
-// Events
+// Analytics
bool post_create_event(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
-bool post_create_analytics_popular_queries(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+bool get_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
-bool del_analytics_popular_queries(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+bool post_create_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+
+bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
// Misc helpers
diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cpp
index 2d51d215..4ff2295f 100644
--- a/src/analytics_manager.cpp
+++ b/src/analytics_manager.cpp
@@ -5,26 +5,39 @@
#include "http_client.h"
#include "collection_manager.h"
-Option<bool> AnalyticsManager::create_index(nlohmann::json& payload, bool write_to_disk) {
+Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool write_to_disk) {
/*
Sample payload:
{
- "name": "top_queries",
+ "name": "top_search_queries",
+ "type": "popular_queries",
"limit": 1000,
"source": {
"collections": ["brands", "products"]
},
"destination": {
- "collection": "top_queries"
+ "collection": "top_search_queries"
}
}
*/
+ if(!payload.contains("type") || !payload["type"].is_string()) {
+ return Option<bool>(400, "Request payload contains invalid type.");
+ }
+
if(!payload.contains("name") || !payload["name"].is_string()) {
return Option<bool>(400, "Bad or missing name.");
}
+ if(payload["type"] == POPULAR_QUERIES_TYPE) {
+ return create_popular_queries_index(payload, write_to_disk);
+ }
+
+ return Option<bool>(400, "Invalid type.");
+}
+
+Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &payload, bool write_to_disk) {
if(!payload.contains("source") || !payload["source"].is_object()) {
return Option<bool>(400, "Bad or missing source.");
}
@@ -35,10 +48,10 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json& payload, bool write_
const std::string& suggestion_config_name = payload["name"].get<std::string>();
- size_t max_suggestions = 1000;
+ size_t limit = 1000;
if(payload.contains("limit") && payload["limit"].is_number_integer()) {
- max_suggestions = payload["limit"].get<size_t>();
+ limit = payload["limit"].get<size_t>();
}
if(suggestion_configs.find(suggestion_config_name) != suggestion_configs.end()) {
@@ -58,7 +71,7 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json& payload, bool write_
suggestion_config_t suggestion_config;
suggestion_config.name = suggestion_config_name;
suggestion_config.suggestion_collection = suggestion_collection;
- suggestion_config.max_suggestions = max_suggestions;
+ suggestion_config.limit = limit;
for(const auto& coll: payload["source"]["collections"]) {
if(!coll.is_string()) {
@@ -77,12 +90,11 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json& payload, bool write_
query_collection_mapping[query_coll].push_back(suggestion_collection);
}
- PopularQueries* popularQueries = new PopularQueries(max_suggestions);
+ PopularQueries* popularQueries = new PopularQueries(limit);
popular_queries.emplace(suggestion_collection, popularQueries);
if(write_to_disk) {
- payload["type"] = RESOURCE_TYPE;
- auto suggestion_key = std::string(ANALYTICS_CONFIG_PREFIX) + "_" + suggestion_config_name;
+ auto suggestion_key = std::string(ANALYTICS_RULE_PREFIX) + "_" + suggestion_config_name;
bool inserted = store->insert(suggestion_key, payload.dump());
if(!inserted) {
return Option<bool>(500, "Error while storing the config to disk.");
@@ -100,13 +112,40 @@ AnalyticsManager::~AnalyticsManager() {
}
}
-Option<bool> AnalyticsManager::remove_suggestion_index(const std::string &name) {
+Option<nlohmann::json> AnalyticsManager::list_rules() {
+ std::unique_lock lock(mutex);
+
+ nlohmann::json rules = nlohmann::json::object();
+ rules["rules"]= nlohmann::json::array();
+
+ for(const auto& suggestion_config: suggestion_configs) {
+ nlohmann::json rule;
+ suggestion_config.second.to_json(rule);
+ rule["type"] = POPULAR_QUERIES_TYPE;
+ rules["rules"].push_back(rule);
+ }
+
+ return Option<nlohmann::json>(rules);
+}
+
+Option<bool> AnalyticsManager::remove_rule(const string &name) {
+ std::unique_lock lock(mutex);
+
+ auto suggestion_configs_it = suggestion_configs.find(name);
+ if(suggestion_configs_it != suggestion_configs.end()) {
+ return remove_popular_queries_index(name);
+ }
+
+ return Option<bool>(404, "Rule not found.");
+}
+
+Option<bool> AnalyticsManager::remove_popular_queries_index(const std::string &name) {
std::unique_lock lock(mutex);
auto suggestion_configs_it = suggestion_configs.find(name);
if(suggestion_configs_it == suggestion_configs.end()) {
- return Option<bool>(404, "Index not found.");
+ return Option<bool>(404, "Rule not found.");
}
const auto& suggestion_collection = suggestion_configs_it->second.suggestion_collection;
@@ -122,7 +161,7 @@ Option<bool> AnalyticsManager::remove_suggestion_index(const std::string &name)
suggestion_configs.erase(name);
- auto suggestion_key = std::string(ANALYTICS_CONFIG_PREFIX) + "_" + name;
+ auto suggestion_key = std::string(ANALYTICS_RULE_PREFIX) + "_" + name;
bool erased = store->remove(suggestion_key);
if(!erased) {
return Option<bool>(500, "Error while deleting from disk.");
@@ -250,3 +289,4 @@ void AnalyticsManager::dispose() {
void AnalyticsManager::init(Store* store) {
this->store = store;
}
+
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index b1ba49a8..b69795d9 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -1300,15 +1300,13 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection
// restore query suggestions configs
std::vector<std::string> analytics_config_jsons;
- cm.store->scan_fill(AnalyticsManager::ANALYTICS_CONFIG_PREFIX,
- std::string(AnalyticsManager::ANALYTICS_CONFIG_PREFIX) + "`",
+ cm.store->scan_fill(AnalyticsManager::ANALYTICS_RULE_PREFIX,
+ std::string(AnalyticsManager::ANALYTICS_RULE_PREFIX) + "`",
analytics_config_jsons);
for(const auto& analytics_config_json: analytics_config_jsons) {
nlohmann::json analytics_config = nlohmann::json::parse(analytics_config_json);
- if(analytics_config["type"] == AnalyticsManager::RESOURCE_TYPE) {
- AnalyticsManager::get_instance().create_index(analytics_config, false);
- }
+ AnalyticsManager::get_instance().create_rule(analytics_config, false);
}
// Fetch records from the store and re-create memory index
diff --git a/src/core_api.cpp b/src/core_api.cpp
index d5d881a2..0973c6d9 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -2061,7 +2061,19 @@ bool post_create_event(const std::shared_ptr<http_req>& req, const std::shared_p
return false;
}
-bool post_create_analytics_popular_queries(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+bool get_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ auto rules_op = AnalyticsManager::get_instance().list_rules();
+
+ if(rules_op.ok()) {
+ res->set_200(rules_op.get().dump());
+ return true;
+ }
+
+ res->set(rules_op.code(), rules_op.error());
+ return false;
+}
+
+bool post_create_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
nlohmann::json req_json;
try {
@@ -2072,7 +2084,7 @@ bool post_create_analytics_popular_queries(const std::shared_ptr<http_req>& req,
return false;
}
- auto op = AnalyticsManager::get_instance().create_index(req_json);
+ auto op = AnalyticsManager::get_instance().create_rule(req_json);
if(!op.ok()) {
res->set(op.code(), op.error());
@@ -2083,8 +2095,8 @@ bool post_create_analytics_popular_queries(const std::shared_ptr<http_req>& req,
return true;
}
-bool del_analytics_popular_queries(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
- auto op = AnalyticsManager::get_instance().remove_suggestion_index(req->params["name"]);
+bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ auto op = AnalyticsManager::get_instance().remove_rule(req->params["name"]);
if(!op.ok()) {
res->set(op.code(), op.error());
return false;
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index c1734747..30b7281d 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -69,8 +69,9 @@ void master_server_routes() {
server->del("/presets/:name", del_preset);
// analytics
- server->post("/analytics/popular-queries", post_create_analytics_popular_queries);
- server->del("/analytics/popular-queries/:name", del_analytics_popular_queries);
+ server->get("/analytics/rules", get_analytics_rules);
+ server->post("/analytics/rules", post_create_analytics_rules);
+ server->del("/analytics/rules/:name", del_analytics_rules);
server->post("/analytics/events", post_create_event);
// meta