Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Refactor analytics paths.
Kishore Nallan5abec3f6e05c
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 239319fe..bbb8e2d5 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -20,7 +20,14 @@ private:std::string name;std::string suggestion_collection;std::vector<std::string> query_collections;- size_t max_suggestions;+ size_t limit;++ void to_json(nlohmann::json& obj) const {+ obj["name"] = name;+ obj["suggestion_collection"] = suggestion_collection;+ obj["query_collections"] = query_collections;+ obj["limit"] = limit;+ }};// config name => config@@ -38,10 +45,14 @@ private:~AnalyticsManager();+ Option<bool> remove_popular_queries_index(const std::string& name);++ Option<bool> create_popular_queries_index(nlohmann::json &payload, bool write_to_disk);+public:- static constexpr const char* ANALYTICS_CONFIG_PREFIX = "$AC";- static constexpr const char* RESOURCE_TYPE = "popular_queries";+ static constexpr const char* ANALYTICS_RULE_PREFIX = "$AR";+ static constexpr const char* POPULAR_QUERIES_TYPE = "popular_queries";static AnalyticsManager& get_instance() {static AnalyticsManager instance;@@ -55,9 +66,11 @@ public:void run(ReplicationState* raft_server);- Option<bool> create_index(nlohmann::json& payload, bool write_to_disk = true);+ Option<nlohmann::json> list_rules();++ Option<bool> create_rule(nlohmann::json& payload, bool write_to_disk = true);- Option<bool> remove_suggestion_index(const std::string& name);+ Option<bool> remove_rule(const std::string& name);void add_suggestion(const std::string& query_collection,std::string& query, const bool live_query, const std::string& user_id);diff --git a/include/core_api.h b/include/core_api.hindex 4a18a26f..9d31d65a 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -141,13 +141,15 @@ bool del_throttle(const std::shared_ptr<http_req>& req, const std::shared_ptr<htbool del_exceed(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);-// Events+// Analyticsbool post_create_event(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);-bool post_create_analytics_popular_queries(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool get_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);-bool del_analytics_popular_queries(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool post_create_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);++bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);// Misc helpersdiff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex 2d51d215..4ff2295f 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -5,26 +5,39 @@#include "http_client.h"#include "collection_manager.h"-Option<bool> AnalyticsManager::create_index(nlohmann::json& payload, bool write_to_disk) {+Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool write_to_disk) {/*Sample payload:{- "name": "top_queries",+ "name": "top_search_queries",+ "type": "popular_queries","limit": 1000,"source": {"collections": ["brands", "products"]},"destination": {- "collection": "top_queries"+ "collection": "top_search_queries"}}*/+ if(!payload.contains("type") || !payload["type"].is_string()) {+ return Option<bool>(400, "Request payload contains invalid type.");+ }+if(!payload.contains("name") || !payload["name"].is_string()) {return Option<bool>(400, "Bad or missing name.");}+ if(payload["type"] == POPULAR_QUERIES_TYPE) {+ return create_popular_queries_index(payload, write_to_disk);+ }++ return Option<bool>(400, "Invalid type.");+}++Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &payload, bool write_to_disk) {if(!payload.contains("source") || !payload["source"].is_object()) {return Option<bool>(400, "Bad or missing source.");}@@ -35,10 +48,10 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json& payload, bool write_const std::string& suggestion_config_name = payload["name"].get<std::string>();- size_t max_suggestions = 1000;+ size_t limit = 1000;if(payload.contains("limit") && payload["limit"].is_number_integer()) {- max_suggestions = payload["limit"].get<size_t>();+ limit = payload["limit"].get<size_t>();}if(suggestion_configs.find(suggestion_config_name) != suggestion_configs.end()) {@@ -58,7 +71,7 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json& payload, bool write_suggestion_config_t suggestion_config;suggestion_config.name = suggestion_config_name;suggestion_config.suggestion_collection = suggestion_collection;- suggestion_config.max_suggestions = max_suggestions;+ suggestion_config.limit = limit;for(const auto& coll: payload["source"]["collections"]) {if(!coll.is_string()) {@@ -77,12 +90,11 @@ Option<bool> AnalyticsManager::create_index(nlohmann::json& payload, bool write_query_collection_mapping[query_coll].push_back(suggestion_collection);}- PopularQueries* popularQueries = new PopularQueries(max_suggestions);+ PopularQueries* popularQueries = new PopularQueries(limit);popular_queries.emplace(suggestion_collection, popularQueries);if(write_to_disk) {- payload["type"] = RESOURCE_TYPE;- auto suggestion_key = std::string(ANALYTICS_CONFIG_PREFIX) + "_" + suggestion_config_name;+ auto suggestion_key = std::string(ANALYTICS_RULE_PREFIX) + "_" + suggestion_config_name;bool inserted = store->insert(suggestion_key, payload.dump());if(!inserted) {return Option<bool>(500, "Error while storing the config to disk.");@@ -100,13 +112,40 @@ AnalyticsManager::~AnalyticsManager() {}}-Option<bool> AnalyticsManager::remove_suggestion_index(const std::string &name) {+Option<nlohmann::json> AnalyticsManager::list_rules() {+ std::unique_lock lock(mutex);++ nlohmann::json rules = nlohmann::json::object();+ rules["rules"]= nlohmann::json::array();++ for(const auto& suggestion_config: suggestion_configs) {+ nlohmann::json rule;+ suggestion_config.second.to_json(rule);+ rule["type"] = POPULAR_QUERIES_TYPE;+ rules["rules"].push_back(rule);+ }++ return Option<nlohmann::json>(rules);+}++Option<bool> AnalyticsManager::remove_rule(const string &name) {+ std::unique_lock lock(mutex);++ auto suggestion_configs_it = suggestion_configs.find(name);+ if(suggestion_configs_it != suggestion_configs.end()) {+ return remove_popular_queries_index(name);+ }++ return Option<bool>(404, "Rule not found.");+}++Option<bool> AnalyticsManager::remove_popular_queries_index(const std::string &name) {std::unique_lock lock(mutex);auto suggestion_configs_it = suggestion_configs.find(name);if(suggestion_configs_it == suggestion_configs.end()) {- return Option<bool>(404, "Index not found.");+ return Option<bool>(404, "Rule not found.");}const auto& suggestion_collection = suggestion_configs_it->second.suggestion_collection;@@ -122,7 +161,7 @@ Option<bool> AnalyticsManager::remove_suggestion_index(const std::string &name)suggestion_configs.erase(name);- auto suggestion_key = std::string(ANALYTICS_CONFIG_PREFIX) + "_" + name;+ auto suggestion_key = std::string(ANALYTICS_RULE_PREFIX) + "_" + name;bool erased = store->remove(suggestion_key);if(!erased) {return Option<bool>(500, "Error while deleting from disk.");@@ -250,3 +289,4 @@ void AnalyticsManager::dispose() {void AnalyticsManager::init(Store* store) {this->store = store;}+diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex b1ba49a8..b69795d9 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -1300,15 +1300,13 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collection// restore query suggestions configsstd::vector<std::string> analytics_config_jsons;- cm.store->scan_fill(AnalyticsManager::ANALYTICS_CONFIG_PREFIX,- std::string(AnalyticsManager::ANALYTICS_CONFIG_PREFIX) + "`",+ cm.store->scan_fill(AnalyticsManager::ANALYTICS_RULE_PREFIX,+ std::string(AnalyticsManager::ANALYTICS_RULE_PREFIX) + "`",analytics_config_jsons);for(const auto& analytics_config_json: analytics_config_jsons) {nlohmann::json analytics_config = nlohmann::json::parse(analytics_config_json);- if(analytics_config["type"] == AnalyticsManager::RESOURCE_TYPE) {- AnalyticsManager::get_instance().create_index(analytics_config, false);- }+ AnalyticsManager::get_instance().create_rule(analytics_config, false);}// Fetch records from the store and re-create memory indexdiff --git a/src/core_api.cpp b/src/core_api.cppindex d5d881a2..0973c6d9 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -2061,7 +2061,19 @@ bool post_create_event(const std::shared_ptr<http_req>& req, const std::shared_preturn false;}-bool post_create_analytics_popular_queries(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+bool get_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ auto rules_op = AnalyticsManager::get_instance().list_rules();++ if(rules_op.ok()) {+ res->set_200(rules_op.get().dump());+ return true;+ }++ res->set(rules_op.code(), rules_op.error());+ return false;+}++bool post_create_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {nlohmann::json req_json;try {@@ -2072,7 +2084,7 @@ bool post_create_analytics_popular_queries(const std::shared_ptr<http_req>& req,return false;}- auto op = AnalyticsManager::get_instance().create_index(req_json);+ auto op = AnalyticsManager::get_instance().create_rule(req_json);if(!op.ok()) {res->set(op.code(), op.error());@@ -2083,8 +2095,8 @@ bool post_create_analytics_popular_queries(const std::shared_ptr<http_req>& req,return true;}-bool del_analytics_popular_queries(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {- auto op = AnalyticsManager::get_instance().remove_suggestion_index(req->params["name"]);+bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ auto op = AnalyticsManager::get_instance().remove_rule(req->params["name"]);if(!op.ok()) {res->set(op.code(), op.error());return false;diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex c1734747..30b7281d 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -69,8 +69,9 @@ void master_server_routes() {server->del("/presets/:name", del_preset);// analytics- server->post("/analytics/popular-queries", post_create_analytics_popular_queries);- server->del("/analytics/popular-queries/:name", del_analytics_popular_queries);+ server->get("/analytics/rules", get_analytics_rules);+ server->post("/analytics/rules", post_create_analytics_rules);+ server->del("/analytics/rules/:name", del_analytics_rules);server->post("/analytics/events", post_create_event);// meta