Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add upsert + get APIs for analytics rules.
Kishore Nallan8e1f6caaf15b
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 353085b6..4207f7cf 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -24,10 +24,11 @@ private:void to_json(nlohmann::json& obj) const {obj["name"] = name;+ obj["type"] = POPULAR_QUERIES_TYPE;obj["params"] = nlohmann::json::object();- obj["params"]["suggestion_collection"] = suggestion_collection;- obj["params"]["query_collections"] = query_collections;obj["params"]["limit"] = limit;+ obj["params"]["source"]["collections"] = query_collections;+ obj["params"]["destination"]["collection"] = suggestion_collection;}};@@ -48,7 +49,9 @@ private:Option<bool> remove_popular_queries_index(const std::string& name);- Option<bool> create_popular_queries_index(nlohmann::json &payload, bool write_to_disk);+ Option<bool> create_popular_queries_index(nlohmann::json &payload,+ bool upsert,+ bool write_to_disk);public:@@ -69,12 +72,14 @@ public:Option<nlohmann::json> list_rules();- Option<bool> create_rule(nlohmann::json& payload, bool write_to_disk = true);+ Option<nlohmann::json> get_rule(const std::string& name);++ Option<bool> create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk);Option<bool> remove_rule(const std::string& name);void add_suggestion(const std::string& query_collection,- std::string& query, const bool live_query, const std::string& user_id);+ std::string& query, bool live_query, const std::string& user_id);void stop();diff --git a/include/core_api.h b/include/core_api.hindex 780d3d1b..a13b1db8 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -147,8 +147,12 @@ bool post_create_event(const std::shared_ptr<http_req>& req, const std::shared_pbool get_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool get_analytics_rule(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool post_create_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool put_upsert_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);// Misc helpersdiff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex 4851dd46..dcb6a64a 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -5,7 +5,7 @@#include "http_client.h"#include "collection_manager.h"-Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool write_to_disk) {+Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool upsert, bool write_to_disk) {/*Sample payload:@@ -37,16 +37,23 @@ Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool write_t}if(payload["type"] == POPULAR_QUERIES_TYPE) {- return create_popular_queries_index(payload, write_to_disk);+ return create_popular_queries_index(payload, upsert, write_to_disk);}return Option<bool>(400, "Invalid type.");}-Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &payload, bool write_to_disk) {+Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &payload, bool upsert, bool write_to_disk) {// params and name are validated upstream- const auto& params = payload["params"];const std::string& suggestion_config_name = payload["name"].get<std::string>();+ bool already_exists = suggestion_configs.find(suggestion_config_name) != suggestion_configs.end();++ if(!upsert && already_exists) {+ return Option<bool>(400, "There's already another configuration with the name `" ++ suggestion_config_name + "`.");+ }++ const auto& params = payload["params"];if(!params.contains("source") || !params["source"].is_object()) {return Option<bool>(400, "Bad or missing source.");@@ -56,18 +63,12 @@ Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &paylreturn Option<bool>(400, "Bad or missing destination.");}-size_t limit = 1000;if(params.contains("limit") && params["limit"].is_number_integer()) {limit = params["limit"].get<size_t>();}- if(suggestion_configs.find(suggestion_config_name) != suggestion_configs.end()) {- return Option<bool>(400, "There's already another configuration with the name `" +- suggestion_config_name + "`.");- }-if(!params["source"].contains("collections") || !params["source"]["collections"].is_array()) {return Option<bool>(400, "Must contain a valid list of source collections.");}@@ -93,6 +94,14 @@ Option<bool> AnalyticsManager::create_popular_queries_index(nlohmann::json &paylstd::unique_lock lock(mutex);+ if(already_exists) {+ // remove the previous configuration with same name (upsert)+ Option<bool> remove_op = remove_popular_queries_index(suggestion_config_name);+ if(!remove_op.ok()) {+ return Option<bool>(500, "Error erasing the existing configuration.");;+ }+ }+suggestion_configs.emplace(suggestion_config_name, suggestion_config);for(const auto& query_coll: suggestion_config.query_collections) {@@ -130,13 +139,25 @@ Option<nlohmann::json> AnalyticsManager::list_rules() {for(const auto& suggestion_config: suggestion_configs) {nlohmann::json rule;suggestion_config.second.to_json(rule);- rule["type"] = POPULAR_QUERIES_TYPE;rules["rules"].push_back(rule);}return Option<nlohmann::json>(rules);}+Option<nlohmann::json> AnalyticsManager::get_rule(const string& name) {+ nlohmann::json rule;+ std::unique_lock lock(mutex);++ auto suggestion_config_it = suggestion_configs.find(name);+ if(suggestion_config_it == suggestion_configs.end()) {+ return Option<nlohmann::json>(404, "Rule not found.");+ }++ suggestion_config_it->second.to_json(rule);+ return Option<nlohmann::json>(rule);+}+Option<bool> AnalyticsManager::remove_rule(const string &name) {std::unique_lock lock(mutex);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex f8d32b4e..6c281595 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -285,6 +285,17 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const siter->Next();}+ // restore query suggestions configs+ std::vector<std::string> analytics_config_jsons;+ store->scan_fill(AnalyticsManager::ANALYTICS_RULE_PREFIX,+ std::string(AnalyticsManager::ANALYTICS_RULE_PREFIX) + "`",+ analytics_config_jsons);++ for(const auto& analytics_config_json: analytics_config_jsons) {+ nlohmann::json analytics_config = nlohmann::json::parse(analytics_config_json);+ AnalyticsManager::get_instance().create_rule(analytics_config, false, false);+ }+delete iter;LOG(INFO) << "Loaded " << num_collections << " collection(s).";@@ -1312,17 +1323,6 @@ Option<bool> CollectionManager::load_collection(const nlohmann::json &collectioncollection->add_synonym(collection_synonym, false);}- // restore query suggestions configs- std::vector<std::string> analytics_config_jsons;- cm.store->scan_fill(AnalyticsManager::ANALYTICS_RULE_PREFIX,- std::string(AnalyticsManager::ANALYTICS_RULE_PREFIX) + "`",- analytics_config_jsons);-- for(const auto& analytics_config_json: analytics_config_jsons) {- nlohmann::json analytics_config = nlohmann::json::parse(analytics_config_json);- AnalyticsManager::get_instance().create_rule(analytics_config, false);- }-// Fetch records from the store and re-create memory indexconst std::string seq_id_prefix = collection->get_seq_id_collection_prefix();std::string upper_bound_key = collection->get_seq_id_collection_prefix() + "`"; // cannot inline thisdiff --git a/src/core_api.cpp b/src/core_api.cppindex 5d04ad07..0c9cd0e5 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -2078,13 +2078,25 @@ bool post_create_event(const std::shared_ptr<http_req>& req, const std::shared_pbool get_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {auto rules_op = AnalyticsManager::get_instance().list_rules();- if(rules_op.ok()) {- res->set_200(rules_op.get().dump());- return true;+ if(!rules_op.ok()) {+ res->set(rules_op.code(), rules_op.error());+ return false;}- res->set(rules_op.code(), rules_op.error());- return false;+ res->set_200(rules_op.get().dump());+ return true;+}++bool get_analytics_rule(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ auto rules_op = AnalyticsManager::get_instance().get_rule(req->params["name"]);++ if(!rules_op.ok()) {+ res->set(rules_op.code(), rules_op.error());+ return false;+ }++ res->set_200(rules_op.get().dump());+ return true;}bool post_create_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {@@ -2098,7 +2110,7 @@ bool post_create_analytics_rules(const std::shared_ptr<http_req>& req, const stdreturn false;}- auto op = AnalyticsManager::get_instance().create_rule(req_json);+ auto op = AnalyticsManager::get_instance().create_rule(req_json, false, true);if(!op.ok()) {res->set(op.code(), op.error());@@ -2109,6 +2121,29 @@ bool post_create_analytics_rules(const std::shared_ptr<http_req>& req, const stdreturn true;}+bool put_upsert_analytics_rules(const std::shared_ptr<http_req> &req, const std::shared_ptr<http_res> &res) {+ nlohmann::json req_json;++ try {+ req_json = nlohmann::json::parse(req->body);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ res->set_400("Bad JSON.");+ return false;+ }++ req_json["name"] = req->params["name"];+ auto op = AnalyticsManager::get_instance().create_rule(req_json, true, true);++ if(!op.ok()) {+ res->set(op.code(), op.error());+ return false;+ }++ res->set_200(req_json.dump());+ return true;+}+bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {auto op = AnalyticsManager::get_instance().remove_rule(req->params["name"]);if(!op.ok()) {@@ -2116,11 +2151,10 @@ bool del_analytics_rules(const std::shared_ptr<http_req>& req, const std::sharedreturn false;}- res->set_200(R"({"ok": true)");+ res->set_200(R"({"ok": true})");return true;}-bool post_proxy(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {HttpProxy& proxy = HttpProxy::get_instance();@@ -2180,4 +2214,4 @@ bool post_proxy(const std::shared_ptr<http_req>& req, const std::shared_ptr<httpres->set_200(response.body);return true;-}\ No newline at end of file+}diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex d4df4f2a..8a47e1bb 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -70,7 +70,9 @@ void master_server_routes() {// analyticsserver->get("/analytics/rules", get_analytics_rules);+ server->get("/analytics/rules/:name", get_analytics_rule);server->post("/analytics/rules", post_create_analytics_rules);+ server->put("/analytics/rules/:name", put_upsert_analytics_rules);server->del("/analytics/rules/:name", del_analytics_rules);server->post("/analytics/events", post_create_event);diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex 0030f221..a86bf809 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -78,7 +78,7 @@ TEST_F(AnalyticsManagerTest, AddSuggestion) {}})"_json;- auto create_op = analyticsManager.create_rule(analytics_rule);+ auto create_op = analyticsManager.create_rule(analytics_rule, false, true);ASSERT_TRUE(create_op.ok());std::string q = "foobar";@@ -88,4 +88,97 @@ TEST_F(AnalyticsManagerTest, AddSuggestion) {auto userQueries = popularQueries["top_queries"]->get_user_prefix_queries()["1"];ASSERT_EQ(1, userQueries.size());ASSERT_EQ("foobar", userQueries[0].query);++ // add another query which is more popular+ q = "buzzfoo";+ analyticsManager.add_suggestion("titles", q, true, "1");+ analyticsManager.add_suggestion("titles", q, true, "2");+ analyticsManager.add_suggestion("titles", q, true, "3");++ popularQueries = analyticsManager.get_popular_queries();+ userQueries = popularQueries["top_queries"]->get_user_prefix_queries()["1"];+ ASSERT_EQ(2, userQueries.size());+ ASSERT_EQ("foobar", userQueries[0].query);+ ASSERT_EQ("buzzfoo", userQueries[1].query);}++TEST_F(AnalyticsManagerTest, GetAndDeleteSuggestions) {+ nlohmann::json analytics_rule = R"({+ "name": "top_search_queries",+ "type": "popular_queries",+ "params": {+ "limit": 100,+ "source": {+ "collections": ["titles"]+ },+ "destination": {+ "collection": "top_queries"+ }+ }+ })"_json;++ auto create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_TRUE(create_op.ok());++ analytics_rule = R"({+ "name": "top_search_queries2",+ "type": "popular_queries",+ "params": {+ "limit": 100,+ "source": {+ "collections": ["titles"]+ },+ "destination": {+ "collection": "top_queries"+ }+ }+ })"_json;++ create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_TRUE(create_op.ok());++ auto rules = analyticsManager.list_rules().get()["rules"];+ ASSERT_EQ(2, rules.size());++ ASSERT_TRUE(analyticsManager.get_rule("top_search_queries").ok());+ ASSERT_TRUE(analyticsManager.get_rule("top_search_queries2").ok());++ auto missing_rule_op = analyticsManager.get_rule("top_search_queriesX");+ ASSERT_FALSE(missing_rule_op.ok());+ ASSERT_EQ(404, missing_rule_op.code());+ ASSERT_EQ("Rule not found.", missing_rule_op.error());++ // upsert rule that already exists+ analytics_rule = R"({+ "name": "top_search_queries2",+ "type": "popular_queries",+ "params": {+ "limit": 100,+ "source": {+ "collections": ["titles"]+ },+ "destination": {+ "collection": "top_queriesUpdated"+ }+ }+ })"_json;+ create_op = analyticsManager.create_rule(analytics_rule, true, true);+ ASSERT_TRUE(create_op.ok());+ auto existing_rule = analyticsManager.get_rule("top_search_queries2").get();+ ASSERT_EQ("top_queriesUpdated", existing_rule["params"]["destination"]["collection"].get<std::string>());++ // reject when upsert is not enabled+ create_op = analyticsManager.create_rule(analytics_rule, false, true);+ ASSERT_FALSE(create_op.ok());+ ASSERT_EQ("There's already another configuration with the name `top_search_queries2`.", create_op.error());++ // try deleting both rules+ analyticsManager.remove_rule("top_search_queries");+ analyticsManager.remove_rule("top_search_queries2");++ missing_rule_op = analyticsManager.get_rule("top_search_queries");+ ASSERT_FALSE(missing_rule_op.ok());+ missing_rule_op = analyticsManager.get_rule("top_search_queries2");+ ASSERT_FALSE(missing_rule_op.ok());+}+