Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Handle aliasing for analytics query.
Kishore Nallan4c32ead921b8
diff --git a/include/analytics_manager.h b/include/analytics_manager.hindex 9920cfcb..353085b6 100644--- a/include/analytics_manager.h+++ b/include/analytics_manager.h@@ -79,4 +79,8 @@ public:void stop();void dispose();++ void persist_suggestions(ReplicationState *raft_server, uint64_t prev_persistence_s);++ std::unordered_map<std::string, PopularQueries*> get_popular_queries();};diff --git a/src/analytics_manager.cpp b/src/analytics_manager.cppindex 225101cc..26d58ac6 100644--- a/src/analytics_manager.cpp+++ b/src/analytics_manager.cpp@@ -12,12 +12,14 @@ Option<bool> AnalyticsManager::create_rule(nlohmann::json& payload, bool write_t{"name": "top_search_queries","type": "popular_queries",- "limit": 1000,- "source": {- "collections": ["brands", "products"]- },- "destination": {- "collection": "top_search_queries"+ "params": {+ "limit": 1000,+ "source": {+ "collections": ["brands", "products"]+ },+ "destination": {+ "collection": "top_search_queries"+ }}}*/@@ -207,76 +209,86 @@ void AnalyticsManager::run(ReplicationState* raft_server) {break;}- for(const auto& suggestion_config: suggestion_configs) {- const std::string& sink_name = suggestion_config.first;- const std::string& suggestion_coll = suggestion_config.second.suggestion_collection;+ persist_suggestions(raft_server, prev_persistence_s);- auto popular_queries_it = popular_queries.find(suggestion_coll);- if(popular_queries_it == popular_queries.end()) {- continue;- }+ lk.unlock();+ }- // need to prepare the counts as JSON docs for import into the suggestion collection- // {"id": "432432", "q": "foo", "$operations": {"increment": {"count": 100}}}+ dispose();+}- PopularQueries* popularQueries = popular_queries_it->second;+void AnalyticsManager::persist_suggestions(ReplicationState *raft_server, uint64_t prev_persistence_s) {+ // lock is held by caller+ for(const auto& suggestion_config: suggestion_configs) {+ const std::string& sink_name = suggestion_config.first;+ const std::string& suggestion_coll = suggestion_config.second.suggestion_collection;- // aggregate prefix queries to their final form- auto now = std::chrono::system_clock::now().time_since_epoch();- auto now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(now).count();- popularQueries->compact_user_queries(now_ts_us);+ auto popular_queries_it = popular_queries.find(suggestion_coll);+ if(popular_queries_it == popular_queries.end()) {+ continue;+ }- auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(now).count();+ // need to prepare the counts as JSON docs for import into the suggestion collection+ // {"id": "432432", "q": "foo", "$operations": {"increment": {"count": 100}}}- if(now_ts_seconds - prev_persistence_s < Config::get_instance().get_analytics_flush_interval()) {- // we will persist aggregation every hour- continue;- }+ PopularQueries* popularQueries = popular_queries_it->second;- prev_persistence_s = now_ts_seconds;+ // aggregate prefix queries to their final form+ auto now = std::chrono::system_clock::now().time_since_epoch();+ auto now_ts_us = std::chrono::duration_cast<std::chrono::microseconds>(now).count();+ popularQueries->compact_user_queries(now_ts_us);- std::string import_payload;- popularQueries->serialize_as_docs(import_payload);+ auto now_ts_seconds = std::chrono::duration_cast<std::chrono::seconds>(now).count();- if(import_payload.empty()) {- continue;- }+ if(now_ts_seconds - prev_persistence_s < Config::get_instance().get_analytics_flush_interval()) {+ // we will persist aggregation every hour+ continue;+ }++ prev_persistence_s = now_ts_seconds;++ std::string import_payload;+ popularQueries->serialize_as_docs(import_payload);- // send http request- std::string leader_url = raft_server->get_leader_url();- if(!leader_url.empty()) {- const std::string& resource_url = leader_url + "collections/" + suggestion_coll +- "/documents/import?action=emplace";- std::string res;- std::map<std::string, std::string> res_headers;- long status_code = HttpClient::post_response(resource_url, import_payload,- res, res_headers, 10*1000);-- if(status_code != 200) {- LOG(ERROR) << "Error while sending query suggestions events to leader. "- << "Status code: " << status_code << ", response: " << res;- } else {- LOG(INFO) << "Sent query suggestions to leader for aggregation.";- popularQueries->reset_local_counts();-- if(raft_server->is_leader()) {- // try to run top-K compaction of suggestion collection- auto coll = CollectionManager::get_instance().get_collection(suggestion_coll);- if (coll == nullptr) {- LOG(ERROR) << "No collection found for suggestions aggregation: " + suggestion_coll;- continue;- }-- coll->truncate_after_top_k("count", popularQueries->get_k());+ if(import_payload.empty()) {+ continue;+ }++ // send http request+ std::string leader_url = raft_server->get_leader_url();+ if(!leader_url.empty()) {+ const std::string& base_url = leader_url + "collections/" + suggestion_coll;+ std::string res;++ const std::string& update_url = base_url + "/documents/import?action=emplace";+ std::map<std::string, std::string> res_headers;+ long status_code = HttpClient::post_response(update_url, import_payload,+ res, res_headers, 10*1000);++ if(status_code != 200) {+ LOG(ERROR) << "Error while sending query suggestions events to leader. "+ << "Status code: " << status_code << ", response: " << res;+ } else {+ LOG(INFO) << "Query aggregation for collection: " + suggestion_coll;+ popularQueries->reset_local_counts();++ if(raft_server->is_leader()) {+ // try to run top-K compaction of suggestion collection+ const std::string top_k_param = "count:" + std::to_string(popularQueries->get_k());+ const std::string& truncate_topk_url = base_url + "/documents?top_k_by=" + top_k_param;+ res.clear();+ res_headers.clear();+ status_code = HttpClient::delete_response(truncate_topk_url, res, res_headers, 10*1000);+ if(status_code != 200) {+ LOG(ERROR) << "Error while running top K for query suggestions collection. "+ << "Status code: " << status_code << ", response: " << res;+ } else {+ LOG(INFO) << "Top K aggregation for collection: " + suggestion_coll;}}}}-- lk.unlock();}-- dispose();}void AnalyticsManager::stop() {@@ -285,6 +297,8 @@ void AnalyticsManager::stop() {}void AnalyticsManager::dispose() {+ std::unique_lock lk(mutex);+for(auto& kv: popular_queries) {delete kv.second;}@@ -296,3 +310,7 @@ void AnalyticsManager::init(Store* store) {this->store = store;}+std::unordered_map<std::string, PopularQueries*> AnalyticsManager::get_popular_queries() {+ std::unique_lock lk(mutex);+ return popular_queries;+}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex b69795d9..6c9398e3 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -734,7 +734,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re}CollectionManager & collectionManager = CollectionManager::get_instance();- auto collection = collectionManager.get_collection(req_params["collection"]);+ const std::string& orig_coll_name = req_params["collection"];+ auto collection = collectionManager.get_collection(orig_coll_name);if(collection == nullptr) {return Option<bool>(404, "Not found.");@@ -1064,9 +1065,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reif(Config::get_instance().get_enable_search_analytics()) {if(result.count("found") != 0 && result["found"].get<size_t>() != 0) {- std::string processed_query = raw_query;- Tokenizer::normalize_ascii(processed_query);- AnalyticsManager::get_instance().add_suggestion(collection->get_name(), processed_query,+ std::string analytics_query = raw_query;+ AnalyticsManager::get_instance().add_suggestion(orig_coll_name, analytics_query,true, req_params["x-typesense-user-id"]);}}diff --git a/src/core_api.cpp b/src/core_api.cppindex 0973c6d9..61fd894d 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1114,6 +1114,37 @@ bool del_remove_documents(const std::shared_ptr<http_req>& req, const std::shareconst char *BATCH_SIZE = "batch_size";const char *FILTER_BY = "filter_by";+ const char *TOP_K_BY = "top_k_by";++ if(req->params.count(TOP_K_BY) != 0) {+ std::vector<std::string> parts;+ StringUtils::split(req->params[TOP_K_BY], parts, ":");++ if(parts.size() != 2 || !StringUtils::is_uint32_t(parts[1])) {+ req->last_chunk_aggregate = true;+ res->final = true;+ res->set_400("The `top_k_by` parameter is not valid.");+ stream_response(req, res);+ return false;+ }++ const std::string& field_name = parts[0];+ const size_t k = std::stoull(parts[1]);+ auto op = collection->truncate_after_top_k(field_name, k);++ req->last_chunk_aggregate = true;+ res->final = true;++ if(!op.ok()) {+ res->set_500(op.error());+ stream_response(req, res);+ return false;+ }++ res->set_200(R"({"ok": true})");+ stream_response(req, res);+ return true;+ }if(req->params.count(BATCH_SIZE) == 0) {req->params[BATCH_SIZE] = "1000000000"; // 1 Billiondiff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppnew file mode 100644index 00000000..0030f221--- /dev/null+++ b/test/analytics_manager_test.cpp@@ -0,0 +1,91 @@+#include <gtest/gtest.h>+#include <string>+#include <vector>+#include <collection_manager.h>+#include <analytics_manager.h>+#include "collection.h"++class AnalyticsManagerTest : public ::testing::Test {+protected:+ Store *store;+ CollectionManager& collectionManager = CollectionManager::get_instance();+ std::atomic<bool> quit = false;++ std::vector<std::string> query_fields;+ std::vector<sort_by> sort_fields;++ AnalyticsManager& analyticsManager = AnalyticsManager::get_instance();++ void setupCollection() {+ std::string state_dir_path = "/tmp/typesense_test/analytics_manager_test";+ LOG(INFO) << "Truncating and creating: " << state_dir_path;+ system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());+ system("mkdir -p /tmp/typesense_test/models");++ store = new Store(state_dir_path);+ collectionManager.init(store, 1.0, "auth_key", quit);+ collectionManager.load(8, 1000);++ analyticsManager.init(store);+ }++ virtual void SetUp() {+ setupCollection();+ }++ virtual void TearDown() {+ collectionManager.dispose();+ delete store;+ }+};++TEST_F(AnalyticsManagerTest, AddSuggestion) {+ nlohmann::json titles_schema = R"({+ "name": "titles",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection* titles_coll = collectionManager.create_collection(titles_schema).get();++ nlohmann::json doc;+ doc["title"] = "Cool trousers";+ ASSERT_TRUE(titles_coll->add(doc.dump()).ok());++ // create a collection to store suggestions+ nlohmann::json suggestions_schema = R"({+ "name": "top_queries",+ "fields": [+ {"name": "q", "type": "string" },+ {"name": "count", "type": "int32" }+ ]+ })"_json;++ Collection* suggestions_coll = collectionManager.create_collection(suggestions_schema).get();++ nlohmann::json analytics_rule = R"({+ "name": "top_search_queries",+ "type": "popular_queries",+ "params": {+ "limit": 100,+ "source": {+ "collections": ["titles"]+ },+ "destination": {+ "collection": "top_queries"+ }+ }+ })"_json;++ auto create_op = analyticsManager.create_rule(analytics_rule);+ ASSERT_TRUE(create_op.ok());++ std::string q = "foobar";+ analyticsManager.add_suggestion("titles", q, true, "1");++ auto popularQueries = analyticsManager.get_popular_queries();+ auto userQueries = popularQueries["top_queries"]->get_user_prefix_queries()["1"];+ ASSERT_EQ(1, userQueries.size());+ ASSERT_EQ("foobar", userQueries[0].query);+}