Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
add analytics db and cmdline, clickevent tests
krunalb86d7308e5b1
diff --git a/include/tsconfig.h b/include/tsconfig.hindex 5e754d84..1de30f2b 100644--- a/include/tsconfig.h+++ b/include/tsconfig.h@@ -11,6 +11,7 @@ class Config {private:std::string data_dir;std::string log_dir;+ std::string analytics_dir;std::string api_key;@@ -124,6 +125,10 @@ public:this->log_dir = log_dir;}+ void set_analytics_dir(const std::string& analytics_dir) {+ this->analytics_dir = analytics_dir;+ }+void set_api_key(const std::string & api_key) {this->api_key = api_key;}@@ -191,6 +196,9 @@ public:return this->log_dir;}+ std::string get_analytics_dir() const {+ return this->analytics_dir;+ }std::string get_api_key() const {return this->api_key;@@ -350,6 +358,7 @@ public:void load_config_env() {this->data_dir = get_env("TYPESENSE_DATA_DIR");this->log_dir = get_env("TYPESENSE_LOG_DIR");+ this->analytics_dir = get_env("TYPESENSE_ANALYTICS_DIR");this->api_key = get_env("TYPESENSE_API_KEY");// @deprecated@@ -495,6 +504,10 @@ public:this->log_dir = reader.Get("server", "log-dir", "");}+ if(reader.Exists("server", "analytics-dir")) {+ this->analytics_dir = reader.Get("server", "analytics-dir", "");+ }+if(reader.Exists("server", "api-key")) {this->api_key = reader.Get("server", "api-key", "");}@@ -654,6 +667,10 @@ public:this->log_dir = options.get<std::string>("log-dir");}+ if(options.exist("analytics-dir")) {+ this->analytics_dir = options.get<std::string>("analytics-dir");+ }+if(options.exist("api-key")) {this->api_key = options.get<std::string>("api-key");}@@ -825,6 +842,10 @@ public:return Option<bool>(500, "API key is not specified.");}+ if(analytics_dir.empty()) {+ return Option<bool>(500, "Analytics Directory is not specified.");+ }+return Option<bool>(true);}diff --git a/src/event_manager.cpp b/src/event_manager.cppindex 82b30d87..1f2d30f4 100644--- a/src/event_manager.cpp+++ b/src/event_manager.cpp@@ -75,6 +75,8 @@ bool EventManager::add_event(const nlohmann::json& event) {const std::string& collection = event_data_val["collection"].get<std::string>();AnalyticsManager::get_instance().add_click_event(collection, query, user_id, doc_id, position);+ } else {+ return false;}}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex 9d6c343e..d90c574f 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -63,6 +63,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {options.add<std::string>("data-dir", 'd', "Directory where data will be stored.", true);options.add<std::string>("api-key", 'a', "API key that allows all operations.", true);options.add<std::string>("search-only-api-key", 's', "[DEPRECATED: use API key management end-point] API key that allows only searches.", false);+ options.add<std::string>("analytics-dir", 'n', "Directory where Analytics will be stored.", true);options.add<std::string>("api-address", '\0', "Address to which Typesense API service binds.", false, "0.0.0.0");options.add<uint32_t>("api-port", '\0', "Port on which Typesense API service listens.", false, 8108);@@ -374,7 +375,7 @@ int run_server(const Config & config, const std::string & version, void (*masterstd::string db_dir = config.get_data_dir() + "/db";std::string state_dir = config.get_data_dir() + "/state";std::string meta_dir = config.get_data_dir() + "/meta";- std::string analytics_dir = config.get_data_dir() + "/analytics";+ std::string analytics_dir = config.get_analytics_dir();size_t thread_pool_size = config.get_thread_pool_size();diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cppindex ced59850..bb7e3643 100644--- a/test/analytics_manager_test.cpp+++ b/test/analytics_manager_test.cpp@@ -4,10 +4,11 @@#include <collection_manager.h>#include <analytics_manager.h>#include "collection.h"+#include "core_api.h"class AnalyticsManagerTest : public ::testing::Test {protected:- Store *store;+ Store *store, *analytics_store;CollectionManager& collectionManager = CollectionManager::get_instance();std::atomic<bool> quit = false;@@ -18,15 +19,17 @@ protected:void setupCollection() {std::string state_dir_path = "/tmp/typesense_test/analytics_manager_test";+ std::string analytics_db_path = "/tmp/typesense_test/analytics_db";LOG(INFO) << "Truncating and creating: " << state_dir_path;system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());system("mkdir -p /tmp/typesense_test/models");store = new Store(state_dir_path);+ analytics_store = new Store(analytics_db_path);collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);- analyticsManager.init(store);+ analyticsManager.init(store, analytics_store);}virtual void SetUp() {@@ -36,6 +39,7 @@ protected:virtual void TearDown() {collectionManager.dispose();delete store;+ delete analytics_store;}};@@ -201,3 +205,142 @@ TEST_F(AnalyticsManagerTest, GetAndDeleteSuggestions) {ASSERT_FALSE(missing_rule_op.ok());}+TEST_F(AnalyticsManagerTest, ClickEventsValidation) {+ nlohmann::json titles_schema = R"({+ "name": "titles",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection* titles_coll = collectionManager.create_collection(titles_schema).get();++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ //wrong type+ nlohmann::json event1 = R"({+ "type": "click",+ "data": {+ "q": "technology",+ "collection": "titles",+ "doc_id": "21",+ "position": 2,+ "user_id": "13"+ }+ })"_json;++ req->body = event1.dump();+ ASSERT_FALSE(post_create_event(req, res));++ //missing query param+ nlohmann::json event2 = R"({+ "type": "query_click",+ "data": {+ "collection": "titles",+ "doc_id": "21",+ "position": 2,+ "user_id": "13"+ }+ })"_json;++ req->body = event2.dump();+ ASSERT_FALSE(post_create_event(req, res));++ //should be string type+ nlohmann::json event3 = R"({+ "type": "query_click",+ "data": {+ "q": "technology",+ "collection": "titles",+ "doc_id": 21,+ "position": 2,+ "user_id": "13"+ }+ })"_json;++ req->body = event3.dump();+ ASSERT_FALSE(post_create_event(req, res));++ //correct params+ nlohmann::json event4 = R"({+ "type": "query_click",+ "data": {+ "q": "technology",+ "collection": "titles",+ "doc_id": "21",+ "position": 2,+ "user_id": "13"+ }+ })"_json;++ req->body = event4.dump();+ ASSERT_TRUE(post_create_event(req, res));+}++TEST_F(AnalyticsManagerTest, ClickEventsStoreRetrieveal) {+ nlohmann::json titles_schema = R"({+ "name": "titles",+ "fields": [+ {"name": "title", "type": "string"}+ ]+ })"_json;++ Collection* titles_coll = collectionManager.create_collection(titles_schema).get();++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ nlohmann::json event1 = R"({+ "type": "query_click",+ "data": {+ "q": "technology",+ "collection": "titles",+ "doc_id": "21",+ "position": 2,+ "user_id": "13"+ }+ })"_json;++ req->body = event1.dump();+ ASSERT_TRUE(post_create_event(req, res));++ nlohmann::json event2 = R"({+ "type": "query_click",+ "data": {+ "q": "technology",+ "collection": "titles",+ "doc_id": "21",+ "position": 4,+ "user_id": "11"+ }+ })"_json;++ req->body = event2.dump();+ ASSERT_TRUE(post_create_event(req, res));++ event1["collection_id"] = "0";+ event1["timestamp"] = 1521512521;+ event2["collection_id"] = "0";+ event2["timestamp"] = 1521514354;+ nlohmann::json click_events = nlohmann::json::array();+ click_events.push_back(event1);+ click_events.push_back(event2);++ req->body = click_events.dump();+ ASSERT_TRUE(post_replicate_click_event(req, res));++ auto result = analyticsManager.get_click_events();++ ASSERT_EQ("0", result[0]["collection_id"]);+ ASSERT_EQ("13", result[0]["data"]["user_id"]);+ ASSERT_EQ("21", result[0]["data"]["doc_id"]);+ ASSERT_EQ(2, result[0]["data"]["position"]);+ ASSERT_EQ("technology", result[0]["data"]["q"]);++ ASSERT_EQ("0", result[1]["collection_id"]);+ ASSERT_EQ("11", result[1]["data"]["user_id"]);+ ASSERT_EQ("21", result[1]["data"]["doc_id"]);+ ASSERT_EQ(4, result[1]["data"]["position"]);+ ASSERT_EQ("technology", result[1]["data"]["q"]);+}\ No newline at end of filediff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cppindex 89e89986..5ee10b4e 100644--- a/test/collection_manager_test.cpp+++ b/test/collection_manager_test.cpp@@ -9,7 +9,7 @@class CollectionManagerTest : public ::testing::Test {protected:- Store *store;+ Store *store, *analytics_store;CollectionManager & collectionManager = CollectionManager::get_instance();std::atomic<bool> quit = false;Collection *collection1;@@ -18,14 +18,16 @@ protected:void setupCollection() {std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db";+ std::string analytics_db_path = "/tmp/typesense_test/analytics_db";LOG(INFO) << "Truncating and creating: " << state_dir_path;system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);+ analytics_store = new Store(analytics_db_path);collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);- AnalyticsManager::get_instance().init(store);+ AnalyticsManager::get_instance().init(store, analytics_store);schema = R"({"name": "collection1",diff --git a/test/valid_config.ini b/test/valid_config.iniindex 7bf445d3..037362d7 100644--- a/test/valid_config.ini+++ b/test/valid_config.ini@@ -8,4 +8,6 @@ api-key = 1234log-dir = /tmp/logslisten-port = 9090-enable-cors = True\ No newline at end of file+enable-cors = True++analytics-dir=/tmp/analytics