Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

add analytics db and cmdline, clickevent tests

krunalb86d7308e5b1
diff --git a/include/tsconfig.h b/include/tsconfig.h
index 5e754d84..1de30f2b 100644
--- a/include/tsconfig.h
+++ b/include/tsconfig.h
@@ -11,6 +11,7 @@ class Config {
private:
std::string data_dir;
std::string log_dir;
+ std::string analytics_dir;
std::string api_key;
@@ -124,6 +125,10 @@ public:
this->log_dir = log_dir;
}
+ void set_analytics_dir(const std::string& analytics_dir) {
+ this->analytics_dir = analytics_dir;
+ }
+
void set_api_key(const std::string & api_key) {
this->api_key = api_key;
}
@@ -191,6 +196,9 @@ public:
return this->log_dir;
}
+ std::string get_analytics_dir() const {
+ return this->analytics_dir;
+ }
std::string get_api_key() const {
return this->api_key;
@@ -350,6 +358,7 @@ public:
void load_config_env() {
this->data_dir = get_env("TYPESENSE_DATA_DIR");
this->log_dir = get_env("TYPESENSE_LOG_DIR");
+ this->analytics_dir = get_env("TYPESENSE_ANALYTICS_DIR");
this->api_key = get_env("TYPESENSE_API_KEY");
// @deprecated
@@ -495,6 +504,10 @@ public:
this->log_dir = reader.Get("server", "log-dir", "");
}
+ if(reader.Exists("server", "analytics-dir")) {
+ this->analytics_dir = reader.Get("server", "analytics-dir", "");
+ }
+
if(reader.Exists("server", "api-key")) {
this->api_key = reader.Get("server", "api-key", "");
}
@@ -654,6 +667,10 @@ public:
this->log_dir = options.get<std::string>("log-dir");
}
+ if(options.exist("analytics-dir")) {
+ this->analytics_dir = options.get<std::string>("analytics-dir");
+ }
+
if(options.exist("api-key")) {
this->api_key = options.get<std::string>("api-key");
}
@@ -825,6 +842,10 @@ public:
return Option<bool>(500, "API key is not specified.");
}
+ if(analytics_dir.empty()) {
+ return Option<bool>(500, "Analytics Directory is not specified.");
+ }
+
return Option<bool>(true);
}
diff --git a/src/event_manager.cpp b/src/event_manager.cpp
index 82b30d87..1f2d30f4 100644
--- a/src/event_manager.cpp
+++ b/src/event_manager.cpp
@@ -75,6 +75,8 @@ bool EventManager::add_event(const nlohmann::json& event) {
const std::string& collection = event_data_val["collection"].get<std::string>();
AnalyticsManager::get_instance().add_click_event(collection, query, user_id, doc_id, position);
+ } else {
+ return false;
}
}
diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cpp
index 9d6c343e..d90c574f 100644
--- a/src/typesense_server_utils.cpp
+++ b/src/typesense_server_utils.cpp
@@ -63,6 +63,7 @@ void init_cmdline_options(cmdline::parser & options, int argc, char **argv) {
options.add<std::string>("data-dir", 'd', "Directory where data will be stored.", true);
options.add<std::string>("api-key", 'a', "API key that allows all operations.", true);
options.add<std::string>("search-only-api-key", 's', "[DEPRECATED: use API key management end-point] API key that allows only searches.", false);
+ options.add<std::string>("analytics-dir", 'n', "Directory where Analytics will be stored.", true);
options.add<std::string>("api-address", '\0', "Address to which Typesense API service binds.", false, "0.0.0.0");
options.add<uint32_t>("api-port", '\0', "Port on which Typesense API service listens.", false, 8108);
@@ -374,7 +375,7 @@ int run_server(const Config & config, const std::string & version, void (*master
std::string db_dir = config.get_data_dir() + "/db";
std::string state_dir = config.get_data_dir() + "/state";
std::string meta_dir = config.get_data_dir() + "/meta";
- std::string analytics_dir = config.get_data_dir() + "/analytics";
+ std::string analytics_dir = config.get_analytics_dir();
size_t thread_pool_size = config.get_thread_pool_size();
diff --git a/test/analytics_manager_test.cpp b/test/analytics_manager_test.cpp
index ced59850..bb7e3643 100644
--- a/test/analytics_manager_test.cpp
+++ b/test/analytics_manager_test.cpp
@@ -4,10 +4,11 @@
#include <collection_manager.h>
#include <analytics_manager.h>
#include "collection.h"
+#include "core_api.h"
class AnalyticsManagerTest : public ::testing::Test {
protected:
- Store *store;
+ Store *store, *analytics_store;
CollectionManager& collectionManager = CollectionManager::get_instance();
std::atomic<bool> quit = false;
@@ -18,15 +19,17 @@ protected:
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/analytics_manager_test";
+ std::string analytics_db_path = "/tmp/typesense_test/analytics_db";
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
system("mkdir -p /tmp/typesense_test/models");
store = new Store(state_dir_path);
+ analytics_store = new Store(analytics_db_path);
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
- analyticsManager.init(store);
+ analyticsManager.init(store, analytics_store);
}
virtual void SetUp() {
@@ -36,6 +39,7 @@ protected:
virtual void TearDown() {
collectionManager.dispose();
delete store;
+ delete analytics_store;
}
};
@@ -201,3 +205,142 @@ TEST_F(AnalyticsManagerTest, GetAndDeleteSuggestions) {
ASSERT_FALSE(missing_rule_op.ok());
}
+TEST_F(AnalyticsManagerTest, ClickEventsValidation) {
+ nlohmann::json titles_schema = R"({
+ "name": "titles",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* titles_coll = collectionManager.create_collection(titles_schema).get();
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ //wrong type
+ nlohmann::json event1 = R"({
+ "type": "click",
+ "data": {
+ "q": "technology",
+ "collection": "titles",
+ "doc_id": "21",
+ "position": 2,
+ "user_id": "13"
+ }
+ })"_json;
+
+ req->body = event1.dump();
+ ASSERT_FALSE(post_create_event(req, res));
+
+ //missing query param
+ nlohmann::json event2 = R"({
+ "type": "query_click",
+ "data": {
+ "collection": "titles",
+ "doc_id": "21",
+ "position": 2,
+ "user_id": "13"
+ }
+ })"_json;
+
+ req->body = event2.dump();
+ ASSERT_FALSE(post_create_event(req, res));
+
+ //should be string type
+ nlohmann::json event3 = R"({
+ "type": "query_click",
+ "data": {
+ "q": "technology",
+ "collection": "titles",
+ "doc_id": 21,
+ "position": 2,
+ "user_id": "13"
+ }
+ })"_json;
+
+ req->body = event3.dump();
+ ASSERT_FALSE(post_create_event(req, res));
+
+ //correct params
+ nlohmann::json event4 = R"({
+ "type": "query_click",
+ "data": {
+ "q": "technology",
+ "collection": "titles",
+ "doc_id": "21",
+ "position": 2,
+ "user_id": "13"
+ }
+ })"_json;
+
+ req->body = event4.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+}
+
+TEST_F(AnalyticsManagerTest, ClickEventsStoreRetrieveal) {
+ nlohmann::json titles_schema = R"({
+ "name": "titles",
+ "fields": [
+ {"name": "title", "type": "string"}
+ ]
+ })"_json;
+
+ Collection* titles_coll = collectionManager.create_collection(titles_schema).get();
+
+ std::shared_ptr<http_req> req = std::make_shared<http_req>();
+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);
+
+ nlohmann::json event1 = R"({
+ "type": "query_click",
+ "data": {
+ "q": "technology",
+ "collection": "titles",
+ "doc_id": "21",
+ "position": 2,
+ "user_id": "13"
+ }
+ })"_json;
+
+ req->body = event1.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+
+ nlohmann::json event2 = R"({
+ "type": "query_click",
+ "data": {
+ "q": "technology",
+ "collection": "titles",
+ "doc_id": "21",
+ "position": 4,
+ "user_id": "11"
+ }
+ })"_json;
+
+ req->body = event2.dump();
+ ASSERT_TRUE(post_create_event(req, res));
+
+ event1["collection_id"] = "0";
+ event1["timestamp"] = 1521512521;
+ event2["collection_id"] = "0";
+ event2["timestamp"] = 1521514354;
+ nlohmann::json click_events = nlohmann::json::array();
+ click_events.push_back(event1);
+ click_events.push_back(event2);
+
+ req->body = click_events.dump();
+ ASSERT_TRUE(post_replicate_click_event(req, res));
+
+ auto result = analyticsManager.get_click_events();
+
+ ASSERT_EQ("0", result[0]["collection_id"]);
+ ASSERT_EQ("13", result[0]["data"]["user_id"]);
+ ASSERT_EQ("21", result[0]["data"]["doc_id"]);
+ ASSERT_EQ(2, result[0]["data"]["position"]);
+ ASSERT_EQ("technology", result[0]["data"]["q"]);
+
+ ASSERT_EQ("0", result[1]["collection_id"]);
+ ASSERT_EQ("11", result[1]["data"]["user_id"]);
+ ASSERT_EQ("21", result[1]["data"]["doc_id"]);
+ ASSERT_EQ(4, result[1]["data"]["position"]);
+ ASSERT_EQ("technology", result[1]["data"]["q"]);
+}
\ No newline at end of file
diff --git a/test/collection_manager_test.cpp b/test/collection_manager_test.cpp
index 89e89986..5ee10b4e 100644
--- a/test/collection_manager_test.cpp
+++ b/test/collection_manager_test.cpp
@@ -9,7 +9,7 @@
class CollectionManagerTest : public ::testing::Test {
protected:
- Store *store;
+ Store *store, *analytics_store;
CollectionManager & collectionManager = CollectionManager::get_instance();
std::atomic<bool> quit = false;
Collection *collection1;
@@ -18,14 +18,16 @@ protected:
void setupCollection() {
std::string state_dir_path = "/tmp/typesense_test/coll_manager_test_db";
+ std::string analytics_db_path = "/tmp/typesense_test/analytics_db";
LOG(INFO) << "Truncating and creating: " << state_dir_path;
system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());
store = new Store(state_dir_path);
+ analytics_store = new Store(analytics_db_path);
collectionManager.init(store, 1.0, "auth_key", quit);
collectionManager.load(8, 1000);
- AnalyticsManager::get_instance().init(store);
+ AnalyticsManager::get_instance().init(store, analytics_store);
schema = R"({
"name": "collection1",
diff --git a/test/valid_config.ini b/test/valid_config.ini
index 7bf445d3..037362d7 100644
--- a/test/valid_config.ini
+++ b/test/valid_config.ini
@@ -8,4 +8,6 @@ api-key = 1234
log-dir = /tmp/logs
listen-port = 9090
-enable-cors = True
\ No newline at end of file
+enable-cors = True
+
+analytics-dir=/tmp/analytics