Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
update stopwords support and tests
krunal368e5c178944
diff --git a/src/stopwords_manager.cpp b/src/stopwords_manager.cppindex 52e3d4f0..69ec9bf7 100644--- a/src/stopwords_manager.cpp+++ b/src/stopwords_manager.cpp@@ -19,7 +19,7 @@ Option<bool> StopwordsManager::get_stopword(const std::string& stopword_name, spreturn Option<bool>(true);}- return Option<bool>(404, "stopword `" + stopword_name +"` not found.");+ return Option<bool>(404, "Stopword `" + stopword_name +"` not found.");}Option<bool> StopwordsManager::upsert_stopword(const std::string& stopword_name, const nlohmann::json& stopwords, const std::string& locale) {@@ -31,17 +31,18 @@ Option<bool> StopwordsManager::upsert_stopword(const std::string& stopword_name,}std::vector<std::string> tokens;+ spp::sparse_hash_set<std::string> stopwords_set;for (const auto &stopword: stopwords.items()) {const auto& val = stopword.value().get<std::string>();Tokenizer(val, true, false, locale, {}, {}).tokenize(tokens);for(const auto& tok : tokens) {- stopword_configs[stopword_name].emplace(tok);+ stopwords_set.emplace(tok);}tokens.clear();}-+ stopword_configs[stopword_name] = stopwords_set;return Option<bool>(true);}diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex 983a3fec..29ea703a 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -1161,219 +1161,4 @@ TEST_F(CoreAPIUtilsTest, TestProxyTimeout) {ASSERT_EQ(408, resp->status_code);ASSERT_EQ("Server error on remote server. Please try again later.", nlohmann::json::parse(resp->body)["message"]);-}--TEST_F(CoreAPIUtilsTest, StopwordsBasics) {- nlohmann::json schema = R"({- "name": "coll1",- "fields": [- {"name": "title", "type": "string" },- {"name": "points", "type": "int32" }- ]- })"_json;-- auto op = collectionManager.create_collection(schema);- ASSERT_TRUE(op.ok());- Collection *coll1 = op.get();-- nlohmann::json doc;- doc["title"] = "The Dark Knight Europe";- doc["points"] = 10;- coll1->add(doc.dump(), CREATE);-- doc["title"] = "An American America";- doc["points"] = 12;- coll1->add(doc.dump(), CREATE);-- doc["title"] = "An the";- doc["points"] = 17;- coll1->add(doc.dump(), CREATE);-- doc["title"] = "A Deadman";- doc["points"] = 13;- coll1->add(doc.dump(), CREATE);-- doc["title"] = "A Village Of The Deadman";- doc["points"] = 20;- coll1->add(doc.dump(), CREATE);-- //when all words in query are stopwords- auto stopword_value = R"(- {"stopwords": ["the", "a", "an"], "locale": "en"}- )"_json;-- std::shared_ptr<http_req> req = std::make_shared<http_req>();- std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);- req->params["collection"] = "coll1";- req->params["name"] = "articles";- req->body = stopword_value.dump();-- auto result = put_upsert_stopword(req, res);- if(!result) {- LOG(ERROR) << res->body;- FAIL();- }-- req->params["collection"] = "coll1";- req->params["q"] = "the";- req->params["query_by"] = "title";- req->params["stopwords"] = "articles";-- nlohmann::json embedded_params;- auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(- std::chrono::system_clock::now().time_since_epoch()).count();- std::string json_results;-- auto search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);- if(!search_op.error().empty()) {- LOG(ERROR) << search_op.error();- }- ASSERT_TRUE(search_op.ok());- nlohmann::json results = nlohmann::json::parse(json_results);- ASSERT_EQ(0, results["hits"].size());-- req->params.clear();- json_results.clear();-- //when not all words in query are stopwords then it should match the remaining words- stopword_value = R"(- {"stopwords": ["america", "europe"], "locale": "en"}- )"_json;-- req->params["collection"] = "coll1";- req->params["name"] = "continents";- req->body = stopword_value.dump();-- result = put_upsert_stopword(req, res);- if(!result) {- LOG(ERROR) << res->body;- FAIL();- }-- req->params["q"] = "America Man";- req->params["query_by"] = "title";- req->params["stopwords"] = "continents";-- search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);- ASSERT_TRUE(search_op.ok());- results = nlohmann::json::parse(json_results);- ASSERT_EQ(0, results["hits"].size());-- req->params.clear();- json_results.clear();-- req->params["collection"] = "coll1";- req->params["q"] = "a deadman";- req->params["query_by"] = "title";- req->params["stopwords"] = "articles";-- search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);- ASSERT_TRUE(search_op.ok());- results = nlohmann::json::parse(json_results);- ASSERT_EQ(2, results["hits"].size());-- req->params.clear();- json_results.clear();-- //try deteting nonexisting stopword- req->params["collection"] = "coll1";- req->params["name"] = "country";-- result = del_stopword(req, res);- ASSERT_EQ(404, res->status_code);- ASSERT_STREQ("{\"message\": \"stopword `country` not found.\"}", res->body.c_str());-- req->params.clear();- json_results.clear();-- //detete stopword and apply in search- req->params["collection"] = "coll1";- req->params["name"] = "continents";-- result = del_stopword(req, res);- if(!result) {- LOG(ERROR) << res->body;- FAIL();- }-- req->params["collection"] = "coll1";- req->params["q"] = "America";- req->params["query_by"] = "title";- req->params["stopwords"] = "continents";-- search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);- ASSERT_TRUE(search_op.ok());- results = nlohmann::json::parse(json_results);- ASSERT_EQ(1, results["hits"].size());-- collectionManager.drop_collection("coll1");-}--TEST_F(CoreAPIUtilsTest, StopwordsValidation) {- nlohmann::json schema = R"({- "name": "coll1",- "fields": [- {"name": "title", "type": "string" },- {"name": "points", "type": "int32" }- ]- })"_json;-- auto op = collectionManager.create_collection(schema);- ASSERT_TRUE(op.ok());- Collection *coll1 = op.get();-- std::shared_ptr<http_req> req = std::make_shared<http_req>();- std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);-- auto stopword_value = R"(- {"stopwords": ["america", "europe"]}- )"_json;-- req->params["collection"] = "coll1";- req->params["name"] = "continents";- req->body = stopword_value.dump();-- auto result = put_upsert_stopword(req, res);- ASSERT_EQ(400, res->status_code);- ASSERT_EQ("{\"message\": \"Parameter `locale` is required\"}", res->body);-- //with a typo- stopword_value = R"(- {"stopword": ["america", "europe"], "locale": "en"}- )"_json;-- req->params["collection"] = "coll1";- req->params["name"] = "continents";- req->body = stopword_value.dump();-- result = put_upsert_stopword(req, res);- ASSERT_EQ(400, res->status_code);- ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required\"}", res->body.c_str());-- //check for value types- stopword_value = R"(- {"stopwords": ["america", "europe"], "locale": 12}- )"_json;-- req->params["collection"] = "coll1";- req->params["name"] = "continents";- req->body = stopword_value.dump();-- result = put_upsert_stopword(req, res);- ASSERT_EQ(400, res->status_code);- ASSERT_STREQ("{\"message\": \"Parameter `locale` is required as string value\"}", res->body.c_str());-- stopword_value = R"(- {"stopwords": [1, 5, 2], "locale": "ko"}- )"_json;-- req->params["collection"] = "coll1";- req->params["name"] = "continents";- req->body = stopword_value.dump();-- result = put_upsert_stopword(req, res);- ASSERT_EQ(400, res->status_code);- ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required as string array value\"}", res->body.c_str());-- collectionManager.drop_collection("coll1");}\ No newline at end of filediff --git a/test/stopwords_manager_test.cpp b/test/stopwords_manager_test.cppindex d21d3324..76b406f5 100644--- a/test/stopwords_manager_test.cpp+++ b/test/stopwords_manager_test.cpp@@ -1,17 +1,26 @@#include <gtest/gtest.h>-#include "include/sparsepp.h"-#include "include/stopwords_manager.h"-#include "include/store.h"+#include "collection.h"+#include <vector>+#include <collection_manager.h>+#include <core_api.h>+#include "stopwords_manager.h"class StopwordsManagerTest : public ::testing::Test {protected:Store *store;+ CollectionManager & collectionManager = CollectionManager::get_instance();+ StopwordsManager& stopwordsManager = StopwordsManager::get_instance();+ std::atomic<bool> quit = false;virtual void SetUp() {std::string state_dir_path = "/tmp/typesense_test/stopwords_manager";LOG(INFO) << "Truncating and creating: " << state_dir_path;system(("rm -rf "+state_dir_path+" && mkdir -p "+state_dir_path).c_str());store = new Store(state_dir_path);++ collectionManager.init(store, 1.0, "auth_key", quit);+ collectionManager.load(8, 1000);+ stopwordsManager.init(store);}virtual void TearDown() {@@ -20,9 +29,6 @@ protected:};TEST_F(StopwordsManagerTest, UpsertGetStopwords) {- StopwordsManager stopwordsManager;- stopwordsManager.init(store);-auto stopwords1 = R"({"stopwords": ["america", "europe"]})"_json;@@ -46,15 +52,28 @@ TEST_F(StopwordsManagerTest, UpsertGetStopwords) {auto stopword_config = stopwordsManager.get_stopwords();ASSERT_EQ(3, stopword_config.size()); //total stopwords set+ ASSERT_TRUE(stopword_config.find("countries") != stopword_config.end());+ ASSERT_TRUE(stopword_config.find("articles") != stopword_config.end());+ ASSERT_TRUE(stopword_config.find("continents") != stopword_config.end());+ASSERT_EQ(3, stopword_config["articles"].size());+ ASSERT_TRUE(stopword_config["articles"].find("a") != stopword_config["articles"].end());+ ASSERT_TRUE(stopword_config["articles"].find("an") != stopword_config["articles"].end());+ ASSERT_TRUE(stopword_config["articles"].find("the") != stopword_config["articles"].end());+ASSERT_EQ(2, stopword_config["continents"].size());+ ASSERT_TRUE(stopword_config["continents"].find("america") != stopword_config["continents"].end());+ ASSERT_TRUE(stopword_config["continents"].find("europe") != stopword_config["continents"].end());+ASSERT_EQ(5, stopword_config["countries"].size()); //with tokenization United States will be splited into two+ ASSERT_TRUE(stopword_config["countries"].find("india") != stopword_config["countries"].end());+ ASSERT_TRUE(stopword_config["countries"].find("united") != stopword_config["countries"].end());+ ASSERT_TRUE(stopword_config["countries"].find("states") != stopword_config["countries"].end());+ ASSERT_TRUE(stopword_config["countries"].find("china") != stopword_config["countries"].end());+ ASSERT_TRUE(stopword_config["countries"].find("japan") != stopword_config["countries"].end());}TEST_F(StopwordsManagerTest, GetStopword) {- StopwordsManager stopwordsManager;- stopwordsManager.init(store);-auto stopwords = R"({"stopwords": ["a", "an", "the"]})"_json;auto upsert_op = stopwordsManager.upsert_stopword("articles", stopwords["stopwords"], "en");@@ -72,7 +91,7 @@ TEST_F(StopwordsManagerTest, GetStopword) {get_op = stopwordsManager.get_stopword("country", stopwords_set);ASSERT_FALSE(get_op.ok());ASSERT_EQ(404, get_op.code());- ASSERT_EQ("stopword `country` not found.", get_op.error());+ ASSERT_EQ("Stopword `country` not found.", get_op.error());//try fetching stopwords with tokenstopwords = R"({"stopwords": ["India", "United States", "Japan"]})"_json;@@ -86,9 +105,6 @@ TEST_F(StopwordsManagerTest, GetStopword) {}TEST_F(StopwordsManagerTest, DeleteStopword) {- StopwordsManager stopwordsManager;- stopwordsManager.init(store);-auto stopwords1 = R"({"stopwords": ["america", "europe"]})"_json;@@ -112,11 +128,255 @@ TEST_F(StopwordsManagerTest, DeleteStopword) {auto get_op = stopwordsManager.get_stopword("articles", stopwords_set);ASSERT_FALSE(get_op.ok());ASSERT_EQ(404, get_op.code());- ASSERT_EQ("stopword `articles` not found.", get_op.error());+ ASSERT_EQ("Stopword `articles` not found.", get_op.error());//delete non-existing stopworddel_op = stopwordsManager.delete_stopword("states");ASSERT_FALSE(del_op.ok());ASSERT_EQ(404, del_op.code());ASSERT_EQ("Stopword `states` not found.", del_op.error());+}++TEST_F(StopwordsManagerTest, UpdateStopword) {+ auto stopwords = R"(+ {"stopwords": ["america", "europe"]}+ )"_json;++ auto upsert_op = stopwordsManager.upsert_stopword("continents", stopwords["stopwords"], "en");+ ASSERT_TRUE(upsert_op.ok());++ auto stopword_config = stopwordsManager.get_stopwords();++ ASSERT_EQ(2, stopword_config["continents"].size());+ ASSERT_TRUE(stopword_config["continents"].find("america") != stopword_config["continents"].end());+ ASSERT_TRUE(stopword_config["continents"].find("europe") != stopword_config["continents"].end());++ //adding new words with same name should replace the stopwords set+ stopwords = R"(+ {"stopwords": ["india", "china", "japan"]}+ )"_json;+ upsert_op = stopwordsManager.upsert_stopword("continents", stopwords["stopwords"], "en");+ ASSERT_TRUE(upsert_op.ok());++ stopword_config = stopwordsManager.get_stopwords();++ ASSERT_EQ(3, stopword_config["continents"].size());+ ASSERT_TRUE(stopword_config["continents"].find("china") != stopword_config["continents"].end());+ ASSERT_TRUE(stopword_config["continents"].find("india") != stopword_config["continents"].end());+ ASSERT_TRUE(stopword_config["continents"].find("japan") != stopword_config["continents"].end());+}++TEST_F(StopwordsManagerTest, StopwordsBasics) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string" },+ {"name": "points", "type": "int32" }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection *coll1 = op.get();++ nlohmann::json doc;+ doc["title"] = "The Dark Knight Europe";+ doc["points"] = 10;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "An American America";+ doc["points"] = 12;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "An the";+ doc["points"] = 17;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "A Deadman";+ doc["points"] = 13;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "A Village Of The Deadman";+ doc["points"] = 20;+ coll1->add(doc.dump(), CREATE);++ //when all words in query are stopwords+ auto stopword_value = R"(+ {"stopwords": ["the", "a", "an"], "locale": "en"}+ )"_json;++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);+ req->params["collection"] = "coll1";+ req->params["name"] = "articles";+ req->body = stopword_value.dump();++ auto result = put_upsert_stopword(req, res);+ if(!result) {+ LOG(ERROR) << res->body;+ FAIL();+ }++ req->params["collection"] = "coll1";+ req->params["q"] = "the";+ req->params["query_by"] = "title";+ req->params["stopwords"] = "articles";++ nlohmann::json embedded_params;+ auto now_ts = std::chrono::duration_cast<std::chrono::microseconds>(+ std::chrono::system_clock::now().time_since_epoch()).count();+ std::string json_results;++ auto search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);+ if(!search_op.error().empty()) {+ LOG(ERROR) << search_op.error();+ }+ ASSERT_TRUE(search_op.ok());+ nlohmann::json results = nlohmann::json::parse(json_results);+ ASSERT_EQ(0, results["hits"].size());++ req->params.clear();+ json_results.clear();++ //when not all words in query are stopwords then it should match the remaining words+ stopword_value = R"(+ {"stopwords": ["america", "europe"], "locale": "en"}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ if(!result) {+ LOG(ERROR) << res->body;+ FAIL();+ }++ req->params["q"] = "America Man";+ req->params["query_by"] = "title";+ req->params["stopwords"] = "continents";++ search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);+ ASSERT_TRUE(search_op.ok());+ results = nlohmann::json::parse(json_results);+ ASSERT_EQ(0, results["hits"].size());++ req->params.clear();+ json_results.clear();++ req->params["collection"] = "coll1";+ req->params["q"] = "a deadman";+ req->params["query_by"] = "title";+ req->params["stopwords"] = "articles";++ search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);+ ASSERT_TRUE(search_op.ok());+ results = nlohmann::json::parse(json_results);+ ASSERT_EQ(2, results["hits"].size());++ req->params.clear();+ json_results.clear();++ //try deteting nonexisting stopword+ req->params["collection"] = "coll1";+ req->params["name"] = "state";++ result = del_stopword(req, res);+ ASSERT_EQ(404, res->status_code);+ ASSERT_STREQ("{\"message\": \"Stopword `state` not found.\"}", res->body.c_str());++ req->params.clear();+ json_results.clear();++ //detete stopword and apply in search+ req->params["collection"] = "coll1";+ req->params["name"] = "continents";++ result = del_stopword(req, res);+ if(!result) {+ LOG(ERROR) << res->body;+ FAIL();+ }++ req->params["collection"] = "coll1";+ req->params["q"] = "America";+ req->params["query_by"] = "title";+ req->params["stopwords"] = "continents";++ search_op = collectionManager.do_search(req->params, embedded_params, json_results, now_ts);+ ASSERT_TRUE(search_op.ok());+ results = nlohmann::json::parse(json_results);+ ASSERT_EQ(1, results["hits"].size());++ collectionManager.drop_collection("coll1");+}++TEST_F(StopwordsManagerTest, StopwordsValidation) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string" },+ {"name": "points", "type": "int32" }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection *coll1 = op.get();++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ auto stopword_value = R"(+ {"stopwords": ["america", "europe"]}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ auto result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_EQ("{\"message\": \"Parameter `locale` is required\"}", res->body);++ //with a typo+ stopword_value = R"(+ {"stopword": ["america", "europe"], "locale": "en"}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required\"}", res->body.c_str());++ //check for value types+ stopword_value = R"(+ {"stopwords": ["america", "europe"], "locale": 12}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_STREQ("{\"message\": \"Parameter `locale` is required as string value\"}", res->body.c_str());++ stopword_value = R"(+ {"stopwords": [1, 5, 2], "locale": "ko"}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required as string array value\"}", res->body.c_str());++ collectionManager.drop_collection("coll1");}\ No newline at end of file