Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
refactor and adding tests
krunal1313a2796450aed9
diff --git a/include/collection.h b/include/collection.hindex 960e25df..1998f66c 100644--- a/include/collection.h+++ b/include/collection.h@@ -391,7 +391,7 @@ public:void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,std::vector<std::vector<std::string>>& q_exclude_tokens,std::vector<std::vector<std::string>>& q_phrases,- const std::string& locale, const bool already_segmented, const std::string& stopword) const;+ const std::string& locale, const bool already_segmented, const std::string& stopword_set="") const;// PUBLIC OPERATIONS@@ -592,3 +592,4 @@ bool Collection::highlight_nested_field(const nlohmann::json& hdoc, nlohmann::js}}+diff --git a/include/collection_manager.h b/include/collection_manager.hindex 7c355241..02b4dcbe 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -68,8 +68,6 @@ private:spp::sparse_hash_map<std::string, nlohmann::json> preset_configs;- spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> stopword_configs;-// Auto incrementing ID assigned to each collection// Using a ID instead of a collection's name makes renaming possiblestd::atomic<uint32_t> next_collection_id;@@ -103,7 +101,6 @@ public:static constexpr const char* SYMLINK_PREFIX = "$SL";static constexpr const char* PRESET_PREFIX = "$PS";static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";- static constexpr const char* STOPWORD_PREFIX = "$SW";static CollectionManager & get_instance() {static CollectionManager instance;@@ -204,15 +201,4 @@ public:Option<bool> upsert_preset(const std::string & preset_name, const nlohmann::json& preset_config);Option<bool> delete_preset(const std::string & preset_name);-- //stopwords- spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> get_stopwords() const;-- Option<bool> get_stopword(const std::string&, spp::sparse_hash_set<std::string>&) const;-- Option<bool> upsert_stopword(const std::string&, const nlohmann::json&, const std::string&);-- Option<bool> delete_stopword(const std::string&);-- static std::string get_stopword_key(const std::string & stopword_name);-};\ No newline at end of file+};diff --git a/include/stopwords_manager.h b/include/stopwords_manager.hnew file mode 100644index 00000000..90a461a9--- /dev/null+++ b/include/stopwords_manager.h@@ -0,0 +1,42 @@+#pragma once++#include "sparsepp.h"+#include "option.h"+#include "json.hpp"+#include "shared_mutex"+#include "mutex"+#include "store.h"++class StopwordsManager{+private:+ spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> stopword_configs;++ static std::string get_stopword_key(const std::string & stopword_name);++ mutable std::shared_mutex mutex;++ Store* store = nullptr;+public:+ StopwordsManager() = default;++ ~StopwordsManager() {+ stopword_configs.clear();+ }++ static constexpr const char* STOPWORD_PREFIX = "$SW";++ static StopwordsManager& get_instance() {+ static StopwordsManager instance;+ return instance;+ }++ void init(Store* store);++ spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> get_stopwords() const;++ Option<bool> get_stopword(const std::string&, spp::sparse_hash_set<std::string>&) const;++ Option<bool> upsert_stopword(const std::string&, const nlohmann::json&, const std::string&);++ Option<bool> delete_stopword(const std::string&);+};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex 25d4a655..21d4086a 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -18,6 +18,7 @@#include "thread_local_vars.h"#include "vector_query_ops.h"#include "text_embedder_manager.h"+#include "stopwords_manager.h"const std::string override_t::MATCH_EXACT = "exact";const std::string override_t::MATCH_CONTAINS = "contains";@@ -2430,7 +2431,7 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::sstd::vector<std::string> tokens;spp::sparse_hash_set<std::string> stopwords_list;if(!stopwords_set.empty()) {- const auto &stopword_op = CollectionManager::get_instance().get_stopword(stopwords_set, stopwords_list);+ const auto &stopword_op = StopwordsManager::get_instance().get_stopword(stopwords_set, stopwords_list);if (!stopword_op.ok()) {LOG(ERROR) << "Error fetching stopword_list for stopword " << stopwords_set << " "<<stopword_op.error();}@@ -4878,3 +4879,4 @@ Option<bool> Collection::truncate_after_top_k(const string &field_name, size_t kreturn Option<bool>(true);}+diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 0e8910cb..40a43983 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -316,7 +316,6 @@ void CollectionManager::dispose() {collections.clear();collection_symlinks.clear();preset_configs.clear();- stopword_configs.clear();store->close();}@@ -754,7 +753,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re}//check if stopword set is supplied- const auto stopword_it = req_params.find("stopword");+ const auto stopword_it = req_params.find("stopwords");std::string stopwords_set="";if(stopword_it != req_params.end()) {@@ -1477,63 +1476,6 @@ Option<bool> CollectionManager::delete_preset(const string& preset_name) {return Option<bool>(true);}--spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> CollectionManager::get_stopwords() const {- std::shared_lock lock(mutex);- return stopword_configs;-}--Option<bool> CollectionManager::get_stopword(const string& stopword_name, spp::sparse_hash_set<std::string>& stopwords) const {- std::shared_lock lock(mutex);-- const auto& it = stopword_configs.find(stopword_name);- if(it != stopword_configs.end()) {- stopwords = it->second;- return Option<bool>(true);- }-- return Option<bool>(404, "Not found.");-}--Option<bool> CollectionManager::upsert_stopword(const string& stopword_name, const nlohmann::json& stopwords, const std::string& locale) {- std::unique_lock lock(mutex);-- bool inserted = store->insert(get_stopword_key(stopword_name), stopwords.dump());- if(!inserted) {- return Option<bool>(500, "Unable to insert into store.");- }-- std::vector<std::string> tokens;-- for (const auto &stopword: stopwords.items()) {- const auto& val = stopword.value().get<std::string>();- Tokenizer(val, true, false, locale, {}, {}).tokenize(tokens);-- for(const auto& tok : tokens) {- stopword_configs[stopword_name].emplace(tok);- }- tokens.clear();- }-- return Option<bool>(true);-}--std::string CollectionManager::get_stopword_key(const string& stopword_name) {- return std::string(STOPWORD_PREFIX) + "_" + stopword_name;-}--Option<bool> CollectionManager::delete_stopword(const string& stopword_name) {- std::unique_lock lock(mutex);-- bool removed = store->remove(get_stopword_key(stopword_name));- if(!removed) {- return Option<bool>(500, "Unable to delete from store.");- }-- stopword_configs.erase(stopword_name);- return Option<bool>(true);-}-Option<Collection*> CollectionManager::clone_collection(const string& existing_name, const nlohmann::json& req_json) {std::shared_lock lock(mutex);@@ -1593,3 +1535,4 @@ Option<Collection*> CollectionManager::clone_collection(const string& existing_nreturn Option<Collection*>(new_coll);}+diff --git a/src/core_api.cpp b/src/core_api.cppindex 80eada4c..a51f24c0 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -16,6 +16,7 @@#include "ratelimit_manager.h"#include "event_manager.h"#include "http_proxy.h"+#include "include/stopwords_manager.h"using namespace std::chrono_literals;@@ -1850,8 +1851,8 @@ bool del_preset(const std::shared_ptr<http_req>& req, const std::shared_ptr<http}bool get_stopwords(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {- CollectionManager & collectionManager = CollectionManager::get_instance();- const spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>>& stopwords = collectionManager.get_stopwords();+ StopwordsManager& stopwordManager = StopwordsManager::get_instance();+ const spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>>& stopwords = stopwordManager.get_stopwords();nlohmann::json res_json = nlohmann::json::object();res_json["stopwords"] = nlohmann::json::array();@@ -1870,10 +1871,10 @@ bool get_stopwords(const std::shared_ptr<http_req>& req, const std::shared_ptr<hbool get_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {const std::string & stopword_name = req->params["stopword_name"];- CollectionManager & collectionManager = CollectionManager::get_instance();+ StopwordsManager& stopwordManager = StopwordsManager::get_instance();spp::sparse_hash_set<std::string> stopwords;- Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopwords);+ Option<bool> stopword_op = stopwordManager.get_stopword(stopword_name, stopwords);if(!stopword_op.ok()) {res->set(stopword_op.code(), stopword_op.error());@@ -1901,18 +1902,33 @@ bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::sharedreturn false;}- CollectionManager & collectionManager = CollectionManager::get_instance();+ StopwordsManager& stopwordManager = StopwordsManager::get_instance();const std::string & stopword_name = req->params["name"];const char* STOPWORD_VALUES = "stopwords";const char* STOPWORD_LOCALE = "locale";- if(req_json.count(STOPWORD_VALUES) == 0) {- res->set_400(std::string("Parameter `") + STOPWORD_VALUES + "` is required.");+ if(req_json.count(STOPWORD_VALUES) == 0){+ res->set_400(std::string("Parameter `") + STOPWORD_VALUES + "` is required");+ return false;+ }++ if((!req_json[STOPWORD_VALUES].is_array()) || (!req_json[STOPWORD_VALUES][0].is_string())) {+ res->set_400(std::string("Parameter `") + STOPWORD_VALUES + "` is required as string array value");+ return false;+ }++ if(req_json.count(STOPWORD_LOCALE) == 0) {+ res->set_400(std::string("Parameter `") + STOPWORD_LOCALE + "` is required");return false;}- Option<bool> success_op = collectionManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUES], req_json[STOPWORD_LOCALE]);+ if(!req_json[STOPWORD_LOCALE].is_string()) {+ res->set_400(std::string("Parameter `") + STOPWORD_LOCALE + "` is required as string value");+ return false;+ }++ Option<bool> success_op = stopwordManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUES], req_json[STOPWORD_LOCALE]);if(!success_op.ok()) {res->set_500(success_op.error());return false;@@ -1926,16 +1942,16 @@ bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::sharedbool del_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {const std::string & stopword_name = req->params["name"];- CollectionManager & collectionManager = CollectionManager::get_instance();+ StopwordsManager& stopwordManager = StopwordsManager::get_instance();spp::sparse_hash_set<std::string> stopwords;- Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopwords);+ Option<bool> stopword_op = stopwordManager.get_stopword(stopword_name, stopwords);if(!stopword_op.ok()) {res->set(stopword_op.code(), stopword_op.error());return false;}- Option<bool> delete_op = collectionManager.delete_stopword(stopword_name);+ Option<bool> delete_op = stopwordManager.delete_stopword(stopword_name);if(!delete_op.ok()) {res->set_500(delete_op.error());@@ -2256,4 +2272,4 @@ bool post_proxy(const std::shared_ptr<http_req>& req, const std::shared_ptr<httpres->set_200(response.body);return true;-}\ No newline at end of file+}diff --git a/src/stopwords_manager.cpp b/src/stopwords_manager.cppnew file mode 100644index 00000000..a29f6c18--- /dev/null+++ b/src/stopwords_manager.cpp@@ -0,0 +1,62 @@+#include "include/stopwords_manager.h"+#include "include/tokenizer.h"++void StopwordsManager::init(Store* _store) {+ store = _store;+}++spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> StopwordsManager::get_stopwords() const {+ std::shared_lock lock(mutex);+ return stopword_configs;+}++Option<bool> StopwordsManager::get_stopword(const std::string& stopword_name, spp::sparse_hash_set<std::string>& stopwords) const {+ std::shared_lock lock(mutex);++ const auto& it = stopword_configs.find(stopword_name);+ if(it != stopword_configs.end()) {+ stopwords = it->second;+ return Option<bool>(true);+ }++ return Option<bool>(404, "Not found.");+}++Option<bool> StopwordsManager::upsert_stopword(const std::string& stopword_name, const nlohmann::json& stopwords, const std::string& locale) {+ std::unique_lock lock(mutex);++ bool inserted = store->insert(get_stopword_key(stopword_name), stopwords.dump());+ if(!inserted) {+ return Option<bool>(500, "Unable to insert into store.");+ }++ std::vector<std::string> tokens;++ for (const auto &stopword: stopwords.items()) {+ const auto& val = stopword.value().get<std::string>();+ Tokenizer(val, true, false, locale, {}, {}).tokenize(tokens);++ for(const auto& tok : tokens) {+ stopword_configs[stopword_name].emplace(tok);+ }+ tokens.clear();+ }++ return Option<bool>(true);+}++std::string StopwordsManager::get_stopword_key(const std::string& stopword_name) {+ return std::string(STOPWORD_PREFIX) + "_" + stopword_name;+}++Option<bool> StopwordsManager::delete_stopword(const std::string& stopword_name) {+ std::unique_lock lock(mutex);++ bool removed = store->remove(get_stopword_key(stopword_name));+ if(!removed) {+ return Option<bool>(500, "Unable to delete from store.");+ }++ stopword_configs.erase(stopword_name);+ return Option<bool>(true);+}diff --git a/src/typesense_server_utils.cpp b/src/typesense_server_utils.cppindex c628c569..3c96e25b 100644--- a/src/typesense_server_utils.cpp+++ b/src/typesense_server_utils.cpp@@ -21,6 +21,7 @@#include "typesense_server_utils.h"#include "file_utils.h"#include "threadpool.h"+#include "stopwords_manager.h"#ifndef ASAN_BUILD#include "jemalloc.h"@@ -424,7 +425,10 @@ int run_server(const Config & config, const std::string & version, void (*masterCollectionManager & collectionManager = CollectionManager::get_instance();collectionManager.init(&store, &app_thread_pool, config.get_max_memory_ratio(),config.get_api_key(), quit_raft_service, batch_indexer);-++ StopwordsManager& stopwordsManager = StopwordsManager::get_instance();+ stopwordsManager.init(&store);+RateLimitManager *rateLimitManager = RateLimitManager::getInstance();auto rate_limit_manager_init = rateLimitManager->init(&meta_store);@@ -523,3 +527,4 @@ int run_server(const Config & config, const std::string & version, void (*masterreturn ret_code;}+diff --git a/test/core_api_utils_test.cpp b/test/core_api_utils_test.cppindex 95af7546..9221dbce 100644--- a/test/core_api_utils_test.cpp+++ b/test/core_api_utils_test.cpp@@ -4,11 +4,13 @@#include <collection_manager.h>#include <core_api.h>#include "core_api_utils.h"+#include "stopwords_manager.h"class CoreAPIUtilsTest : public ::testing::Test {protected:Store *store;CollectionManager & collectionManager = CollectionManager::get_instance();+ StopwordsManager& stopwordsManager = StopwordsManager::get_instance();std::atomic<bool> quit = false;std::vector<std::string> query_fields;@@ -22,6 +24,8 @@ protected:store = new Store(state_dir_path);collectionManager.init(store, 1.0, "auth_key", quit);collectionManager.load(8, 1000);++ stopwordsManager.init(store);}virtual void SetUp() {@@ -1116,4 +1120,202 @@ TEST_F(CoreAPIUtilsTest, TestProxyInvalid) {ASSERT_EQ(400, resp->status_code);ASSERT_EQ("Headers must be a JSON object.", nlohmann::json::parse(resp->body)["message"]);-}\ No newline at end of file+}++TEST_F(CoreAPIUtilsTest, StopwordsBasics) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string" },+ {"name": "points", "type": "int32" }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection *coll1 = op.get();++ nlohmann::json doc;+ doc["title"] = "The Dark Knight Europe";+ doc["points"] = 10;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "An American America";+ doc["points"] = 12;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "An the";+ doc["points"] = 17;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "A Deadman";+ doc["points"] = 13;+ coll1->add(doc.dump(), CREATE);++ doc["title"] = "A Village Of The Deadman";+ doc["points"] = 20;+ coll1->add(doc.dump(), CREATE);++ //when all words in query are stopwords+ auto stopword_value = R"(+ {"stopwords": ["the", "a", "an"], "locale": "en"}+ )"_json;++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);+ req->params["collection"] = "coll1";+ req->params["name"] = "articles";+ req->body = stopword_value.dump();++ auto result = put_upsert_stopword(req, res);+ if(!result) {+ LOG(ERROR) << res->body;+ FAIL();+ }++ nlohmann::json body;++ body["searches"] = nlohmann::json::array();+ nlohmann::json search;+ search["collection"] = "coll1";+ search["q"] = "the";+ search["query_by"] = "title";+ body["searches"].push_back(search);++ req->body = body.dump();+ nlohmann::json embedded_params;+ embedded_params["stopwords"] = "articles";+ req->embedded_params_vec.push_back(embedded_params);++ post_multi_search(req, res);+ nlohmann::json results = nlohmann::json::parse(res->body)["results"][0];++ ASSERT_EQ(0, results["hits"].size());++ req->params.clear();+ req->embedded_params_vec.clear();+ body.clear();++ //when not all words in query are stopwords then it should match the remaining words+ stopword_value = R"(+ {"stopwords": ["america", "europe"], "locale": "en"}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ if(!result) {+ LOG(ERROR) << res->body;+ FAIL();+ }++ body["searches"] = nlohmann::json::array();+ search["collection"] = "coll1";+ search["q"] = "America Man";+ search["query_by"] = "title";+ body["searches"].push_back(search);++ req->body = body.dump();+ embedded_params["stopwords"] = "continents";+ req->embedded_params_vec.push_back(embedded_params);++ post_multi_search(req, res);+ results = nlohmann::json::parse(res->body)["results"][0];++ ASSERT_EQ(0, results["hits"].size());++ req->params.clear();+ req->embedded_params_vec.clear();+ body.clear();++ req->params["collection"] = "coll1";++ body["searches"] = nlohmann::json::array();+ search["collection"] = "coll1";+ search["q"] = "a deadman";+ search["query_by"] = "title";+ body["searches"].push_back(search);++ req->body = body.dump();+ embedded_params["stopwords"] = "articles";+ req->embedded_params_vec.push_back(embedded_params);++ post_multi_search(req, res);+ results = nlohmann::json::parse(res->body)["results"][0];++ ASSERT_EQ(2, results["hits"].size());++ collectionManager.drop_collection("coll1");+}+++TEST_F(CoreAPIUtilsTest, StopwordsValidation) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "title", "type": "string" },+ {"name": "points", "type": "int32" }+ ]+ })"_json;++ auto op = collectionManager.create_collection(schema);+ ASSERT_TRUE(op.ok());+ Collection *coll1 = op.get();++ std::shared_ptr<http_req> req = std::make_shared<http_req>();+ std::shared_ptr<http_res> res = std::make_shared<http_res>(nullptr);++ auto stopword_value = R"(+ {"stopwords": ["america", "europe"]}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ auto result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_STREQ("{\"message\": \"Parameter `locale` is required\"}", res->body.c_str());++ //with a typo+ stopword_value = R"(+ {"stopword": ["america", "europe"], "locale": "en"}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required\"}", res->body.c_str());++ //check for value types+ stopword_value = R"(+ {"stopwords": ["america", "europe"], "locale": 12}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_STREQ("{\"message\": \"Parameter `locale` is required as string value\"}", res->body.c_str());++ stopword_value = R"(+ {"stopwords": [1, 5, 2], "locale": "ko"}+ )"_json;++ req->params["collection"] = "coll1";+ req->params["name"] = "continents";+ req->body = stopword_value.dump();++ result = put_upsert_stopword(req, res);+ ASSERT_EQ(400, res->status_code);+ ASSERT_STREQ("{\"message\": \"Parameter `stopwords` is required as string array value\"}", res->body.c_str());++ collectionManager.drop_collection("coll1");+}