Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
adding locale and tokenization
krunal1313a40a1b60fc29
diff --git a/include/collection.h b/include/collection.hindex 2f24c686..960e25df 100644--- a/include/collection.h+++ b/include/collection.h@@ -421,7 +421,6 @@ public:Option<nlohmann::json> search(std::string query, const std::vector<std::string> & search_fields,const std::string & filter_query, const std::vector<std::string> & facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,- const std::string& stopword="",size_t per_page = 10, size_t page = 1,token_ordering token_order = FREQUENCY, const std::vector<bool>& prefixes = {true},size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,@@ -464,7 +463,8 @@ public:const size_t facet_sample_percent = 100,const size_t facet_sample_threshold = 0,const size_t page_offset = UINT32_MAX,- facet_index_type_t facet_index_type = HASH+ facet_index_type_t facet_index_type = HASH,+ const std::string& stopwords_set="") const;Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;diff --git a/include/collection_manager.h b/include/collection_manager.hindex 03ed047c..7c355241 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -68,7 +68,7 @@ private:spp::sparse_hash_map<std::string, nlohmann::json> preset_configs;- spp::sparse_hash_map<std::string, nlohmann::json> stopword_configs;+ spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> stopword_configs;// Auto incrementing ID assigned to each collection// Using a ID instead of a collection's name makes renaming possible@@ -206,11 +206,11 @@ public:Option<bool> delete_preset(const std::string & preset_name);//stopwords- spp::sparse_hash_map<std::string, nlohmann::json> get_stopwords() const;+ spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> get_stopwords() const;- Option<bool> get_stopword(const std::string&, nlohmann::json&) const;+ Option<bool> get_stopword(const std::string&, spp::sparse_hash_set<std::string>&) const;- Option<bool> upsert_stopword(const std::string&, const nlohmann::json&);+ Option<bool> upsert_stopword(const std::string&, const nlohmann::json&, const std::string&);Option<bool> delete_stopword(const std::string&);diff --git a/src/collection.cpp b/src/collection.cppindex 616df0b4..25d4a655 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1035,7 +1035,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const std::vector<std::string>& raw_search_fields,const std::string & filter_query, const std::vector<std::string>& facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,- const std::string& stopword, const size_t per_page, const size_t page,+ const size_t per_page, const size_t page,token_ordering token_order, const std::vector<bool>& prefixes,const size_t drop_tokens_threshold,const spp::sparse_hash_set<std::string> & include_fields,@@ -1077,7 +1077,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const size_t facet_sample_percent,const size_t facet_sample_threshold,const size_t page_offset,- facet_index_type_t facet_index_type) const {+ facet_index_type_t facet_index_type,+ const std::string& stopwords_set) const {std::shared_lock lock(mutex);@@ -1483,7 +1484,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,field_query_tokens.emplace_back(query_tokens_t{});parse_search_query(query, q_include_tokens,field_query_tokens[0].q_exclude_tokens, field_query_tokens[0].q_phrases, "",- false, stopword);+ false, stopwords_set);for(size_t i = 0; i < q_include_tokens.size(); i++) {auto& q_include_token = q_include_tokens[i];field_query_tokens[0].q_include_tokens.emplace_back(i, q_include_token, (i == q_include_tokens.size() - 1),@@ -1495,7 +1496,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,parse_search_query(query, q_include_tokens,field_query_tokens[0].q_exclude_tokens,field_query_tokens[0].q_phrases,- field_locale, pre_segmented_query, stopword);+ field_locale, pre_segmented_query, stopwords_set);// process filter overrides first, before synonyms (order is important)@@ -2421,17 +2422,17 @@ void Collection::process_filter_overrides(std::vector<const override_t*>& filtervoid Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,std::vector<std::vector<std::string>>& q_exclude_tokens,std::vector<std::vector<std::string>>& q_phrases,- const std::string& locale, const bool already_segmented, const std::string& stopword) const {+ const std::string& locale, const bool already_segmented, const std::string& stopwords_set) const {if(query == "*") {q_exclude_tokens = {};q_include_tokens = {query};} else {std::vector<std::string> tokens;- nlohmann::json stopwords_list;- if(!stopword.empty()) {- const auto &stopword_op = CollectionManager::get_instance().get_stopword(stopword, stopwords_list);+ spp::sparse_hash_set<std::string> stopwords_list;+ if(!stopwords_set.empty()) {+ const auto &stopword_op = CollectionManager::get_instance().get_stopword(stopwords_set, stopwords_list);if (!stopword_op.ok()) {- LOG(ERROR) << "Error fetching stopword_list for stopword " << stopword << " "<<stopword_op.error();+ LOG(ERROR) << "Error fetching stopword_list for stopword " << stopwords_set << " "<<stopword_op.error();}}@@ -2445,9 +2446,7 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::sTokenizer(query, true, false, locale, custom_symbols, token_separators).tokenize(tokens);}- for (const auto &search_item: stopwords_list.items()) {- auto val = search_item.value().get<std::string>();- std::transform(val.begin(), val.end(), val.begin(), ::tolower);+ for (const auto val: stopwords_list) {tokens.erase(std::remove(tokens.begin(), tokens.end(), val), tokens.end());}@@ -2534,8 +2533,13 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s}if(q_include_tokens.empty()) {- // this can happen if the only query token is an exclusion token- q_include_tokens.emplace_back("##hrhdh##");+ if(!stopwords_set.empty()) {+ //this can happen when all tokens in the include are stopwords+ q_include_tokens.emplace_back("##hrhdh##");+ } else {+ // this can happen if the only query token is an exclusion token+ q_include_tokens.emplace_back("*");+ }}}}diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex 00bd6fab..0e8910cb 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -664,8 +664,6 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *QUERY_BY_WEIGHTS = "query_by_weights";const char *SORT_BY = "sort_by";- const char *STOPWORDS = "stopwords";-const char *FACET_BY = "facet_by";const char *FACET_QUERY = "facet_query";const char *FACET_QUERY_NUM_TYPOS = "facet_query_num_typos";@@ -1037,7 +1035,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re}Option<nlohmann::json> result_op = collection->search(raw_query, search_fields, simple_filter_query, facet_fields,- sort_fields, num_typos, stopwords_set,+ sort_fields, num_typos,per_page,page,token_order, prefixes, drop_tokens_threshold,@@ -1078,8 +1076,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& rematch_type,facet_sample_percent,facet_sample_threshold,- offset- );+ offset,+ facet_index_type_t::HASH,+ stopwords_set+ );uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::high_resolution_clock::now() - begin).count();@@ -1478,32 +1478,43 @@ Option<bool> CollectionManager::delete_preset(const string& preset_name) {}-spp::sparse_hash_map<std::string, nlohmann::json> CollectionManager::get_stopwords() const {+spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> CollectionManager::get_stopwords() const {std::shared_lock lock(mutex);return stopword_configs;}-Option<bool> CollectionManager::get_stopword(const string& stopword_name, nlohmann::json& stopword) const {+Option<bool> CollectionManager::get_stopword(const string& stopword_name, spp::sparse_hash_set<std::string>& stopwords) const {std::shared_lock lock(mutex);const auto& it = stopword_configs.find(stopword_name);if(it != stopword_configs.end()) {- stopword = it->second;+ stopwords = it->second;return Option<bool>(true);}return Option<bool>(404, "Not found.");}-Option<bool> CollectionManager::upsert_stopword(const string& stopword_name, const nlohmann::json& stopword_config) {+Option<bool> CollectionManager::upsert_stopword(const string& stopword_name, const nlohmann::json& stopwords, const std::string& locale) {std::unique_lock lock(mutex);- bool inserted = store->insert(get_stopword_key(stopword_name), stopword_config.dump());+ bool inserted = store->insert(get_stopword_key(stopword_name), stopwords.dump());if(!inserted) {return Option<bool>(500, "Unable to insert into store.");}- stopword_configs[stopword_name] = stopword_config;+ std::vector<std::string> tokens;++ for (const auto &stopword: stopwords.items()) {+ const auto& val = stopword.value().get<std::string>();+ Tokenizer(val, true, false, locale, {}, {}).tokenize(tokens);++ for(const auto& tok : tokens) {+ stopword_configs[stopword_name].emplace(tok);+ }+ tokens.clear();+ }+return Option<bool>(true);}@@ -1514,7 +1525,7 @@ std::string CollectionManager::get_stopword_key(const string& stopword_name) {Option<bool> CollectionManager::delete_stopword(const string& stopword_name) {std::unique_lock lock(mutex);- bool removed = store->remove(get_preset_key(stopword_name));+ bool removed = store->remove(get_stopword_key(stopword_name));if(!removed) {return Option<bool>(500, "Unable to delete from store.");}diff --git a/src/core_api.cpp b/src/core_api.cppindex 34eaf954..80eada4c 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1851,14 +1851,16 @@ bool del_preset(const std::shared_ptr<http_req>& req, const std::shared_ptr<httpbool get_stopwords(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {CollectionManager & collectionManager = CollectionManager::get_instance();- const spp::sparse_hash_map<std::string, nlohmann::json> & stopwords = collectionManager.get_stopwords();+ const spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>>& stopwords = collectionManager.get_stopwords();nlohmann::json res_json = nlohmann::json::object();res_json["stopwords"] = nlohmann::json::array();- for(const auto& preset_kv: stopwords) {+ for(const auto& stopwords_kv: stopwords) {nlohmann::json stopword;- stopword["name"] = preset_kv.first;- stopword["value"] = preset_kv.second;+ stopword["name"] = stopwords_kv.first;+ for(const auto& val : stopwords_kv.second) {+ stopword["value"].push_back(val);+ }res_json["stopwords"].push_back(stopword);}@@ -1870,8 +1872,8 @@ bool get_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<htconst std::string & stopword_name = req->params["stopword_name"];CollectionManager & collectionManager = CollectionManager::get_instance();- nlohmann::json stopword;- Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopword);+ spp::sparse_hash_set<std::string> stopwords;+ Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopwords);if(!stopword_op.ok()) {res->set(stopword_op.code(), stopword_op.error());@@ -1880,7 +1882,9 @@ bool get_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<htnlohmann::json res_json;res_json["name"] = stopword_name;- res_json["value"] = stopword;+ for(const auto& stopword : stopwords) {+ res_json["value"].push_back(stopword);+ }res->set_200(res_json.dump());return true;@@ -1900,14 +1904,15 @@ bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::sharedCollectionManager & collectionManager = CollectionManager::get_instance();const std::string & stopword_name = req->params["name"];- const char* STOPWORD_VALUE = "value";+ const char* STOPWORD_VALUES = "stopwords";+ const char* STOPWORD_LOCALE = "locale";- if(req_json.count(STOPWORD_VALUE) == 0) {- res->set_400(std::string("Parameter `") + STOPWORD_VALUE + "` is required.");+ if(req_json.count(STOPWORD_VALUES) == 0) {+ res->set_400(std::string("Parameter `") + STOPWORD_VALUES + "` is required.");return false;}- Option<bool> success_op = collectionManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUE]);+ Option<bool> success_op = collectionManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUES], req_json[STOPWORD_LOCALE]);if(!success_op.ok()) {res->set_500(success_op.error());return false;@@ -1923,8 +1928,8 @@ bool del_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<htconst std::string & stopword_name = req->params["name"];CollectionManager & collectionManager = CollectionManager::get_instance();- nlohmann::json stopword;- Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopword);+ spp::sparse_hash_set<std::string> stopwords;+ Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopwords);if(!stopword_op.ok()) {res->set(stopword_op.code(), stopword_op.error());return false;@@ -1939,7 +1944,10 @@ bool del_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<htnlohmann::json res_json;res_json["name"] = stopword_name;- res_json["value"] = stopword;+ for(const auto& stopword : stopwords) {+ res_json["value"].push_back(stopword);+ }+res->set_200(res_json.dump());return true;}