Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
remove stopwords after tokenization
krunal13132c0c108d0e66
diff --git a/include/collection.h b/include/collection.hindex c56d1964..e962c526 100644--- a/include/collection.h+++ b/include/collection.h@@ -421,6 +421,7 @@ public:Option<nlohmann::json> search(std::string query, const std::vector<std::string> & search_fields,const std::string & filter_query, const std::vector<std::string> & facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,+ const std::string& stopword="",size_t per_page = 10, size_t page = 1,token_ordering token_order = FREQUENCY, const std::vector<bool>& prefixes = {true},size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,diff --git a/src/collection.cpp b/src/collection.cppindex 098ea064..30ef2231 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1035,7 +1035,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,const std::vector<std::string>& raw_search_fields,const std::string & filter_query, const std::vector<std::string>& facet_fields,const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,- const size_t per_page, const size_t page,+ const std::string& stopword, const size_t per_page, const size_t page,token_ordering token_order, const std::vector<bool>& prefixes,const size_t drop_tokens_threshold,const spp::sparse_hash_set<std::string> & include_fields,@@ -1516,6 +1516,20 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}}+ nlohmann::json stopwords_list;+ const auto &stopword_op = CollectionManager::get_instance().get_stopword(stopword, stopwords_list);+ if (stopword_op.ok()) {+ auto &include_tokens = field_query_tokens[0].q_include_tokens;+ for (const auto &search_item: stopwords_list.items()) {+ auto val = search_item.value().get<std::string>();+ std::transform(val.begin(), val.end(), val.begin(), ::tolower);+ include_tokens.erase(std::remove_if(include_tokens.begin(), include_tokens.end(),+ [&](const auto& token) {+ return token.value == val;+ }), include_tokens.end());+ }+ }+for(size_t i = 1; i < search_fields.size(); i++) {field_query_tokens.emplace_back(query_tokens_t{});field_query_tokens[i] = field_query_tokens[0];diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex bad33766..ba6dcd1f 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -755,22 +755,12 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re}}- //remove the stopwords from search query+ //check if stopword set is suppliedconst auto stopword_it = req_params.find("stopword");+ std::string stopwords_set="";if(stopword_it != req_params.end()) {- nlohmann::json stopword;- const auto& stopword_op = CollectionManager::get_instance().get_stopword(stopword_it->second, stopword);- if(stopword_op.ok()) {- for(const auto& search_item: stopword.items()) {- auto& raw_query = req_params[QUERY];- const auto& val = search_item.value().get<std::string>();- auto pos = raw_query.find(search_item.value());- if(pos != std::string::npos) {- raw_query.erase(pos, val.size());- }- }- }+ stopwords_set = stopword_it->second;}CollectionManager & collectionManager = CollectionManager::get_instance();