Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

remove stopwords after tokenization

krunal13132c0c108d0e66
diff --git a/include/collection.h b/include/collection.h
index c56d1964..e962c526 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -421,6 +421,7 @@ public:
Option<nlohmann::json> search(std::string query, const std::vector<std::string> & search_fields,
const std::string & filter_query, const std::vector<std::string> & facet_fields,
const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,
+ const std::string& stopword="",
size_t per_page = 10, size_t page = 1,
token_ordering token_order = FREQUENCY, const std::vector<bool>& prefixes = {true},
size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
diff --git a/src/collection.cpp b/src/collection.cpp
index 098ea064..30ef2231 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1035,7 +1035,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
const std::vector<std::string>& raw_search_fields,
const std::string & filter_query, const std::vector<std::string>& facet_fields,
const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,
- const size_t per_page, const size_t page,
+ const std::string& stopword, const size_t per_page, const size_t page,
token_ordering token_order, const std::vector<bool>& prefixes,
const size_t drop_tokens_threshold,
const spp::sparse_hash_set<std::string> & include_fields,
@@ -1516,6 +1516,20 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
}
+ nlohmann::json stopwords_list;
+ const auto &stopword_op = CollectionManager::get_instance().get_stopword(stopword, stopwords_list);
+ if (stopword_op.ok()) {
+ auto &include_tokens = field_query_tokens[0].q_include_tokens;
+ for (const auto &search_item: stopwords_list.items()) {
+ auto val = search_item.value().get<std::string>();
+ std::transform(val.begin(), val.end(), val.begin(), ::tolower);
+ include_tokens.erase(std::remove_if(include_tokens.begin(), include_tokens.end(),
+ [&](const auto& token) {
+ return token.value == val;
+ }), include_tokens.end());
+ }
+ }
+
for(size_t i = 1; i < search_fields.size(); i++) {
field_query_tokens.emplace_back(query_tokens_t{});
field_query_tokens[i] = field_query_tokens[0];
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index bad33766..ba6dcd1f 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -755,22 +755,12 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
}
}
- //remove the stopwords from search query
+ //check if stopword set is supplied
const auto stopword_it = req_params.find("stopword");
+ std::string stopwords_set="";
if(stopword_it != req_params.end()) {
- nlohmann::json stopword;
- const auto& stopword_op = CollectionManager::get_instance().get_stopword(stopword_it->second, stopword);
- if(stopword_op.ok()) {
- for(const auto& search_item: stopword.items()) {
- auto& raw_query = req_params[QUERY];
- const auto& val = search_item.value().get<std::string>();
- auto pos = raw_query.find(search_item.value());
- if(pos != std::string::npos) {
- raw_query.erase(pos, val.size());
- }
- }
- }
+ stopwords_set = stopword_it->second;
}
CollectionManager & collectionManager = CollectionManager::get_instance();