Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

adding locale and tokenization

krunal1313a40a1b60fc29
diff --git a/include/collection.h b/include/collection.h
index 2f24c686..960e25df 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -421,7 +421,6 @@ public:
Option<nlohmann::json> search(std::string query, const std::vector<std::string> & search_fields,
const std::string & filter_query, const std::vector<std::string> & facet_fields,
const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,
- const std::string& stopword="",
size_t per_page = 10, size_t page = 1,
token_ordering token_order = FREQUENCY, const std::vector<bool>& prefixes = {true},
size_t drop_tokens_threshold = Index::DROP_TOKENS_THRESHOLD,
@@ -464,7 +463,8 @@ public:
const size_t facet_sample_percent = 100,
const size_t facet_sample_threshold = 0,
const size_t page_offset = UINT32_MAX,
- facet_index_type_t facet_index_type = HASH
+ facet_index_type_t facet_index_type = HASH,
+ const std::string& stopwords_set=""
) const;
Option<bool> get_filter_ids(const std::string & filter_query, filter_result_t& filter_result) const;
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 03ed047c..7c355241 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -68,7 +68,7 @@ private:
spp::sparse_hash_map<std::string, nlohmann::json> preset_configs;
- spp::sparse_hash_map<std::string, nlohmann::json> stopword_configs;
+ spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> stopword_configs;
// Auto incrementing ID assigned to each collection
// Using a ID instead of a collection's name makes renaming possible
@@ -206,11 +206,11 @@ public:
Option<bool> delete_preset(const std::string & preset_name);
//stopwords
- spp::sparse_hash_map<std::string, nlohmann::json> get_stopwords() const;
+ spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> get_stopwords() const;
- Option<bool> get_stopword(const std::string&, nlohmann::json&) const;
+ Option<bool> get_stopword(const std::string&, spp::sparse_hash_set<std::string>&) const;
- Option<bool> upsert_stopword(const std::string&, const nlohmann::json&);
+ Option<bool> upsert_stopword(const std::string&, const nlohmann::json&, const std::string&);
Option<bool> delete_stopword(const std::string&);
diff --git a/src/collection.cpp b/src/collection.cpp
index 616df0b4..25d4a655 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1035,7 +1035,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
const std::vector<std::string>& raw_search_fields,
const std::string & filter_query, const std::vector<std::string>& facet_fields,
const std::vector<sort_by> & sort_fields, const std::vector<uint32_t>& num_typos,
- const std::string& stopword, const size_t per_page, const size_t page,
+ const size_t per_page, const size_t page,
token_ordering token_order, const std::vector<bool>& prefixes,
const size_t drop_tokens_threshold,
const spp::sparse_hash_set<std::string> & include_fields,
@@ -1077,7 +1077,8 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
const size_t facet_sample_percent,
const size_t facet_sample_threshold,
const size_t page_offset,
- facet_index_type_t facet_index_type) const {
+ facet_index_type_t facet_index_type,
+ const std::string& stopwords_set) const {
std::shared_lock lock(mutex);
@@ -1483,7 +1484,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
field_query_tokens.emplace_back(query_tokens_t{});
parse_search_query(query, q_include_tokens,
field_query_tokens[0].q_exclude_tokens, field_query_tokens[0].q_phrases, "",
- false, stopword);
+ false, stopwords_set);
for(size_t i = 0; i < q_include_tokens.size(); i++) {
auto& q_include_token = q_include_tokens[i];
field_query_tokens[0].q_include_tokens.emplace_back(i, q_include_token, (i == q_include_tokens.size() - 1),
@@ -1495,7 +1496,7 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
parse_search_query(query, q_include_tokens,
field_query_tokens[0].q_exclude_tokens,
field_query_tokens[0].q_phrases,
- field_locale, pre_segmented_query, stopword);
+ field_locale, pre_segmented_query, stopwords_set);
// process filter overrides first, before synonyms (order is important)
@@ -2421,17 +2422,17 @@ void Collection::process_filter_overrides(std::vector<const override_t*>& filter
void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
std::vector<std::vector<std::string>>& q_exclude_tokens,
std::vector<std::vector<std::string>>& q_phrases,
- const std::string& locale, const bool already_segmented, const std::string& stopword) const {
+ const std::string& locale, const bool already_segmented, const std::string& stopwords_set) const {
if(query == "*") {
q_exclude_tokens = {};
q_include_tokens = {query};
} else {
std::vector<std::string> tokens;
- nlohmann::json stopwords_list;
- if(!stopword.empty()) {
- const auto &stopword_op = CollectionManager::get_instance().get_stopword(stopword, stopwords_list);
+ spp::sparse_hash_set<std::string> stopwords_list;
+ if(!stopwords_set.empty()) {
+ const auto &stopword_op = CollectionManager::get_instance().get_stopword(stopwords_set, stopwords_list);
if (!stopword_op.ok()) {
- LOG(ERROR) << "Error fetching stopword_list for stopword " << stopword << " "<<stopword_op.error();
+ LOG(ERROR) << "Error fetching stopword_list for stopword " << stopwords_set << " "<<stopword_op.error();
}
}
@@ -2445,9 +2446,7 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
Tokenizer(query, true, false, locale, custom_symbols, token_separators).tokenize(tokens);
}
- for (const auto &search_item: stopwords_list.items()) {
- auto val = search_item.value().get<std::string>();
- std::transform(val.begin(), val.end(), val.begin(), ::tolower);
+ for (const auto val: stopwords_list) {
tokens.erase(std::remove(tokens.begin(), tokens.end(), val), tokens.end());
}
@@ -2534,8 +2533,13 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
}
if(q_include_tokens.empty()) {
- // this can happen if the only query token is an exclusion token
- q_include_tokens.emplace_back("##hrhdh##");
+ if(!stopwords_set.empty()) {
+ //this can happen when all tokens in the include are stopwords
+ q_include_tokens.emplace_back("##hrhdh##");
+ } else {
+ // this can happen if the only query token is an exclusion token
+ q_include_tokens.emplace_back("*");
+ }
}
}
}
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index 00bd6fab..0e8910cb 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -664,8 +664,6 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
const char *QUERY_BY_WEIGHTS = "query_by_weights";
const char *SORT_BY = "sort_by";
- const char *STOPWORDS = "stopwords";
-
const char *FACET_BY = "facet_by";
const char *FACET_QUERY = "facet_query";
const char *FACET_QUERY_NUM_TYPOS = "facet_query_num_typos";
@@ -1037,7 +1035,7 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
}
Option<nlohmann::json> result_op = collection->search(raw_query, search_fields, simple_filter_query, facet_fields,
- sort_fields, num_typos, stopwords_set,
+ sort_fields, num_typos,
per_page,
page,
token_order, prefixes, drop_tokens_threshold,
@@ -1078,8 +1076,10 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
match_type,
facet_sample_percent,
facet_sample_threshold,
- offset
- );
+ offset,
+ facet_index_type_t::HASH,
+ stopwords_set
+ );
uint64_t timeMillis = std::chrono::duration_cast<std::chrono::milliseconds>(
std::chrono::high_resolution_clock::now() - begin).count();
@@ -1478,32 +1478,43 @@ Option<bool> CollectionManager::delete_preset(const string& preset_name) {
}
-spp::sparse_hash_map<std::string, nlohmann::json> CollectionManager::get_stopwords() const {
+spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>> CollectionManager::get_stopwords() const {
std::shared_lock lock(mutex);
return stopword_configs;
}
-Option<bool> CollectionManager::get_stopword(const string& stopword_name, nlohmann::json& stopword) const {
+Option<bool> CollectionManager::get_stopword(const string& stopword_name, spp::sparse_hash_set<std::string>& stopwords) const {
std::shared_lock lock(mutex);
const auto& it = stopword_configs.find(stopword_name);
if(it != stopword_configs.end()) {
- stopword = it->second;
+ stopwords = it->second;
return Option<bool>(true);
}
return Option<bool>(404, "Not found.");
}
-Option<bool> CollectionManager::upsert_stopword(const string& stopword_name, const nlohmann::json& stopword_config) {
+Option<bool> CollectionManager::upsert_stopword(const string& stopword_name, const nlohmann::json& stopwords, const std::string& locale) {
std::unique_lock lock(mutex);
- bool inserted = store->insert(get_stopword_key(stopword_name), stopword_config.dump());
+ bool inserted = store->insert(get_stopword_key(stopword_name), stopwords.dump());
if(!inserted) {
return Option<bool>(500, "Unable to insert into store.");
}
- stopword_configs[stopword_name] = stopword_config;
+ std::vector<std::string> tokens;
+
+ for (const auto &stopword: stopwords.items()) {
+ const auto& val = stopword.value().get<std::string>();
+ Tokenizer(val, true, false, locale, {}, {}).tokenize(tokens);
+
+ for(const auto& tok : tokens) {
+ stopword_configs[stopword_name].emplace(tok);
+ }
+ tokens.clear();
+ }
+
return Option<bool>(true);
}
@@ -1514,7 +1525,7 @@ std::string CollectionManager::get_stopword_key(const string& stopword_name) {
Option<bool> CollectionManager::delete_stopword(const string& stopword_name) {
std::unique_lock lock(mutex);
- bool removed = store->remove(get_preset_key(stopword_name));
+ bool removed = store->remove(get_stopword_key(stopword_name));
if(!removed) {
return Option<bool>(500, "Unable to delete from store.");
}
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 34eaf954..80eada4c 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1851,14 +1851,16 @@ bool del_preset(const std::shared_ptr<http_req>& req, const std::shared_ptr<http
bool get_stopwords(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
CollectionManager & collectionManager = CollectionManager::get_instance();
- const spp::sparse_hash_map<std::string, nlohmann::json> & stopwords = collectionManager.get_stopwords();
+ const spp::sparse_hash_map<std::string, spp::sparse_hash_set<std::string>>& stopwords = collectionManager.get_stopwords();
nlohmann::json res_json = nlohmann::json::object();
res_json["stopwords"] = nlohmann::json::array();
- for(const auto& preset_kv: stopwords) {
+ for(const auto& stopwords_kv: stopwords) {
nlohmann::json stopword;
- stopword["name"] = preset_kv.first;
- stopword["value"] = preset_kv.second;
+ stopword["name"] = stopwords_kv.first;
+ for(const auto& val : stopwords_kv.second) {
+ stopword["value"].push_back(val);
+ }
res_json["stopwords"].push_back(stopword);
}
@@ -1870,8 +1872,8 @@ bool get_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<ht
const std::string & stopword_name = req->params["stopword_name"];
CollectionManager & collectionManager = CollectionManager::get_instance();
- nlohmann::json stopword;
- Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopword);
+ spp::sparse_hash_set<std::string> stopwords;
+ Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopwords);
if(!stopword_op.ok()) {
res->set(stopword_op.code(), stopword_op.error());
@@ -1880,7 +1882,9 @@ bool get_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<ht
nlohmann::json res_json;
res_json["name"] = stopword_name;
- res_json["value"] = stopword;
+ for(const auto& stopword : stopwords) {
+ res_json["value"].push_back(stopword);
+ }
res->set_200(res_json.dump());
return true;
@@ -1900,14 +1904,15 @@ bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::shared
CollectionManager & collectionManager = CollectionManager::get_instance();
const std::string & stopword_name = req->params["name"];
- const char* STOPWORD_VALUE = "value";
+ const char* STOPWORD_VALUES = "stopwords";
+ const char* STOPWORD_LOCALE = "locale";
- if(req_json.count(STOPWORD_VALUE) == 0) {
- res->set_400(std::string("Parameter `") + STOPWORD_VALUE + "` is required.");
+ if(req_json.count(STOPWORD_VALUES) == 0) {
+ res->set_400(std::string("Parameter `") + STOPWORD_VALUES + "` is required.");
return false;
}
- Option<bool> success_op = collectionManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUE]);
+ Option<bool> success_op = collectionManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUES], req_json[STOPWORD_LOCALE]);
if(!success_op.ok()) {
res->set_500(success_op.error());
return false;
@@ -1923,8 +1928,8 @@ bool del_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<ht
const std::string & stopword_name = req->params["name"];
CollectionManager & collectionManager = CollectionManager::get_instance();
- nlohmann::json stopword;
- Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopword);
+ spp::sparse_hash_set<std::string> stopwords;
+ Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopwords);
if(!stopword_op.ok()) {
res->set(stopword_op.code(), stopword_op.error());
return false;
@@ -1939,7 +1944,10 @@ bool del_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<ht
nlohmann::json res_json;
res_json["name"] = stopword_name;
- res_json["value"] = stopword;
+ for(const auto& stopword : stopwords) {
+ res_json["value"].push_back(stopword);
+ }
+
res->set_200(res_json.dump());
return true;
}