Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

initial commit to add support for stopwords

krunal13138e6bc045552b
diff --git a/include/collection_manager.h b/include/collection_manager.h
index 03ef6b43..03ed047c 100644
--- a/include/collection_manager.h
+++ b/include/collection_manager.h
@@ -68,6 +68,8 @@ private:
spp::sparse_hash_map<std::string, nlohmann::json> preset_configs;
+ spp::sparse_hash_map<std::string, nlohmann::json> stopword_configs;
+
// Auto incrementing ID assigned to each collection
// Using a ID instead of a collection's name makes renaming possible
std::atomic<uint32_t> next_collection_id;
@@ -101,6 +103,7 @@ public:
static constexpr const char* SYMLINK_PREFIX = "$SL";
static constexpr const char* PRESET_PREFIX = "$PS";
static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";
+ static constexpr const char* STOPWORD_PREFIX = "$SW";
static CollectionManager & get_instance() {
static CollectionManager instance;
@@ -201,4 +204,15 @@ public:
Option<bool> upsert_preset(const std::string & preset_name, const nlohmann::json& preset_config);
Option<bool> delete_preset(const std::string & preset_name);
+
+ //stopwords
+ spp::sparse_hash_map<std::string, nlohmann::json> get_stopwords() const;
+
+ Option<bool> get_stopword(const std::string&, nlohmann::json&) const;
+
+ Option<bool> upsert_stopword(const std::string&, const nlohmann::json&);
+
+ Option<bool> delete_stopword(const std::string&);
+
+ static std::string get_stopword_key(const std::string & stopword_name);
};
\ No newline at end of file
diff --git a/include/core_api.h b/include/core_api.h
index 780d3d1b..8959ee2c 100644
--- a/include/core_api.h
+++ b/include/core_api.h
@@ -63,6 +63,16 @@ bool put_upsert_preset(const std::shared_ptr<http_req>& req, const std::shared_p
bool del_preset(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+//stopwords
+
+bool get_stopwords(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+
+bool get_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+
+bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+
+bool del_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
+
// Overrides
bool get_overrides(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);
diff --git a/src/collection_manager.cpp b/src/collection_manager.cpp
index b0c76f69..bad33766 100644
--- a/src/collection_manager.cpp
+++ b/src/collection_manager.cpp
@@ -316,6 +316,7 @@ void CollectionManager::dispose() {
collections.clear();
collection_symlinks.clear();
preset_configs.clear();
+ stopword_configs.clear();
store->close();
}
@@ -663,6 +664,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
const char *QUERY_BY_WEIGHTS = "query_by_weights";
const char *SORT_BY = "sort_by";
+ const char *STOPWORDS = "stopwords";
+
const char *FACET_BY = "facet_by";
const char *FACET_QUERY = "facet_query";
const char *FACET_QUERY_NUM_TYPOS = "facet_query_num_typos";
@@ -752,6 +755,24 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re
}
}
+ //remove the stopwords from search query
+ const auto stopword_it = req_params.find("stopword");
+
+ if(stopword_it != req_params.end()) {
+ nlohmann::json stopword;
+ const auto& stopword_op = CollectionManager::get_instance().get_stopword(stopword_it->second, stopword);
+ if(stopword_op.ok()) {
+ for(const auto& search_item: stopword.items()) {
+ auto& raw_query = req_params[QUERY];
+ const auto& val = search_item.value().get<std::string>();
+ auto pos = raw_query.find(search_item.value());
+ if(pos != std::string::npos) {
+ raw_query.erase(pos, val.size());
+ }
+ }
+ }
+ }
+
CollectionManager & collectionManager = CollectionManager::get_instance();
const std::string& orig_coll_name = req_params["collection"];
auto collection = collectionManager.get_collection(orig_coll_name);
@@ -1466,6 +1487,52 @@ Option<bool> CollectionManager::delete_preset(const string& preset_name) {
return Option<bool>(true);
}
+
+spp::sparse_hash_map<std::string, nlohmann::json> CollectionManager::get_stopwords() const {
+ std::shared_lock lock(mutex);
+ return stopword_configs;
+}
+
+Option<bool> CollectionManager::get_stopword(const string& stopword_name, nlohmann::json& stopword) const {
+ std::shared_lock lock(mutex);
+
+ const auto& it = stopword_configs.find(stopword_name);
+ if(it != stopword_configs.end()) {
+ stopword = it->second;
+ return Option<bool>(true);
+ }
+
+ return Option<bool>(404, "Not found.");
+}
+
+Option<bool> CollectionManager::upsert_stopword(const string& stopword_name, const nlohmann::json& stopword_config) {
+ std::unique_lock lock(mutex);
+
+ bool inserted = store->insert(get_stopword_key(stopword_name), stopword_config.dump());
+ if(!inserted) {
+ return Option<bool>(500, "Unable to insert into store.");
+ }
+
+ stopword_configs[stopword_name] = stopword_config;
+ return Option<bool>(true);
+}
+
+std::string CollectionManager::get_stopword_key(const string& stopword_name) {
+ return std::string(STOPWORD_PREFIX) + "_" + stopword_name;
+}
+
+Option<bool> CollectionManager::delete_stopword(const string& stopword_name) {
+ std::unique_lock lock(mutex);
+
+ bool removed = store->remove(get_preset_key(stopword_name));
+ if(!removed) {
+ return Option<bool>(500, "Unable to delete from store.");
+ }
+
+ stopword_configs.erase(stopword_name);
+ return Option<bool>(true);
+}
+
Option<Collection*> CollectionManager::clone_collection(const string& existing_name, const nlohmann::json& req_json) {
std::shared_lock lock(mutex);
diff --git a/src/core_api.cpp b/src/core_api.cpp
index 7de42f6a..34eaf954 100644
--- a/src/core_api.cpp
+++ b/src/core_api.cpp
@@ -1848,6 +1848,102 @@ bool del_preset(const std::shared_ptr<http_req>& req, const std::shared_ptr<http
res->set_200(res_json.dump());
return true;
}
+
+bool get_stopwords(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ const spp::sparse_hash_map<std::string, nlohmann::json> & stopwords = collectionManager.get_stopwords();
+ nlohmann::json res_json = nlohmann::json::object();
+ res_json["stopwords"] = nlohmann::json::array();
+
+ for(const auto& preset_kv: stopwords) {
+ nlohmann::json stopword;
+ stopword["name"] = preset_kv.first;
+ stopword["value"] = preset_kv.second;
+ res_json["stopwords"].push_back(stopword);
+ }
+
+ res->set_200(res_json.dump());
+ return true;
+}
+
+bool get_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ const std::string & stopword_name = req->params["stopword_name"];
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+
+ nlohmann::json stopword;
+ Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopword);
+
+ if(!stopword_op.ok()) {
+ res->set(stopword_op.code(), stopword_op.error());
+ return false;
+ }
+
+ nlohmann::json res_json;
+ res_json["name"] = stopword_name;
+ res_json["value"] = stopword;
+
+ res->set_200(res_json.dump());
+ return true;
+}
+
+bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ nlohmann::json req_json;
+
+ try {
+ req_json = nlohmann::json::parse(req->body);
+ } catch(const std::exception& e) {
+ LOG(ERROR) << "JSON error: " << e.what();
+ res->set_400("Bad JSON.");
+ return false;
+ }
+
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+ const std::string & stopword_name = req->params["name"];
+
+ const char* STOPWORD_VALUE = "value";
+
+ if(req_json.count(STOPWORD_VALUE) == 0) {
+ res->set_400(std::string("Parameter `") + STOPWORD_VALUE + "` is required.");
+ return false;
+ }
+
+ Option<bool> success_op = collectionManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUE]);
+ if(!success_op.ok()) {
+ res->set_500(success_op.error());
+ return false;
+ }
+
+ req_json["name"] = stopword_name;
+
+ res->set_200(req_json.dump());
+ return true;
+}
+
+bool del_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
+ const std::string & stopword_name = req->params["name"];
+ CollectionManager & collectionManager = CollectionManager::get_instance();
+
+ nlohmann::json stopword;
+ Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopword);
+ if(!stopword_op.ok()) {
+ res->set(stopword_op.code(), stopword_op.error());
+ return false;
+ }
+
+ Option<bool> delete_op = collectionManager.delete_stopword(stopword_name);
+
+ if(!delete_op.ok()) {
+ res->set_500(delete_op.error());
+ return false;
+ }
+
+ nlohmann::json res_json;
+ res_json["name"] = stopword_name;
+ res_json["value"] = stopword;
+ res->set_200(res_json.dump());
+ return true;
+}
+
bool get_rate_limits(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {
RateLimitManager* rateLimitManager = RateLimitManager::getInstance();
diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cpp
index d4df4f2a..a61faed0 100644
--- a/src/main/typesense_server.cpp
+++ b/src/main/typesense_server.cpp
@@ -68,6 +68,11 @@ void master_server_routes() {
server->put("/presets/:name", put_upsert_preset);
server->del("/presets/:name", del_preset);
+ server->get("/stopwords", get_stopwords);
+ server->get("/stopwords/:name", get_stopword);
+ server->put("/stopwords/:name", put_upsert_stopword);
+ server->del("/stopwords/:name", del_stopword);
+
// analytics
server->get("/analytics/rules", get_analytics_rules);
server->post("/analytics/rules", post_create_analytics_rules);