Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
initial commit to add support for stopwords
krunal13138e6bc045552b
diff --git a/include/collection_manager.h b/include/collection_manager.hindex 03ef6b43..03ed047c 100644--- a/include/collection_manager.h+++ b/include/collection_manager.h@@ -68,6 +68,8 @@ private:spp::sparse_hash_map<std::string, nlohmann::json> preset_configs;+ spp::sparse_hash_map<std::string, nlohmann::json> stopword_configs;+// Auto incrementing ID assigned to each collection// Using a ID instead of a collection's name makes renaming possiblestd::atomic<uint32_t> next_collection_id;@@ -101,6 +103,7 @@ public:static constexpr const char* SYMLINK_PREFIX = "$SL";static constexpr const char* PRESET_PREFIX = "$PS";static constexpr const char* BATCHED_INDEXER_STATE_KEY = "$BI";+ static constexpr const char* STOPWORD_PREFIX = "$SW";static CollectionManager & get_instance() {static CollectionManager instance;@@ -201,4 +204,15 @@ public:Option<bool> upsert_preset(const std::string & preset_name, const nlohmann::json& preset_config);Option<bool> delete_preset(const std::string & preset_name);++ //stopwords+ spp::sparse_hash_map<std::string, nlohmann::json> get_stopwords() const;++ Option<bool> get_stopword(const std::string&, nlohmann::json&) const;++ Option<bool> upsert_stopword(const std::string&, const nlohmann::json&);++ Option<bool> delete_stopword(const std::string&);++ static std::string get_stopword_key(const std::string & stopword_name);};\ No newline at end of filediff --git a/include/core_api.h b/include/core_api.hindex 780d3d1b..8959ee2c 100644--- a/include/core_api.h+++ b/include/core_api.h@@ -63,6 +63,16 @@ bool put_upsert_preset(const std::shared_ptr<http_req>& req, const std::shared_pbool del_preset(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+//stopwords++bool get_stopwords(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);++bool get_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);++bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);++bool del_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);+// Overridesbool get_overrides(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res);diff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex b0c76f69..bad33766 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -316,6 +316,7 @@ void CollectionManager::dispose() {collections.clear();collection_symlinks.clear();preset_configs.clear();+ stopword_configs.clear();store->close();}@@ -663,6 +664,8 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& reconst char *QUERY_BY_WEIGHTS = "query_by_weights";const char *SORT_BY = "sort_by";+ const char *STOPWORDS = "stopwords";+const char *FACET_BY = "facet_by";const char *FACET_QUERY = "facet_query";const char *FACET_QUERY_NUM_TYPOS = "facet_query_num_typos";@@ -752,6 +755,24 @@ Option<bool> CollectionManager::do_search(std::map<std::string, std::string>& re}}+ //remove the stopwords from search query+ const auto stopword_it = req_params.find("stopword");++ if(stopword_it != req_params.end()) {+ nlohmann::json stopword;+ const auto& stopword_op = CollectionManager::get_instance().get_stopword(stopword_it->second, stopword);+ if(stopword_op.ok()) {+ for(const auto& search_item: stopword.items()) {+ auto& raw_query = req_params[QUERY];+ const auto& val = search_item.value().get<std::string>();+ auto pos = raw_query.find(search_item.value());+ if(pos != std::string::npos) {+ raw_query.erase(pos, val.size());+ }+ }+ }+ }+CollectionManager & collectionManager = CollectionManager::get_instance();const std::string& orig_coll_name = req_params["collection"];auto collection = collectionManager.get_collection(orig_coll_name);@@ -1466,6 +1487,52 @@ Option<bool> CollectionManager::delete_preset(const string& preset_name) {return Option<bool>(true);}++spp::sparse_hash_map<std::string, nlohmann::json> CollectionManager::get_stopwords() const {+ std::shared_lock lock(mutex);+ return stopword_configs;+}++Option<bool> CollectionManager::get_stopword(const string& stopword_name, nlohmann::json& stopword) const {+ std::shared_lock lock(mutex);++ const auto& it = stopword_configs.find(stopword_name);+ if(it != stopword_configs.end()) {+ stopword = it->second;+ return Option<bool>(true);+ }++ return Option<bool>(404, "Not found.");+}++Option<bool> CollectionManager::upsert_stopword(const string& stopword_name, const nlohmann::json& stopword_config) {+ std::unique_lock lock(mutex);++ bool inserted = store->insert(get_stopword_key(stopword_name), stopword_config.dump());+ if(!inserted) {+ return Option<bool>(500, "Unable to insert into store.");+ }++ stopword_configs[stopword_name] = stopword_config;+ return Option<bool>(true);+}++std::string CollectionManager::get_stopword_key(const string& stopword_name) {+ return std::string(STOPWORD_PREFIX) + "_" + stopword_name;+}++Option<bool> CollectionManager::delete_stopword(const string& stopword_name) {+ std::unique_lock lock(mutex);++ bool removed = store->remove(get_preset_key(stopword_name));+ if(!removed) {+ return Option<bool>(500, "Unable to delete from store.");+ }++ stopword_configs.erase(stopword_name);+ return Option<bool>(true);+}+Option<Collection*> CollectionManager::clone_collection(const string& existing_name, const nlohmann::json& req_json) {std::shared_lock lock(mutex);diff --git a/src/core_api.cpp b/src/core_api.cppindex 7de42f6a..34eaf954 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1848,6 +1848,102 @@ bool del_preset(const std::shared_ptr<http_req>& req, const std::shared_ptr<httpres->set_200(res_json.dump());return true;}++bool get_stopwords(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ CollectionManager & collectionManager = CollectionManager::get_instance();+ const spp::sparse_hash_map<std::string, nlohmann::json> & stopwords = collectionManager.get_stopwords();+ nlohmann::json res_json = nlohmann::json::object();+ res_json["stopwords"] = nlohmann::json::array();++ for(const auto& preset_kv: stopwords) {+ nlohmann::json stopword;+ stopword["name"] = preset_kv.first;+ stopword["value"] = preset_kv.second;+ res_json["stopwords"].push_back(stopword);+ }++ res->set_200(res_json.dump());+ return true;+}++bool get_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ const std::string & stopword_name = req->params["stopword_name"];+ CollectionManager & collectionManager = CollectionManager::get_instance();++ nlohmann::json stopword;+ Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopword);++ if(!stopword_op.ok()) {+ res->set(stopword_op.code(), stopword_op.error());+ return false;+ }++ nlohmann::json res_json;+ res_json["name"] = stopword_name;+ res_json["value"] = stopword;++ res->set_200(res_json.dump());+ return true;+}++bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ nlohmann::json req_json;++ try {+ req_json = nlohmann::json::parse(req->body);+ } catch(const std::exception& e) {+ LOG(ERROR) << "JSON error: " << e.what();+ res->set_400("Bad JSON.");+ return false;+ }++ CollectionManager & collectionManager = CollectionManager::get_instance();+ const std::string & stopword_name = req->params["name"];++ const char* STOPWORD_VALUE = "value";++ if(req_json.count(STOPWORD_VALUE) == 0) {+ res->set_400(std::string("Parameter `") + STOPWORD_VALUE + "` is required.");+ return false;+ }++ Option<bool> success_op = collectionManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUE]);+ if(!success_op.ok()) {+ res->set_500(success_op.error());+ return false;+ }++ req_json["name"] = stopword_name;++ res->set_200(req_json.dump());+ return true;+}++bool del_stopword(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {+ const std::string & stopword_name = req->params["name"];+ CollectionManager & collectionManager = CollectionManager::get_instance();++ nlohmann::json stopword;+ Option<bool> stopword_op = collectionManager.get_stopword(stopword_name, stopword);+ if(!stopword_op.ok()) {+ res->set(stopword_op.code(), stopword_op.error());+ return false;+ }++ Option<bool> delete_op = collectionManager.delete_stopword(stopword_name);++ if(!delete_op.ok()) {+ res->set_500(delete_op.error());+ return false;+ }++ nlohmann::json res_json;+ res_json["name"] = stopword_name;+ res_json["value"] = stopword;+ res->set_200(res_json.dump());+ return true;+}+bool get_rate_limits(const std::shared_ptr<http_req>& req, const std::shared_ptr<http_res>& res) {RateLimitManager* rateLimitManager = RateLimitManager::getInstance();diff --git a/src/main/typesense_server.cpp b/src/main/typesense_server.cppindex d4df4f2a..a61faed0 100644--- a/src/main/typesense_server.cpp+++ b/src/main/typesense_server.cpp@@ -68,6 +68,11 @@ void master_server_routes() {server->put("/presets/:name", put_upsert_preset);server->del("/presets/:name", del_preset);+ server->get("/stopwords", get_stopwords);+ server->get("/stopwords/:name", get_stopword);+ server->put("/stopwords/:name", put_upsert_stopword);+ server->del("/stopwords/:name", del_stopword);+// analyticsserver->get("/analytics/rules", get_analytics_rules);server->post("/analytics/rules", post_create_analytics_rules);