Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
load stopwords on startup
krunal7b90c8afc3fe
diff --git a/include/stopwords_manager.h b/include/stopwords_manager.hindex 90a461a9..050f4067 100644--- a/include/stopwords_manager.h+++ b/include/stopwords_manager.h@@ -36,7 +36,9 @@ public:Option<bool> get_stopword(const std::string&, spp::sparse_hash_set<std::string>&) const;- Option<bool> upsert_stopword(const std::string&, const nlohmann::json&, const std::string&);+ Option<bool> upsert_stopword(const std::string&, const nlohmann::json&);Option<bool> delete_stopword(const std::string&);++ void load_stopword_config(const std::string&, const nlohmann::json&);};\ No newline at end of filediff --git a/src/collection_manager.cpp b/src/collection_manager.cppindex ea495600..6a55d44c 100644--- a/src/collection_manager.cpp+++ b/src/collection_manager.cpp@@ -8,6 +8,7 @@#include "batched_indexer.h"#include "logger.h"#include "magic_enum.hpp"+#include "stopwords_manager.h"constexpr const size_t CollectionManager::DEFAULT_NUM_MEMORY_SHARDS;@@ -285,6 +286,27 @@ Option<bool> CollectionManager::load(const size_t collection_batch_size, const siter->Next();}+ //load stopwords+ std::string stopword_prefix_key = std::string(StopwordsManager::STOPWORD_PREFIX) + "_";+ std::string stopword_upper_bound_key = std::string(StopwordsManager::STOPWORD_PREFIX) + "`"; // cannot inline this+ rocksdb::Slice stopword_upper_bound(stopword_upper_bound_key);++ iter = store->scan(stopword_prefix_key, &stopword_upper_bound);+ while(iter->Valid() && iter->key().starts_with(stopword_prefix_key)) {+ std::vector<std::string> parts;+ std::string stopword_name = iter->key().ToString().substr(stopword_prefix_key.size());+ nlohmann::json stopword_obj = nlohmann::json::parse(iter->value().ToString(), nullptr, false);++ if(!stopword_obj.is_discarded() && stopword_obj.is_object()) {+ StopwordsManager::get_instance().load_stopword_config(stopword_name, stopword_obj);+ } else {+ LOG(INFO) << "Invalid object for stopword " << stopword_name;+ }++ iter->Next();+ }++// restore query suggestions configsstd::vector<std::string> analytics_config_jsons;store->scan_fill(AnalyticsManager::ANALYTICS_RULE_PREFIX,diff --git a/src/core_api.cpp b/src/core_api.cppindex 0a3f8edc..8aef1a3f 100644--- a/src/core_api.cpp+++ b/src/core_api.cpp@@ -1969,7 +1969,7 @@ bool put_upsert_stopword(const std::shared_ptr<http_req>& req, const std::sharedreturn false;}- Option<bool> success_op = stopwordManager.upsert_stopword(stopword_name, req_json[STOPWORD_VALUES], req_json[STOPWORD_LOCALE]);+ Option<bool> success_op = stopwordManager.upsert_stopword(stopword_name, req_json);if(!success_op.ok()) {res->set_500(success_op.error());return false;diff --git a/src/stopwords_manager.cpp b/src/stopwords_manager.cppindex 69ec9bf7..87f917c8 100644--- a/src/stopwords_manager.cpp+++ b/src/stopwords_manager.cpp@@ -22,16 +22,18 @@ Option<bool> StopwordsManager::get_stopword(const std::string& stopword_name, spreturn Option<bool>(404, "Stopword `" + stopword_name +"` not found.");}-Option<bool> StopwordsManager::upsert_stopword(const std::string& stopword_name, const nlohmann::json& stopwords, const std::string& locale) {+Option<bool> StopwordsManager::upsert_stopword(const std::string& stopword_name, const nlohmann::json& stopwords_json) {std::unique_lock lock(mutex);- bool inserted = store->insert(get_stopword_key(stopword_name), stopwords.dump());+ bool inserted = store->insert(get_stopword_key(stopword_name), stopwords_json.dump());if(!inserted) {return Option<bool>(500, "Unable to insert into store.");}std::vector<std::string> tokens;spp::sparse_hash_set<std::string> stopwords_set;+ const auto& stopwords = stopwords_json["stopwords"];+ const auto& locale = stopwords_json["locale"];for (const auto &stopword: stopwords.items()) {const auto& val = stopword.value().get<std::string>();@@ -65,3 +67,23 @@ Option<bool> StopwordsManager::delete_stopword(const std::string& stopword_name)stopword_configs.erase(stopword_name);return Option<bool>(true);}++void StopwordsManager::load_stopword_config(const std::string& stopword_name, const nlohmann::json& stopwords_json) {+ std::unique_lock lock(mutex);++ std::vector<std::string> tokens;+ spp::sparse_hash_set<std::string> stopwords_set;+ const auto& stopwords = stopwords_json["stopwords"];+ const auto& locale = stopwords_json["locale"];++ for (const auto &stopword: stopwords.items()) {+ const auto& val = stopword.value().get<std::string>();+ Tokenizer(val, true, false, locale, {}, {}).tokenize(tokens);++ for(const auto& tok : tokens) {+ stopwords_set.emplace(tok);+ }+ tokens.clear();+ }+ stopword_configs[stopword_name] = stopwords_set;+}\ No newline at end of filediff --git a/test/stopwords_manager_test.cpp b/test/stopwords_manager_test.cppindex 76b406f5..784e88c3 100644--- a/test/stopwords_manager_test.cpp+++ b/test/stopwords_manager_test.cpp@@ -30,24 +30,24 @@ protected:TEST_F(StopwordsManagerTest, UpsertGetStopwords) {auto stopwords1 = R"(- {"stopwords": ["america", "europe"]}+ {"stopwords": ["america", "europe"], "locale": "en"})"_json;- auto upsert_op = stopwordsManager.upsert_stopword("continents", stopwords1["stopwords"], "en");+ auto upsert_op = stopwordsManager.upsert_stopword("continents", stopwords1);ASSERT_TRUE(upsert_op.ok());auto stopwords2 = R"(- {"stopwords": ["a", "an", "the"]}+ {"stopwords": ["a", "an", "the"], "locale": "en"})"_json;- upsert_op = stopwordsManager.upsert_stopword("articles", stopwords2["stopwords"], "en");+ upsert_op = stopwordsManager.upsert_stopword("articles", stopwords2);ASSERT_TRUE(upsert_op.ok());auto stopwords3 = R"(- {"stopwords": ["India", "United States", "Japan", "China"]}+ {"stopwords": ["India", "United States", "Japan", "China"], "locale": "en"})"_json;- upsert_op = stopwordsManager.upsert_stopword("countries", stopwords3["stopwords"], "en");+ upsert_op = stopwordsManager.upsert_stopword("countries", stopwords3);ASSERT_TRUE(upsert_op.ok());auto stopword_config = stopwordsManager.get_stopwords();@@ -74,9 +74,9 @@ TEST_F(StopwordsManagerTest, UpsertGetStopwords) {}TEST_F(StopwordsManagerTest, GetStopword) {- auto stopwords = R"({"stopwords": ["a", "an", "the"]})"_json;+ auto stopwords = R"({"stopwords": ["a", "an", "the"], "locale": "en"})"_json;- auto upsert_op = stopwordsManager.upsert_stopword("articles", stopwords["stopwords"], "en");+ auto upsert_op = stopwordsManager.upsert_stopword("articles", stopwords);ASSERT_TRUE(upsert_op.ok());spp::sparse_hash_set<std::string> stopwords_set;@@ -94,9 +94,9 @@ TEST_F(StopwordsManagerTest, GetStopword) {ASSERT_EQ("Stopword `country` not found.", get_op.error());//try fetching stopwords with token- stopwords = R"({"stopwords": ["India", "United States", "Japan"]})"_json;+ stopwords = R"({"stopwords": ["India", "United States", "Japan"], "locale": "en"})"_json;- upsert_op = stopwordsManager.upsert_stopword("country", stopwords["stopwords"], "en");+ upsert_op = stopwordsManager.upsert_stopword("country", stopwords);ASSERT_TRUE(upsert_op.ok());get_op = stopwordsManager.get_stopword("country", stopwords_set);@@ -106,17 +106,17 @@ TEST_F(StopwordsManagerTest, GetStopword) {TEST_F(StopwordsManagerTest, DeleteStopword) {auto stopwords1 = R"(- {"stopwords": ["america", "europe"]}+ {"stopwords": ["america", "europe"], "locale": "en"})"_json;- auto upsert_op = stopwordsManager.upsert_stopword("continents", stopwords1["stopwords"], "en");+ auto upsert_op = stopwordsManager.upsert_stopword("continents", stopwords1);ASSERT_TRUE(upsert_op.ok());auto stopwords2 = R"(- {"stopwords": ["a", "an", "the"]}+ {"stopwords": ["a", "an", "the"], "locale": "en"})"_json;- upsert_op = stopwordsManager.upsert_stopword("articles", stopwords2["stopwords"], "en");+ upsert_op = stopwordsManager.upsert_stopword("articles", stopwords2);ASSERT_TRUE(upsert_op.ok());spp::sparse_hash_set<std::string> stopwords_set;@@ -138,11 +138,11 @@ TEST_F(StopwordsManagerTest, DeleteStopword) {}TEST_F(StopwordsManagerTest, UpdateStopword) {- auto stopwords = R"(- {"stopwords": ["america", "europe"]}+ auto stopwords_json = R"(+ {"stopwords": ["america", "europe"], "locale": "en"})"_json;- auto upsert_op = stopwordsManager.upsert_stopword("continents", stopwords["stopwords"], "en");+ auto upsert_op = stopwordsManager.upsert_stopword("continents", stopwords_json);ASSERT_TRUE(upsert_op.ok());auto stopword_config = stopwordsManager.get_stopwords();@@ -152,10 +152,10 @@ TEST_F(StopwordsManagerTest, UpdateStopword) {ASSERT_TRUE(stopword_config["continents"].find("europe") != stopword_config["continents"].end());//adding new words with same name should replace the stopwords set- stopwords = R"(- {"stopwords": ["india", "china", "japan"]}+ stopwords_json = R"(+ {"stopwords": ["india", "china", "japan"], "locale": "en"})"_json;- upsert_op = stopwordsManager.upsert_stopword("continents", stopwords["stopwords"], "en");+ upsert_op = stopwordsManager.upsert_stopword("continents", stopwords_json);ASSERT_TRUE(upsert_op.ok());stopword_config = stopwordsManager.get_stopwords();