Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Add stemming for queries (#1548)
Ozan Armağan59cc66248d7f
diff --git a/include/collection.h b/include/collection.hindex 522ff015..cc9bba96 100644--- a/include/collection.h+++ b/include/collection.h@@ -492,7 +492,7 @@ public:void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,std::vector<std::vector<std::string>>& q_exclude_tokens,std::vector<std::vector<std::string>>& q_phrases,- const std::string& locale, const bool already_segmented, const std::string& stopword_set="") const;+ const std::string& locale, const bool already_segmented, const std::string& stopword_set="",const std::shared_ptr<Stemmer> stemmer = nullptr) const;// PUBLIC OPERATIONSdiff --git a/src/collection.cpp b/src/collection.cppindex ce1cfab4..7cf89ab5 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2256,11 +2256,18 @@ Option<nlohmann::json> Collection::search(std::string raw_query,}} else {field_query_tokens.emplace_back(query_tokens_t{});- const std::string & field_locale = search_schema.at(weighted_search_fields[0].name).locale;+ auto most_weighted_field = search_schema.at(weighted_search_fields[0].name);+ const std::string & field_locale = most_weighted_field.locale;++ std::shared_ptr<Stemmer> stemmer = nullptr;+ if(most_weighted_field.stem) {+ stemmer = most_weighted_field.get_stemmer();+ }+parse_search_query(query, q_include_tokens,field_query_tokens[0].q_exclude_tokens,field_query_tokens[0].q_phrases,- field_locale, pre_segmented_query, stopwords_set);+ field_locale, pre_segmented_query, stopwords_set, stemmer);// process filter overrides first, before synonyms (order is important)@@ -3379,7 +3386,7 @@ void Collection::process_filter_overrides(std::vector<const override_t*>& filtervoid Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,std::vector<std::vector<std::string>>& q_exclude_tokens,std::vector<std::vector<std::string>>& q_phrases,- const std::string& locale, const bool already_segmented, const std::string& stopwords_set) const {+ const std::string& locale, const bool already_segmented, const std::string& stopwords_set, const std::shared_ptr<Stemmer> stemmer) const {if(query == "*") {q_exclude_tokens = {};q_include_tokens = {query};@@ -3441,6 +3448,10 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s}}+ if(stemmer) {+ token = stemmer->stem(token);+ }+// retokenize using collection config (handles hyphens being part of the query)std::vector<std::string> sub_tokens;diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex bc013023..651abeba 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -2903,4 +2903,29 @@ TEST_F(CollectionSpecificMoreTest, NumDroppedTokensTest) {ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("alpha beta", res["hits"][0]["document"]["title"]);ASSERT_EQ(1, res["hits"][0]["text_match_info"]["num_tokens_dropped"]);+}++TEST_F(CollectionSpecificMoreTest, TestStemming2) {+ nlohmann::json schema = R"({+ "name": "words",+ "fields": [+ {"name": "word", "type": "string", "stem": true }+ ]+ })"_json;++ auto coll_stem_res = collectionManager.create_collection(schema);+ ASSERT_TRUE(coll_stem_res.ok());+ auto coll_stem = coll_stem_res.get();++ ASSERT_TRUE(coll_stem->add(R"({"word": "Walk"})"_json.dump()).ok());+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walks"})"_json.dump()).ok());+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walked"})"_json.dump()).ok());+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walking"})"_json.dump()).ok());+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walkings"})"_json.dump()).ok());+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walker"})"_json.dump()).ok());+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walkers"})"_json.dump()).ok());++ auto res = coll_stem->search("Walking", {"word"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(7, res["hits"].size());}\ No newline at end of file