Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Add stemming for queries (#1548)

Ozan Armağan59cc66248d7f
diff --git a/include/collection.h b/include/collection.h
index 522ff015..cc9bba96 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -492,7 +492,7 @@ public:
void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
std::vector<std::vector<std::string>>& q_exclude_tokens,
std::vector<std::vector<std::string>>& q_phrases,
- const std::string& locale, const bool already_segmented, const std::string& stopword_set="") const;
+ const std::string& locale, const bool already_segmented, const std::string& stopword_set="",const std::shared_ptr<Stemmer> stemmer = nullptr) const;
// PUBLIC OPERATIONS
diff --git a/src/collection.cpp b/src/collection.cpp
index ce1cfab4..7cf89ab5 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2256,11 +2256,18 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
}
} else {
field_query_tokens.emplace_back(query_tokens_t{});
- const std::string & field_locale = search_schema.at(weighted_search_fields[0].name).locale;
+ auto most_weighted_field = search_schema.at(weighted_search_fields[0].name);
+ const std::string & field_locale = most_weighted_field.locale;
+
+ std::shared_ptr<Stemmer> stemmer = nullptr;
+ if(most_weighted_field.stem) {
+ stemmer = most_weighted_field.get_stemmer();
+ }
+
parse_search_query(query, q_include_tokens,
field_query_tokens[0].q_exclude_tokens,
field_query_tokens[0].q_phrases,
- field_locale, pre_segmented_query, stopwords_set);
+ field_locale, pre_segmented_query, stopwords_set, stemmer);
// process filter overrides first, before synonyms (order is important)
@@ -3379,7 +3386,7 @@ void Collection::process_filter_overrides(std::vector<const override_t*>& filter
void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
std::vector<std::vector<std::string>>& q_exclude_tokens,
std::vector<std::vector<std::string>>& q_phrases,
- const std::string& locale, const bool already_segmented, const std::string& stopwords_set) const {
+ const std::string& locale, const bool already_segmented, const std::string& stopwords_set, const std::shared_ptr<Stemmer> stemmer) const {
if(query == "*") {
q_exclude_tokens = {};
q_include_tokens = {query};
@@ -3441,6 +3448,10 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
}
}
+ if(stemmer) {
+ token = stemmer->stem(token);
+ }
+
// retokenize using collection config (handles hyphens being part of the query)
std::vector<std::string> sub_tokens;
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index bc013023..651abeba 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -2903,4 +2903,29 @@ TEST_F(CollectionSpecificMoreTest, NumDroppedTokensTest) {
ASSERT_EQ("0", res["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("alpha beta", res["hits"][0]["document"]["title"]);
ASSERT_EQ(1, res["hits"][0]["text_match_info"]["num_tokens_dropped"]);
+}
+
+TEST_F(CollectionSpecificMoreTest, TestStemming2) {
+ nlohmann::json schema = R"({
+ "name": "words",
+ "fields": [
+ {"name": "word", "type": "string", "stem": true }
+ ]
+ })"_json;
+
+ auto coll_stem_res = collectionManager.create_collection(schema);
+ ASSERT_TRUE(coll_stem_res.ok());
+ auto coll_stem = coll_stem_res.get();
+
+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walk"})"_json.dump()).ok());
+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walks"})"_json.dump()).ok());
+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walked"})"_json.dump()).ok());
+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walking"})"_json.dump()).ok());
+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walkings"})"_json.dump()).ok());
+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walker"})"_json.dump()).ok());
+ ASSERT_TRUE(coll_stem->add(R"({"word": "Walkers"})"_json.dump()).ok());
+
+ auto res = coll_stem->search("Walking", {"word"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(7, res["hits"].size());
}
\ No newline at end of file