Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix stemming bug when searching with synonyms (#1569)

Ozan Armağan2c21e1306bb3
diff --git a/include/collection.h b/include/collection.h
index 44d84092..a14c00de 100644
--- a/include/collection.h
+++ b/include/collection.h
@@ -492,7 +492,7 @@ public:
void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
std::vector<std::vector<std::string>>& q_exclude_tokens,
std::vector<std::vector<std::string>>& q_phrases,
- const std::string& locale, const bool already_segmented, const std::string& stopword_set="",const std::shared_ptr<Stemmer> stemmer = nullptr) const;
+ const std::string& locale, const bool already_segmented, const std::string& stopword_set="") const;
// PUBLIC OPERATIONS
diff --git a/src/collection.cpp b/src/collection.cpp
index 62e2c726..6947a63d 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2259,15 +2259,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,
auto most_weighted_field = search_schema.at(weighted_search_fields[0].name);
const std::string & field_locale = most_weighted_field.locale;
- std::shared_ptr<Stemmer> stemmer = nullptr;
- if(most_weighted_field.stem) {
- stemmer = most_weighted_field.get_stemmer();
- }
-
parse_search_query(query, q_include_tokens,
field_query_tokens[0].q_exclude_tokens,
field_query_tokens[0].q_phrases,
- field_locale, pre_segmented_query, stopwords_set, stemmer);
+ field_locale, pre_segmented_query, stopwords_set);
// process filter overrides first, before synonyms (order is important)
@@ -3386,7 +3381,7 @@ void Collection::process_filter_overrides(std::vector<const override_t*>& filter
void Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,
std::vector<std::vector<std::string>>& q_exclude_tokens,
std::vector<std::vector<std::string>>& q_phrases,
- const std::string& locale, const bool already_segmented, const std::string& stopwords_set, const std::shared_ptr<Stemmer> stemmer) const {
+ const std::string& locale, const bool already_segmented, const std::string& stopwords_set) const {
if(query == "*") {
q_exclude_tokens = {};
q_include_tokens = {query};
@@ -3448,9 +3443,6 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s
}
}
- if(stemmer) {
- token = stemmer->stem(token);
- }
// retokenize using collection config (handles hyphens being part of the query)
std::vector<std::string> sub_tokens;
diff --git a/src/index.cpp b/src/index.cpp
index 67d5aa58..7497f526 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -3198,12 +3198,22 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::vector<std::vector<token_t>> q_pos_synonyms;
std::vector<std::string> q_include_tokens;
int syn_orig_num_tokens = -1;
-
for(size_t j = 0; j < field_query_tokens[0].q_include_tokens.size(); j++) {
q_include_tokens.push_back(field_query_tokens[0].q_include_tokens[j].value);
}
synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);
+ if (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem) {
+ auto stemmer = search_schema.at(the_fields[0].name).get_stemmer();
+ for(auto& q_include_token: q_include_tokens) {
+ q_include_token = stemmer->stem(q_include_token);
+ }
+
+ for(auto& q_token: field_query_tokens[0].q_include_tokens) {
+ q_token.value = stemmer->stem(q_token.value);
+ }
+ }
+
if(!field_query_tokens[0].q_synonyms.empty()) {
syn_orig_num_tokens = field_query_tokens[0].q_include_tokens.size();
}
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 0e2ad026..615db51b 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -2952,4 +2952,34 @@ TEST_F(CollectionSpecificMoreTest, TestStemming2) {
auto res = coll_stem->search("Walking", {"word"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
ASSERT_EQ(7, res["hits"].size());
+}
+
+TEST_F(CollectionSpecificMoreTest, TestStemmingWithSynonym) {
+ nlohmann::json schema = R"({
+ "name": "words",
+ "fields": [
+ {"name": "word", "type": "string", "stem": true }
+ ]
+ })"_json;
+
+ auto coll_stem_res = collectionManager.create_collection(schema);
+ ASSERT_TRUE(coll_stem_res.ok());
+
+ auto coll_stem = coll_stem_res.get();
+
+ nlohmann::json synonym_json = R"(
+ {
+ "id": "",
+ "synonyms": ["making", "foobar"]
+ }
+ )"_json;
+ auto synonym_op = coll_stem->add_synonym(synonym_json);
+ LOG(INFO) << synonym_op.error();
+ ASSERT_TRUE(synonym_op.ok());
+
+ ASSERT_TRUE(coll_stem->add(R"({"word": "foobar"})"_json.dump()).ok());
+
+ auto res = coll_stem->search("making", {"word"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ("foobar", res["hits"][0]["document"]["word"].get<std::string>());
}
\ No newline at end of file