Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix stemming bug when searching with synonyms (#1569)
Ozan Armağan2c21e1306bb3
diff --git a/include/collection.h b/include/collection.hindex 44d84092..a14c00de 100644--- a/include/collection.h+++ b/include/collection.h@@ -492,7 +492,7 @@ public:void parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,std::vector<std::vector<std::string>>& q_exclude_tokens,std::vector<std::vector<std::string>>& q_phrases,- const std::string& locale, const bool already_segmented, const std::string& stopword_set="",const std::shared_ptr<Stemmer> stemmer = nullptr) const;+ const std::string& locale, const bool already_segmented, const std::string& stopword_set="") const;// PUBLIC OPERATIONSdiff --git a/src/collection.cpp b/src/collection.cppindex 62e2c726..6947a63d 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2259,15 +2259,10 @@ Option<nlohmann::json> Collection::search(std::string raw_query,auto most_weighted_field = search_schema.at(weighted_search_fields[0].name);const std::string & field_locale = most_weighted_field.locale;- std::shared_ptr<Stemmer> stemmer = nullptr;- if(most_weighted_field.stem) {- stemmer = most_weighted_field.get_stemmer();- }-parse_search_query(query, q_include_tokens,field_query_tokens[0].q_exclude_tokens,field_query_tokens[0].q_phrases,- field_locale, pre_segmented_query, stopwords_set, stemmer);+ field_locale, pre_segmented_query, stopwords_set);// process filter overrides first, before synonyms (order is important)@@ -3386,7 +3381,7 @@ void Collection::process_filter_overrides(std::vector<const override_t*>& filtervoid Collection::parse_search_query(const std::string &query, std::vector<std::string>& q_include_tokens,std::vector<std::vector<std::string>>& q_exclude_tokens,std::vector<std::vector<std::string>>& q_phrases,- const std::string& locale, const bool already_segmented, const std::string& stopwords_set, const std::shared_ptr<Stemmer> stemmer) const {+ const std::string& locale, const bool already_segmented, const std::string& stopwords_set) const {if(query == "*") {q_exclude_tokens = {};q_include_tokens = {query};@@ -3448,9 +3443,6 @@ void Collection::parse_search_query(const std::string &query, std::vector<std::s}}- if(stemmer) {- token = stemmer->stem(token);- }// retokenize using collection config (handles hyphens being part of the query)std::vector<std::string> sub_tokens;diff --git a/src/index.cpp b/src/index.cppindex 67d5aa58..7497f526 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -3198,12 +3198,22 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<std::vector<token_t>> q_pos_synonyms;std::vector<std::string> q_include_tokens;int syn_orig_num_tokens = -1;-for(size_t j = 0; j < field_query_tokens[0].q_include_tokens.size(); j++) {q_include_tokens.push_back(field_query_tokens[0].q_include_tokens[j].value);}synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);+ if (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem) {+ auto stemmer = search_schema.at(the_fields[0].name).get_stemmer();+ for(auto& q_include_token: q_include_tokens) {+ q_include_token = stemmer->stem(q_include_token);+ }++ for(auto& q_token: field_query_tokens[0].q_include_tokens) {+ q_token.value = stemmer->stem(q_token.value);+ }+ }+if(!field_query_tokens[0].q_synonyms.empty()) {syn_orig_num_tokens = field_query_tokens[0].q_include_tokens.size();}diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 0e2ad026..615db51b 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -2952,4 +2952,34 @@ TEST_F(CollectionSpecificMoreTest, TestStemming2) {auto res = coll_stem->search("Walking", {"word"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();ASSERT_EQ(7, res["hits"].size());+}++TEST_F(CollectionSpecificMoreTest, TestStemmingWithSynonym) {+ nlohmann::json schema = R"({+ "name": "words",+ "fields": [+ {"name": "word", "type": "string", "stem": true }+ ]+ })"_json;++ auto coll_stem_res = collectionManager.create_collection(schema);+ ASSERT_TRUE(coll_stem_res.ok());++ auto coll_stem = coll_stem_res.get();++ nlohmann::json synonym_json = R"(+ {+ "id": "",+ "synonyms": ["making", "foobar"]+ }+ )"_json;+ auto synonym_op = coll_stem->add_synonym(synonym_json);+ LOG(INFO) << synonym_op.error();+ ASSERT_TRUE(synonym_op.ok());++ ASSERT_TRUE(coll_stem->add(R"({"word": "foobar"})"_json.dump()).ok());++ auto res = coll_stem->search("making", {"word"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ("foobar", res["hits"][0]["document"]["word"].get<std::string>());}\ No newline at end of file