Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix stemming resolved synonyms (#1638)
Ozan Armağand330e32638bd
diff --git a/src/index.cpp b/src/index.cppindex 2c526bb4..6a1f804b 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -3085,7 +3085,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons}synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);- if (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem) {+ const bool& do_stemming = (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem);+ if (do_stemming) {auto stemmer = search_schema.at(the_fields[0].name).get_stemmer();for(auto& q_include_token: q_include_tokens) {q_include_token = stemmer->stem(q_include_token);@@ -3104,7 +3105,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, consstd::vector<token_t> q_pos_syn;for(size_t j=0; j < q_syn_vec.size(); j++) {bool is_prefix = (j == q_syn_vec.size()-1);- q_pos_syn.emplace_back(j, q_syn_vec[j], is_prefix, q_syn_vec[j].size(), 0);+ std::string token_val = q_syn_vec[j];+ if (do_stemming) {+ auto stemmer = search_schema.at(the_fields[0].name).get_stemmer();+ token_val = stemmer->stem(q_syn_vec[j]);+ }+ q_pos_syn.emplace_back(j, token_val, is_prefix, token_val.size(), 0);}q_pos_synonyms.push_back(q_pos_syn);diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cppindex 449245de..e927a1f7 100644--- a/test/collection_synonyms_test.cpp+++ b/test/collection_synonyms_test.cpp@@ -1159,4 +1159,32 @@ TEST_F(CollectionSynonymsTest, SynonymsPagination) {synonym_op = coll3->get_synonyms(limit, offset);ASSERT_FALSE(synonym_op.ok());ASSERT_EQ("Invalid offset param.", synonym_op.error());+}++TEST_F(CollectionSynonymsTest, SynonymWithStemming) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {"name": "name", "type": "string", "stem": true}+ ]+ })"_json;++ auto coll1 = collectionManager.create_collection(schema).get();+ std::vector<std::string> records = {"k8s", "kubernetes"};++ for(size_t i = 0; i < records.size(); i++) {+ nlohmann::json doc;+ doc["id"] = std::to_string(i);+ doc["name"] = records[i];+ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ coll1->add_synonym(R"({"id": "syn-1", "synonyms": ["k8s", "kubernetes"]})"_json);++ auto res = coll1->search("k8s", {"name"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();++ ASSERT_EQ(2, res["hits"].size());+ ASSERT_EQ(2, res["found"].get<uint32_t>());++ collectionManager.drop_collection("coll1");}\ No newline at end of file