Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix stemming resolved synonyms (#1638)

Ozan Armağand330e32638bd
diff --git a/src/index.cpp b/src/index.cpp
index 2c526bb4..6a1f804b 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -3085,7 +3085,8 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
}
synonym_index->synonym_reduction(q_include_tokens, field_query_tokens[0].q_synonyms);
- if (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem) {
+ const bool& do_stemming = (search_schema.find(the_fields[0].name) != search_schema.end() && search_schema.at(the_fields[0].name).stem);
+ if (do_stemming) {
auto stemmer = search_schema.at(the_fields[0].name).get_stemmer();
for(auto& q_include_token: q_include_tokens) {
q_include_token = stemmer->stem(q_include_token);
@@ -3104,7 +3105,12 @@ Option<bool> Index::search(std::vector<query_tokens_t>& field_query_tokens, cons
std::vector<token_t> q_pos_syn;
for(size_t j=0; j < q_syn_vec.size(); j++) {
bool is_prefix = (j == q_syn_vec.size()-1);
- q_pos_syn.emplace_back(j, q_syn_vec[j], is_prefix, q_syn_vec[j].size(), 0);
+ std::string token_val = q_syn_vec[j];
+ if (do_stemming) {
+ auto stemmer = search_schema.at(the_fields[0].name).get_stemmer();
+ token_val = stemmer->stem(q_syn_vec[j]);
+ }
+ q_pos_syn.emplace_back(j, token_val, is_prefix, token_val.size(), 0);
}
q_pos_synonyms.push_back(q_pos_syn);
diff --git a/test/collection_synonyms_test.cpp b/test/collection_synonyms_test.cpp
index 449245de..e927a1f7 100644
--- a/test/collection_synonyms_test.cpp
+++ b/test/collection_synonyms_test.cpp
@@ -1159,4 +1159,32 @@ TEST_F(CollectionSynonymsTest, SynonymsPagination) {
synonym_op = coll3->get_synonyms(limit, offset);
ASSERT_FALSE(synonym_op.ok());
ASSERT_EQ("Invalid offset param.", synonym_op.error());
+}
+
+TEST_F(CollectionSynonymsTest, SynonymWithStemming) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {"name": "name", "type": "string", "stem": true}
+ ]
+ })"_json;
+
+ auto coll1 = collectionManager.create_collection(schema).get();
+ std::vector<std::string> records = {"k8s", "kubernetes"};
+
+ for(size_t i = 0; i < records.size(); i++) {
+ nlohmann::json doc;
+ doc["id"] = std::to_string(i);
+ doc["name"] = records[i];
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ coll1->add_synonym(R"({"id": "syn-1", "synonyms": ["k8s", "kubernetes"]})"_json);
+
+ auto res = coll1->search("k8s", {"name"}, "", {}, {}, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+
+ ASSERT_EQ(2, res["hits"].size());
+ ASSERT_EQ(2, res["found"].get<uint32_t>());
+
+ collectionManager.drop_collection("coll1");
}
\ No newline at end of file