Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Do stemming after other tokenization processing.

Kishore Nallan62d62ec3a3b0
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index f9bb5062..90a22bf4 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -29,12 +29,7 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const
cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
- if(stemmer) {
- auto stemmed_input = stemmer->stem(input);
- init(stemmed_input);
- } else {
- init(input);
- }
+ init(input);
}
@@ -221,7 +216,12 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
}
}
- token = out;
+ if(stemmer) {
+ token = stemmer->stem(out);
+ } else {
+ token = out;
+ }
+
out.clear();
start_index = utf8_start_index;
end_index = text.size() - 1;
@@ -249,7 +249,12 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
continue;
}
- token = out;
+ if(stemmer) {
+ token = stemmer->stem(out);
+ } else {
+ token = out;
+ }
+
out.clear();
token_index = token_counter++;
@@ -320,7 +325,12 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
}
}
- token = out;
+ if(stemmer) {
+ token = stemmer->stem(out);
+ } else {
+ token = out;
+ }
+
out.clear();
end_index = i - 1;
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index d8806f3f..629257ef 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -2212,6 +2212,59 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {
collectionManager.drop_collection("coll1");
}
+TEST_F(CollectionSortingTest, StemmingWithCaps) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {
+ "facet": false,
+ "index": true,
+ "infix": true,
+ "locale": "",
+ "name": "name",
+ "optional": false,
+ "sort": false,
+ "stem": false,
+ "store": true,
+ "type": "string"
+ },
+ {
+ "facet": true,
+ "index": true,
+ "infix": true,
+ "locale": "",
+ "name": "subClass",
+ "optional": true,
+ "sort": false,
+ "stem": true,
+ "store": true,
+ "type": "string"
+ }
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["name"] = "Onion Coo Usa";
+ doc["subClass"] = "ONIONS";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["id"] = "1";
+ doc["name"] = "Mccormick Onion Dip Mix";
+ doc["subClass"] = "GRAVY/SAUCE PACKETS";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ std::vector<sort_by> sort_fields = {};
+
+ auto res = coll1->search("onions", {"subClass","name"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_STREQ("0", res["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", res["hits"][1]["document"]["id"].get<std::string>().c_str());
+}
+
TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {
std::string coll_schema = R"(
{