Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Do stemming after other tokenization processing.
Kishore Nallan62d62ec3a3b0
diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex f9bb5062..90a22bf4 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -29,12 +29,7 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, constcd = iconv_open("ASCII//TRANSLIT", "UTF-8");- if(stemmer) {- auto stemmed_input = stemmer->stem(input);- init(stemmed_input);- } else {- init(input);- }+ init(input);}@@ -221,7 +216,12 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde}}- token = out;+ if(stemmer) {+ token = stemmer->stem(out);+ } else {+ token = out;+ }+out.clear();start_index = utf8_start_index;end_index = text.size() - 1;@@ -249,7 +249,12 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_indecontinue;}- token = out;+ if(stemmer) {+ token = stemmer->stem(out);+ } else {+ token = out;+ }+out.clear();token_index = token_counter++;@@ -320,7 +325,12 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde}}- token = out;+ if(stemmer) {+ token = stemmer->stem(out);+ } else {+ token = out;+ }+out.clear();end_index = i - 1;diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex d8806f3f..629257ef 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -2212,6 +2212,59 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {collectionManager.drop_collection("coll1");}+TEST_F(CollectionSortingTest, StemmingWithCaps) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {+ "facet": false,+ "index": true,+ "infix": true,+ "locale": "",+ "name": "name",+ "optional": false,+ "sort": false,+ "stem": false,+ "store": true,+ "type": "string"+ },+ {+ "facet": true,+ "index": true,+ "infix": true,+ "locale": "",+ "name": "subClass",+ "optional": true,+ "sort": false,+ "stem": true,+ "store": true,+ "type": "string"+ }+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["name"] = "Onion Coo Usa";+ doc["subClass"] = "ONIONS";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["id"] = "1";+ doc["name"] = "Mccormick Onion Dip Mix";+ doc["subClass"] = "GRAVY/SAUCE PACKETS";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ std::vector<sort_by> sort_fields = {};++ auto res = coll1->search("onions", {"subClass","name"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_STREQ("0", res["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", res["hits"][1]["document"]["id"].get<std::string>().c_str());+}+TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {std::string coll_schema = R"({