Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
More tokenized stemming + more tests.
Kishore Nallan9bb75b173d42
diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 90a22bf4..3ee7d655 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -139,6 +139,13 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde}} else if(normalize && is_cyrillic(locale)) {auto raw_text = unicode_text.tempSubStringBetween(start_pos, end_pos);+ if(stemmer) {+ std::string stemmed_word;+ raw_text.toUTF8String(stemmed_word);+ stemmed_word = stemmer->stem(stemmed_word);+ raw_text = icu::UnicodeString::fromUTF8(stemmed_word);+ }+transliterator->transliterate(raw_text);raw_text.toUTF8String(word);StringUtils::replace_all(word, "\"", "");@@ -216,7 +223,8 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde}}- if(stemmer) {+ if(stemmer && !is_cyrillic(locale)) {+ // cyrillic is already stemmed prior to transliterationtoken = stemmer->stem(out);} else {token = out;diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cppindex 629257ef..d8806f3f 100644--- a/test/collection_sorting_test.cpp+++ b/test/collection_sorting_test.cpp@@ -2212,59 +2212,6 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {collectionManager.drop_collection("coll1");}-TEST_F(CollectionSortingTest, StemmingWithCaps) {- nlohmann::json schema = R"({- "name": "coll1",- "fields": [- {- "facet": false,- "index": true,- "infix": true,- "locale": "",- "name": "name",- "optional": false,- "sort": false,- "stem": false,- "store": true,- "type": "string"- },- {- "facet": true,- "index": true,- "infix": true,- "locale": "",- "name": "subClass",- "optional": true,- "sort": false,- "stem": true,- "store": true,- "type": "string"- }- ]- })"_json;-- Collection* coll1 = collectionManager.create_collection(schema).get();-- nlohmann::json doc;- doc["id"] = "0";- doc["name"] = "Onion Coo Usa";- doc["subClass"] = "ONIONS";-- ASSERT_TRUE(coll1->add(doc.dump()).ok());-- doc["id"] = "1";- doc["name"] = "Mccormick Onion Dip Mix";- doc["subClass"] = "GRAVY/SAUCE PACKETS";-- ASSERT_TRUE(coll1->add(doc.dump()).ok());-- std::vector<sort_by> sort_fields = {};-- auto res = coll1->search("onions", {"subClass","name"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();- ASSERT_STREQ("0", res["hits"][0]["document"]["id"].get<std::string>().c_str());- ASSERT_STREQ("1", res["hits"][1]["document"]["id"].get<std::string>().c_str());-}-TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {std::string coll_schema = R"({diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cppindex 829e776d..a67b205a 100644--- a/test/collection_specific_more_test.cpp+++ b/test/collection_specific_more_test.cpp@@ -2820,7 +2820,7 @@ TEST_F(CollectionSpecificMoreTest, DisableHighlightForLongFields) {ASSERT_EQ(1, res_op.get()["hits"][0]["highlight"].size());}-TEST_F(CollectionSpecificMoreTest, TestStemming) {+TEST_F(CollectionSpecificMoreTest, StemmingEnglish) {nlohmann::json schema = R"({"name": "test","fields": [@@ -2829,7 +2829,6 @@ TEST_F(CollectionSpecificMoreTest, TestStemming) {})"_json;auto coll_stem_res = collectionManager.create_collection(schema);- LOG(INFO) << coll_stem_res.error();ASSERT_TRUE(coll_stem_res.ok());auto coll_stem = coll_stem_res.get();@@ -2842,27 +2841,124 @@ TEST_F(CollectionSpecificMoreTest, TestStemming) {})"_json;auto coll_no_stem_res = collectionManager.create_collection(schema);- LOG(INFO) << coll_no_stem_res.error();ASSERT_TRUE(coll_no_stem_res.ok());-auto coll_no_stem = coll_no_stem_res.get();nlohmann::json doc;-doc["name"] = "running";+ASSERT_TRUE(coll_stem->add(doc.dump()).ok());ASSERT_TRUE(coll_no_stem->add(doc.dump()).ok());auto stem_res = coll_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1);ASSERT_TRUE(stem_res.ok());ASSERT_EQ(1, stem_res.get()["hits"].size());- ASSERT_EQ("run", stem_res.get()["hits"][0]["highlight"]["name"]["matched_tokens"][0].get<std::string>());+ ASSERT_EQ("running", stem_res.get()["hits"][0]["highlight"]["name"]["matched_tokens"][0].get<std::string>());+ ASSERT_EQ("<mark>running</mark>", stem_res.get()["hits"][0]["highlight"]["name"]["snippet"].get<std::string>());auto no_stem_res = coll_no_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1);ASSERT_TRUE(no_stem_res.ok());ASSERT_EQ(0, no_stem_res.get()["hits"].size());}+TEST_F(CollectionSpecificMoreTest, StemmingEnglishWithCaps) {+ nlohmann::json schema = R"({+ "name": "coll1",+ "fields": [+ {+ "facet": false,+ "index": true,+ "infix": true,+ "locale": "",+ "name": "name",+ "optional": false,+ "sort": false,+ "stem": false,+ "store": true,+ "type": "string"+ },+ {+ "facet": true,+ "index": true,+ "infix": true,+ "locale": "",+ "name": "subClass",+ "optional": true,+ "sort": false,+ "stem": true,+ "store": true,+ "type": "string"+ }+ ]+ })"_json;++ Collection* coll1 = collectionManager.create_collection(schema).get();++ nlohmann::json doc;+ doc["id"] = "0";+ doc["name"] = "Onion Coo Usa";+ doc["subClass"] = "ONIONS";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["id"] = "1";+ doc["name"] = "Mccormick Onion Dip Mix";+ doc["subClass"] = "GRAVY/SAUCE PACKETS";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ std::vector<sort_by> sort_fields = {};++ auto res = coll1->search("onions", {"subClass","name"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_STREQ("0", res["hits"][0]["document"]["id"].get<std::string>().c_str());+ ASSERT_STREQ("1", res["hits"][1]["document"]["id"].get<std::string>().c_str());+}++TEST_F(CollectionSpecificMoreTest, StemmingEnglishHighlights) {+ nlohmann::json schema = R"({+ "name": "test",+ "fields": [+ {"name": "name", "type": "string", "stem": true}+ ]+ })"_json;++ auto coll_stem_res = collectionManager.create_collection(schema);+ ASSERT_TRUE(coll_stem_res.ok());++ auto coll_stem = coll_stem_res.get();++ nlohmann::json doc;+ doc["name"] = "Running runs";++ ASSERT_TRUE(coll_stem->add(doc.dump()).ok());++ auto res = coll_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1).get();+ ASSERT_EQ(1, res["hits"].size());+ ASSERT_EQ(2, res["hits"][0]["highlight"]["name"]["matched_tokens"].size());+ ASSERT_EQ("Running", res["hits"][0]["highlight"]["name"]["matched_tokens"][0].get<std::string>());+ ASSERT_EQ("runs", res["hits"][0]["highlight"]["name"]["matched_tokens"][1].get<std::string>());+ ASSERT_EQ("<mark>Running</mark> <mark>runs</mark>", res["hits"][0]["highlight"]["name"]["snippet"].get<std::string>());+}++TEST_F(CollectionSpecificMoreTest, StemmingCyrilic) {+ nlohmann::json schema = R"({+ "name": "words",+ "fields": [+ {"name": "word", "type": "string", "stem": true, "locale": "ru"}+ ]+ })"_json;++ auto coll_stem_res = collectionManager.create_collection(schema);+ ASSERT_TRUE(coll_stem_res.ok());+ auto coll_stem = coll_stem_res.get();++ ASSERT_TRUE(coll_stem->add(R"({"word": "доверенное"})"_json.dump()).ok());+ ASSERT_TRUE(coll_stem->add(R"({"word": "доверенные"})"_json.dump()).ok());++ auto res = coll_stem->search("доверенное", {"word"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();+ ASSERT_EQ(2, res["hits"].size());+}+TEST_F(CollectionSpecificMoreTest, NumDroppedTokensTest) {nlohmann::json schema = R"({"name": "coll1",@@ -3097,22 +3193,3 @@ TEST_F(CollectionSpecificMoreTest, EnableTyposForAlphaNumericalTokens) {ASSERT_EQ("(136)214", res["hits"][3]["document"]["title"].get<std::string>());ASSERT_EQ("13/14", res["hits"][4]["document"]["title"].get<std::string>());}--TEST_F(CollectionSpecificMoreTest, TestStemmingCyrilic) {- nlohmann::json schema = R"({- "name": "words",- "fields": [- {"name": "word", "type": "string", "stem": true, "locale": "ru"}- ]- })"_json;-- auto coll_stem_res = collectionManager.create_collection(schema);- ASSERT_TRUE(coll_stem_res.ok());- auto coll_stem = coll_stem_res.get();-- ASSERT_TRUE(coll_stem->add(R"({"word": "доверенное"})"_json.dump()).ok());- ASSERT_TRUE(coll_stem->add(R"({"word": "доверенные"})"_json.dump()).ok());-- auto res = coll_stem->search("доверенное", {"word"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();- ASSERT_EQ(2, res["hits"].size());-}\ No newline at end of file