Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

More tokenized stemming + more tests.

Kishore Nallan9bb75b173d42
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 90a22bf4..3ee7d655 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -139,6 +139,13 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
}
} else if(normalize && is_cyrillic(locale)) {
auto raw_text = unicode_text.tempSubStringBetween(start_pos, end_pos);
+ if(stemmer) {
+ std::string stemmed_word;
+ raw_text.toUTF8String(stemmed_word);
+ stemmed_word = stemmer->stem(stemmed_word);
+ raw_text = icu::UnicodeString::fromUTF8(stemmed_word);
+ }
+
transliterator->transliterate(raw_text);
raw_text.toUTF8String(word);
StringUtils::replace_all(word, "\"", "");
@@ -216,7 +223,8 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
}
}
- if(stemmer) {
+ if(stemmer && !is_cyrillic(locale)) {
+ // cyrillic is already stemmed prior to transliteration
token = stemmer->stem(out);
} else {
token = out;
diff --git a/test/collection_sorting_test.cpp b/test/collection_sorting_test.cpp
index 629257ef..d8806f3f 100644
--- a/test/collection_sorting_test.cpp
+++ b/test/collection_sorting_test.cpp
@@ -2212,59 +2212,6 @@ TEST_F(CollectionSortingTest, OptionalFilteringViaSortingWildcard) {
collectionManager.drop_collection("coll1");
}
-TEST_F(CollectionSortingTest, StemmingWithCaps) {
- nlohmann::json schema = R"({
- "name": "coll1",
- "fields": [
- {
- "facet": false,
- "index": true,
- "infix": true,
- "locale": "",
- "name": "name",
- "optional": false,
- "sort": false,
- "stem": false,
- "store": true,
- "type": "string"
- },
- {
- "facet": true,
- "index": true,
- "infix": true,
- "locale": "",
- "name": "subClass",
- "optional": true,
- "sort": false,
- "stem": true,
- "store": true,
- "type": "string"
- }
- ]
- })"_json;
-
- Collection* coll1 = collectionManager.create_collection(schema).get();
-
- nlohmann::json doc;
- doc["id"] = "0";
- doc["name"] = "Onion Coo Usa";
- doc["subClass"] = "ONIONS";
-
- ASSERT_TRUE(coll1->add(doc.dump()).ok());
-
- doc["id"] = "1";
- doc["name"] = "Mccormick Onion Dip Mix";
- doc["subClass"] = "GRAVY/SAUCE PACKETS";
-
- ASSERT_TRUE(coll1->add(doc.dump()).ok());
-
- std::vector<sort_by> sort_fields = {};
-
- auto res = coll1->search("onions", {"subClass","name"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
- ASSERT_STREQ("0", res["hits"][0]["document"]["id"].get<std::string>().c_str());
- ASSERT_STREQ("1", res["hits"][1]["document"]["id"].get<std::string>().c_str());
-}
-
TEST_F(CollectionSortingTest, OptionalFilteringViaSortingSearch) {
std::string coll_schema = R"(
{
diff --git a/test/collection_specific_more_test.cpp b/test/collection_specific_more_test.cpp
index 829e776d..a67b205a 100644
--- a/test/collection_specific_more_test.cpp
+++ b/test/collection_specific_more_test.cpp
@@ -2820,7 +2820,7 @@ TEST_F(CollectionSpecificMoreTest, DisableHighlightForLongFields) {
ASSERT_EQ(1, res_op.get()["hits"][0]["highlight"].size());
}
-TEST_F(CollectionSpecificMoreTest, TestStemming) {
+TEST_F(CollectionSpecificMoreTest, StemmingEnglish) {
nlohmann::json schema = R"({
"name": "test",
"fields": [
@@ -2829,7 +2829,6 @@ TEST_F(CollectionSpecificMoreTest, TestStemming) {
})"_json;
auto coll_stem_res = collectionManager.create_collection(schema);
- LOG(INFO) << coll_stem_res.error();
ASSERT_TRUE(coll_stem_res.ok());
auto coll_stem = coll_stem_res.get();
@@ -2842,27 +2841,124 @@ TEST_F(CollectionSpecificMoreTest, TestStemming) {
})"_json;
auto coll_no_stem_res = collectionManager.create_collection(schema);
- LOG(INFO) << coll_no_stem_res.error();
ASSERT_TRUE(coll_no_stem_res.ok());
-
auto coll_no_stem = coll_no_stem_res.get();
nlohmann::json doc;
-
doc["name"] = "running";
+
ASSERT_TRUE(coll_stem->add(doc.dump()).ok());
ASSERT_TRUE(coll_no_stem->add(doc.dump()).ok());
auto stem_res = coll_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1);
ASSERT_TRUE(stem_res.ok());
ASSERT_EQ(1, stem_res.get()["hits"].size());
- ASSERT_EQ("run", stem_res.get()["hits"][0]["highlight"]["name"]["matched_tokens"][0].get<std::string>());
+ ASSERT_EQ("running", stem_res.get()["hits"][0]["highlight"]["name"]["matched_tokens"][0].get<std::string>());
+ ASSERT_EQ("<mark>running</mark>", stem_res.get()["hits"][0]["highlight"]["name"]["snippet"].get<std::string>());
auto no_stem_res = coll_no_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1);
ASSERT_TRUE(no_stem_res.ok());
ASSERT_EQ(0, no_stem_res.get()["hits"].size());
}
+TEST_F(CollectionSpecificMoreTest, StemmingEnglishWithCaps) {
+ nlohmann::json schema = R"({
+ "name": "coll1",
+ "fields": [
+ {
+ "facet": false,
+ "index": true,
+ "infix": true,
+ "locale": "",
+ "name": "name",
+ "optional": false,
+ "sort": false,
+ "stem": false,
+ "store": true,
+ "type": "string"
+ },
+ {
+ "facet": true,
+ "index": true,
+ "infix": true,
+ "locale": "",
+ "name": "subClass",
+ "optional": true,
+ "sort": false,
+ "stem": true,
+ "store": true,
+ "type": "string"
+ }
+ ]
+ })"_json;
+
+ Collection* coll1 = collectionManager.create_collection(schema).get();
+
+ nlohmann::json doc;
+ doc["id"] = "0";
+ doc["name"] = "Onion Coo Usa";
+ doc["subClass"] = "ONIONS";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["id"] = "1";
+ doc["name"] = "Mccormick Onion Dip Mix";
+ doc["subClass"] = "GRAVY/SAUCE PACKETS";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ std::vector<sort_by> sort_fields = {};
+
+ auto res = coll1->search("onions", {"subClass","name"}, "", {}, sort_fields, {2}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_STREQ("0", res["hits"][0]["document"]["id"].get<std::string>().c_str());
+ ASSERT_STREQ("1", res["hits"][1]["document"]["id"].get<std::string>().c_str());
+}
+
+TEST_F(CollectionSpecificMoreTest, StemmingEnglishHighlights) {
+ nlohmann::json schema = R"({
+ "name": "test",
+ "fields": [
+ {"name": "name", "type": "string", "stem": true}
+ ]
+ })"_json;
+
+ auto coll_stem_res = collectionManager.create_collection(schema);
+ ASSERT_TRUE(coll_stem_res.ok());
+
+ auto coll_stem = coll_stem_res.get();
+
+ nlohmann::json doc;
+ doc["name"] = "Running runs";
+
+ ASSERT_TRUE(coll_stem->add(doc.dump()).ok());
+
+ auto res = coll_stem->search("run", {"name"}, {}, {}, {}, {0}, 10, 1, FREQUENCY, {false}, 1).get();
+ ASSERT_EQ(1, res["hits"].size());
+ ASSERT_EQ(2, res["hits"][0]["highlight"]["name"]["matched_tokens"].size());
+ ASSERT_EQ("Running", res["hits"][0]["highlight"]["name"]["matched_tokens"][0].get<std::string>());
+ ASSERT_EQ("runs", res["hits"][0]["highlight"]["name"]["matched_tokens"][1].get<std::string>());
+ ASSERT_EQ("<mark>Running</mark> <mark>runs</mark>", res["hits"][0]["highlight"]["name"]["snippet"].get<std::string>());
+}
+
+TEST_F(CollectionSpecificMoreTest, StemmingCyrilic) {
+ nlohmann::json schema = R"({
+ "name": "words",
+ "fields": [
+ {"name": "word", "type": "string", "stem": true, "locale": "ru"}
+ ]
+ })"_json;
+
+ auto coll_stem_res = collectionManager.create_collection(schema);
+ ASSERT_TRUE(coll_stem_res.ok());
+ auto coll_stem = coll_stem_res.get();
+
+ ASSERT_TRUE(coll_stem->add(R"({"word": "доверенное"})"_json.dump()).ok());
+ ASSERT_TRUE(coll_stem->add(R"({"word": "доверенные"})"_json.dump()).ok());
+
+ auto res = coll_stem->search("доверенное", {"word"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
+ ASSERT_EQ(2, res["hits"].size());
+}
+
TEST_F(CollectionSpecificMoreTest, NumDroppedTokensTest) {
nlohmann::json schema = R"({
"name": "coll1",
@@ -3097,22 +3193,3 @@ TEST_F(CollectionSpecificMoreTest, EnableTyposForAlphaNumericalTokens) {
ASSERT_EQ("(136)214", res["hits"][3]["document"]["title"].get<std::string>());
ASSERT_EQ("13/14", res["hits"][4]["document"]["title"].get<std::string>());
}
-
-TEST_F(CollectionSpecificMoreTest, TestStemmingCyrilic) {
- nlohmann::json schema = R"({
- "name": "words",
- "fields": [
- {"name": "word", "type": "string", "stem": true, "locale": "ru"}
- ]
- })"_json;
-
- auto coll_stem_res = collectionManager.create_collection(schema);
- ASSERT_TRUE(coll_stem_res.ok());
- auto coll_stem = coll_stem_res.get();
-
- ASSERT_TRUE(coll_stem->add(R"({"word": "доверенное"})"_json.dump()).ok());
- ASSERT_TRUE(coll_stem->add(R"({"word": "доверенные"})"_json.dump()).ok());
-
- auto res = coll_stem->search("доверенное", {"word"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}, 0).get();
- ASSERT_EQ(2, res["hits"].size());
-}
\ No newline at end of file