Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚑ πŸ” ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix test.

Kishore Nalland17d3356d12a
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index 25515f83..8d1f787b 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -278,7 +278,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
// japanese
tokens.clear();
- Tokenizer("ι€€ε±ˆ", false, false, "ja").tokenize(tokens);
+ Tokenizer("ι€€ε±ˆ", true, false, "ja").tokenize(tokens);
ASSERT_EQ(2, tokens.size());
ASSERT_EQ("た", tokens[0]);
ASSERT_EQ("いく぀", tokens[1]);
@@ -292,7 +292,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
ASSERT_EQ(6, tokens.size());
tokens.clear();
- Tokenizer("γ‚’ι€€ε±ˆγ§γ‚", false, false, "ja").tokenize(tokens);
+ Tokenizer("γ‚’ι€€ε±ˆγ§γ‚", true, false, "ja").tokenize(tokens);
ASSERT_EQ(5, tokens.size());
ASSERT_EQ("あ", tokens[0]);
ASSERT_EQ("た", tokens[1]);
@@ -301,7 +301,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
ASSERT_EQ("あ", tokens[4]);
tokens.clear();
- Tokenizer("ζ€ ζƒ°γͺ犬", false, false, "ja").tokenize(tokens);
+ Tokenizer("ζ€ ζƒ°γͺ犬", true, false, "ja").tokenize(tokens);
ASSERT_EQ(4, tokens.size());
ASSERT_EQ("γŸγ„", tokens[0]);
ASSERT_EQ("だ", tokens[1]);
@@ -309,7 +309,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
ASSERT_EQ("いぬ", tokens[3]);
tokens.clear();
- Tokenizer("δ»ŠγΆγ‚Šζ‹ζ²»γƒ«γƒ„", false, false, "ja").tokenize(tokens);
+ Tokenizer("δ»ŠγΆγ‚Šζ‹ζ²»γƒ«γƒ„", true, false, "ja").tokenize(tokens);
ASSERT_EQ(9, tokens.size());
ASSERT_EQ("いま", tokens[0]);
ASSERT_EQ("γΆγ‚Š", tokens[1]);