Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix edge cases in Japanese tokenization.

Kishore Nallan89d2b247fcd9
diff --git a/WORKSPACE b/WORKSPACE
index 21cb0331..2948d081 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -168,7 +168,7 @@ new_git_repository(
new_git_repository(
name = "kakasi",
build_file = "//bazel:kakasi.BUILD",
- commit = "9e0825a02c7ea5605e968f6208f769f7c49d6860",
+ commit = "77f2d1ce0146d15199ae0db1e61e0b699b0b55f6",
remote = "https://github.com/typesense/kakasi.git",
)
diff --git a/src/collection.cpp b/src/collection.cpp
index 63db8c19..15559c85 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -2735,7 +2735,8 @@ void Collection::highlight_result(const std::string& raw_query, const field &sea
tsl::htrie_set<char> matched_tokens;
- bool use_word_tokenizer = search_field.locale == "th" || Tokenizer::is_cyrillic(search_field.locale);
+ bool use_word_tokenizer = search_field.locale == "th" || search_field.locale == "ja" ||
+ Tokenizer::is_cyrillic(search_field.locale);
bool normalise = !use_word_tokenizer;
std::vector<std::string> raw_query_tokens;
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 45192787..17697bf4 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -60,8 +60,12 @@ void Tokenizer::init(const std::string& input) {
}
else if(locale == "ja") {
- normalized_text = JapaneseLocalizer::get_instance().normalize(input);
- text = normalized_text;
+ if(normalize) {
+ normalized_text = JapaneseLocalizer::get_instance().normalize(input);
+ text = normalized_text;
+ } else {
+ text = input;
+ }
} else if(is_cyrillic(locale)) {
// init transliterator but will only transliterate during tokenization
UErrorCode translit_status = U_ZERO_ERROR;
@@ -133,6 +137,10 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
} else {
LOG(ERROR) << "Unicode error during parsing: " << errcode;
}
+ } else if(normalize && locale == "ja") {
+ auto raw_text = unicode_text.tempSubStringBetween(start_pos, end_pos);
+ raw_text.toUTF8String(word);
+ JapaneseLocalizer::get_instance().normalize(word);
} else {
unicode_text.tempSubStringBetween(start_pos, end_pos).toUTF8String(word);
}
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index e9a2660c..b58a1a03 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -774,6 +774,27 @@ TEST_F(CollectionLocaleTest, SearchOnCyrillicLargeText) {
results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
}
+TEST_F(CollectionLocaleTest, SearchOnJapaneseLargeText) {
+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, "ja"),};
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc;
+ doc["title"] = "王獣を倒すと入手した折れた角。追放された後、この世に存在すべきではないもの。\n獣域ウルブズの中で帝王と呼ばれていても、"
+ "魔獣たちの系譜では、その兄たちの万分の一にも満たないだろう。\n「黄"
+ "金」が無数の獣域ウルブズを捨て紙のように圧縮して偶然にできた異形の魔獣。その角には、黒いウルブズを命じて自分のため"
+ "に空間を溶かす権威が秘めている。";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("王獣を", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_STREQ("<mark>王</mark><mark>獣</mark><mark>を</mark><mark>倒す</mark>と入手した折",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+
+ results = coll1->search("業果材", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_STREQ("に空間を溶かす<mark>権威</mark><mark>が</mark><mark>秘</mark>めている。",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());
+}
+
TEST_F(CollectionLocaleTest, SearchOnArabicText) {
std::vector<field> fields = {field("title", field_types::STRING, true, false, true, ""),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index eaabe031..ff416f7d 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -223,6 +223,26 @@ TEST(TokenizerTest, ShouldTokenizeChineseText) {
}
TEST(TokenizerTest, ShouldTokenizeLocaleText) {
+ std::string text = R"("王獣を倒すと入手した折れた角。追放された後、この世に存在すべきではないもの。\n獣域ウルブズの中で帝王と呼ばれていても、魔獣たちの系譜では、その兄たちの万分の一にも満たないだろう。\n「黄
+金」が無数の獣域ウルブズを捨て紙のように圧縮して偶然にできた異形の魔獣。その角には、黒いウルブズを命じて自分のために空間を溶かす権威が秘めている。")";
+
+ Tokenizer tokenizer(text, true, false, "ja", {}, {});
+
+ LOG(INFO) << "text.size: " << text.size();
+
+ std::string raw_token;
+ size_t raw_token_index = 0, tok_start = 0, tok_end = 0;
+
+ // based on `highlight_affix_num_tokens`
+ size_t snippet_start_offset = 0, snippet_end_offset = (text.empty() ? 0 : text.size() - 1);
+
+ // window used to locate the starting offset for snippet on the text
+ while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {
+ LOG(INFO) << "tok_start: " << tok_start;
+ }
+
+ return ;
+
std::vector<std::string> tokens;
tokens.clear();
@@ -283,6 +303,14 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
ASSERT_EQ("た", tokens[0]);
ASSERT_EQ("いくつ", tokens[1]);
+ tokens.clear();
+ Tokenizer("魈", true, false, "ja").tokenize(tokens);
+ ASSERT_EQ(0, tokens.size());
+
+ tokens.clear();
+ Tokenizer("「業果材", true, false, "ja").tokenize(tokens);
+ ASSERT_EQ(6, tokens.size());
+
tokens.clear();
Tokenizer("ア退屈であ", false, false, "ja").tokenize(tokens);
ASSERT_EQ(5, tokens.size());