Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix edge cases in Japanese tokenization.
Kishore Nallan89d2b247fcd9
diff --git a/WORKSPACE b/WORKSPACEindex 21cb0331..2948d081 100644--- a/WORKSPACE+++ b/WORKSPACE@@ -168,7 +168,7 @@ new_git_repository(new_git_repository(name = "kakasi",build_file = "//bazel:kakasi.BUILD",- commit = "9e0825a02c7ea5605e968f6208f769f7c49d6860",+ commit = "77f2d1ce0146d15199ae0db1e61e0b699b0b55f6",remote = "https://github.com/typesense/kakasi.git",)diff --git a/src/collection.cpp b/src/collection.cppindex 63db8c19..15559c85 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -2735,7 +2735,8 @@ void Collection::highlight_result(const std::string& raw_query, const field &seatsl::htrie_set<char> matched_tokens;- bool use_word_tokenizer = search_field.locale == "th" || Tokenizer::is_cyrillic(search_field.locale);+ bool use_word_tokenizer = search_field.locale == "th" || search_field.locale == "ja" ||+ Tokenizer::is_cyrillic(search_field.locale);bool normalise = !use_word_tokenizer;std::vector<std::string> raw_query_tokens;diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 45192787..17697bf4 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -60,8 +60,12 @@ void Tokenizer::init(const std::string& input) {}else if(locale == "ja") {- normalized_text = JapaneseLocalizer::get_instance().normalize(input);- text = normalized_text;+ if(normalize) {+ normalized_text = JapaneseLocalizer::get_instance().normalize(input);+ text = normalized_text;+ } else {+ text = input;+ }} else if(is_cyrillic(locale)) {// init transliterator but will only transliterate during tokenizationUErrorCode translit_status = U_ZERO_ERROR;@@ -133,6 +137,10 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde} else {LOG(ERROR) << "Unicode error during parsing: " << errcode;}+ } else if(normalize && locale == "ja") {+ auto raw_text = unicode_text.tempSubStringBetween(start_pos, end_pos);+ raw_text.toUTF8String(word);+ JapaneseLocalizer::get_instance().normalize(word);} else {unicode_text.tempSubStringBetween(start_pos, end_pos).toUTF8String(word);}diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex e9a2660c..b58a1a03 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -774,6 +774,27 @@ TEST_F(CollectionLocaleTest, SearchOnCyrillicLargeText) {results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());}+TEST_F(CollectionLocaleTest, SearchOnJapaneseLargeText) {+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, "ja"),};+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc;+ doc["title"] = "王獣を倒すと入手した折れた角。追放された後、この世に存在すべきではないもの。\n獣域ウルブズの中で帝王と呼ばれていても、"+ "魔獣たちの系譜では、その兄たちの万分の一にも満たないだろう。\n「黄"+ "金」が無数の獣域ウルブズを捨て紙のように圧縮して偶然にできた異形の魔獣。その角には、黒いウルブズを命じて自分のため"+ "に空間を溶かす権威が秘めている。";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("王獣を", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_STREQ("<mark>王</mark><mark>獣</mark><mark>を</mark><mark>倒す</mark>と入手した折",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());++ results = coll1->search("業果材", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_STREQ("に空間を溶かす<mark>権威</mark><mark>が</mark><mark>秘</mark>めている。",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>().c_str());+}+TEST_F(CollectionLocaleTest, SearchOnArabicText) {std::vector<field> fields = {field("title", field_types::STRING, true, false, true, ""),};Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex eaabe031..ff416f7d 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -223,6 +223,26 @@ TEST(TokenizerTest, ShouldTokenizeChineseText) {}TEST(TokenizerTest, ShouldTokenizeLocaleText) {+ std::string text = R"("王獣を倒すと入手した折れた角。追放された後、この世に存在すべきではないもの。\n獣域ウルブズの中で帝王と呼ばれていても、魔獣たちの系譜では、その兄たちの万分の一にも満たないだろう。\n「黄+金」が無数の獣域ウルブズを捨て紙のように圧縮して偶然にできた異形の魔獣。その角には、黒いウルブズを命じて自分のために空間を溶かす権威が秘めている。")";++ Tokenizer tokenizer(text, true, false, "ja", {}, {});++ LOG(INFO) << "text.size: " << text.size();++ std::string raw_token;+ size_t raw_token_index = 0, tok_start = 0, tok_end = 0;++ // based on `highlight_affix_num_tokens`+ size_t snippet_start_offset = 0, snippet_end_offset = (text.empty() ? 0 : text.size() - 1);++ // window used to locate the starting offset for snippet on the text+ while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {+ LOG(INFO) << "tok_start: " << tok_start;+ }++ return ;+std::vector<std::string> tokens;tokens.clear();@@ -283,6 +303,14 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {ASSERT_EQ("た", tokens[0]);ASSERT_EQ("いくつ", tokens[1]);+ tokens.clear();+ Tokenizer("魈", true, false, "ja").tokenize(tokens);+ ASSERT_EQ(0, tokens.size());++ tokens.clear();+ Tokenizer("「業果材", true, false, "ja").tokenize(tokens);+ ASSERT_EQ(6, tokens.size());+tokens.clear();Tokenizer("ア退屈であ", false, false, "ja").tokenize(tokens);ASSERT_EQ(5, tokens.size());