Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Support Chinese locale.
Kishore Nallanda3de68129d2
diff --git a/include/tokenizer.h b/include/tokenizer.hindex 8bf62803..04b7ef4e 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -5,6 +5,7 @@#include <iconv.h>#include <unicode/brkiter.h>#include <unicode/normalizer2.h>+#include <unicode/translit.h>#include "japanese_localizer.h"#include "logger.h"@@ -37,6 +38,8 @@ private:// non-deletable singletonconst icu::Normalizer2* nfkd;+ icu::Transliterator* transliterator = nullptr;+inline size_t get_stream_mode(char c) {return (std::isalnum(c) || index_symbols[uint8_t(c)] == 1) ? INDEX : ((c == ' ' || c == '\n') ? SEPARATE : SKIP@@ -58,6 +61,7 @@ public:iconv_close(cd);free(normalized_text);delete bi;+ delete transliterator;}bool next(std::string& token, size_t& token_index, size_t& start_index, size_t& end_index);diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 8b553405..3ce7b7bd 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -6,7 +6,26 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, constconst std::vector<char>& symbols_to_index):i(0), normalize(normalize), no_op(no_op), locale(locale) {- if(locale == "ja") {+ if(locale == "zh") {+ UErrorCode translit_status = U_ZERO_ERROR;+ transliterator = icu::Transliterator::createInstance("Traditional-Simplified",+ UTRANS_FORWARD, translit_status);+ if(U_FAILURE(translit_status)) {+ //LOG(ERROR) << "Unable to create transliteration instance for `zh` locale.";+ transliterator = nullptr;+ text = input;+ } else {+ icu::UnicodeString unicode_input = icu::UnicodeString::fromUTF8(input);+ transliterator->transliterate(unicode_input);+ std::string output;+ unicode_input.toUTF8String(output);+ normalized_text = (char *)malloc(output.size()+1);+ strcpy(normalized_text, output.c_str());+ text = normalized_text;+ }+ }++ else if(locale == "ja") {normalized_text = JapaneseLocalizer::get_instance().normalize(input);text = normalized_text;} else {@@ -81,6 +100,8 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_indefound_token = false;} else if(locale == "ko" && token == "·") {found_token = false;+ } else if(locale == "zh" && (token == "," || token == "─" || token == "。")) {+ found_token = false;} else {if(std::isalnum(token[0]) && is_ascii_char(token[0])) {diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex 47a68bae..8e96bad4 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -30,6 +30,70 @@ protected:}};+TEST_F(CollectionLocaleTest, SearchAgainstChineseText) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "zh"),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"爱并不会因时间而", "Dustin Kensrue"},+ {"很久以前,傳說在臺中北屯的一個地方", "Gord Downie"},+ {"獻給我思念的每一朵雲──海", "Dustin Kensrue"},+ {"看誰先跑到小山丘上。媽媽總是第", "Jamie Phua"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("并",+ {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, true).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("爱<mark>并不</mark>会因时间而", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ // partial token should not match as prefix when prefix is set to false++ results = coll1->search("并",+ {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(0, results["found"].get<size_t>());++ results = coll1->search("上媽",+ {"title", "artist"}, "", {}, {}, {0}, 10, 1, FREQUENCY, true).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("看誰先跑到小山丘<mark>上</mark>。<mark>媽媽</mark>總是第", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ // search using simplified chinese++ results = coll1->search("妈",+ {"title", "artist"}, "", {}, {}, {0}, 10, 1, FREQUENCY, true).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("3", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("看誰先跑到小山丘上。<mark>媽媽</mark>總是第", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+}+TEST_F(CollectionLocaleTest, SearchAgainstThaiText) {Collection *coll1;diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex e53811ba..36b846d7 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -157,6 +157,71 @@ TEST(TokenizerTest, ShouldTokenizeTextWithCustomSpecialChars) {ASSERT_EQ("-more", tokens[2]);}+TEST(TokenizerTest, ShouldTokenizeChineseText) {+ std::vector<std::string> tokens;++ // traditional -> simplified+ Tokenizer("語", false, false, "zh").tokenize(tokens);+ ASSERT_EQ(1, tokens.size());+ ASSERT_EQ("语", tokens[0]);++ tokens.clear();+ Tokenizer("說", false, false, "zh").tokenize(tokens);+ ASSERT_EQ(1, tokens.size());+ ASSERT_EQ("说", tokens[0]);++ // tokenize traditional+ tokens.clear();+ Tokenizer("愛並不會因時間而", false, false, "zh").tokenize(tokens);+ ASSERT_EQ(6, tokens.size());+ ASSERT_EQ("爱", tokens[0]);+ ASSERT_EQ("并不", tokens[1]);+ ASSERT_EQ("会", tokens[2]);+ ASSERT_EQ("因", tokens[3]);+ ASSERT_EQ("时间", tokens[4]);+ ASSERT_EQ("而", tokens[5]);++ // tokenize simplified+ tokens.clear();+ Tokenizer("爱并不会因时间而", false, false, "zh").tokenize(tokens);+ ASSERT_EQ(6, tokens.size());+ ASSERT_EQ("爱", tokens[0]);+ ASSERT_EQ("并不", tokens[1]);+ ASSERT_EQ("会", tokens[2]);+ ASSERT_EQ("因", tokens[3]);+ ASSERT_EQ("时间", tokens[4]);+ ASSERT_EQ("而", tokens[5]);++ // with separators+ tokens.clear();+ Tokenizer("很久以前,傳說在臺中北屯的一個地方", false, false, "zh").tokenize(tokens);+ ASSERT_EQ(10, tokens.size());+ ASSERT_EQ("很久", tokens[0]);+ ASSERT_EQ("以前", tokens[1]);+ ASSERT_EQ("传说", tokens[2]);+ ASSERT_EQ("在", tokens[3]);+ ASSERT_EQ("台中", tokens[4]);+ ASSERT_EQ("北", tokens[5]);+ ASSERT_EQ("屯", tokens[6]);+ ASSERT_EQ("的", tokens[7]);+ ASSERT_EQ("一个", tokens[8]);+ ASSERT_EQ("地方", tokens[9]);++ tokens.clear();+ Tokenizer("朵雲──海", false, false, "zh").tokenize(tokens);+ ASSERT_EQ(3, tokens.size());+ ASSERT_EQ("朵", tokens[0]);+ ASSERT_EQ("云", tokens[1]);+ ASSERT_EQ("海", tokens[2]);++ tokens.clear();+ Tokenizer("山丘上。媽媽", false, false, "zh").tokenize(tokens);+ ASSERT_EQ(3, tokens.size());+ ASSERT_EQ("山丘", tokens[0]);+ ASSERT_EQ("上", tokens[1]);+ ASSERT_EQ("妈妈", tokens[2]);+}+TEST(TokenizerTest, ShouldTokenizeLocaleText) {std::string tstr = "ลงรถไฟ";std::vector<std::string> ttokens;