Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Normalize thai text via nfkc.
Kishore Nallan6729b72b1a0a
diff --git a/include/tokenizer.h b/include/tokenizer.hindex 35195637..36f3af90 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -36,8 +36,9 @@ private:int32_t utf8_start_index = 0;char* normalized_text = nullptr;- // non-deletable singleton- const icu::Normalizer2* nfkd;+ // non-deletable singletons+ const icu::Normalizer2* nfkd = nullptr;+ const icu::Normalizer2* nfkc = nullptr;icu::Transliterator* transliterator = nullptr;diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 3c9cb3d0..41aab66e 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -16,7 +16,14 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const}UErrorCode errcode = U_ZERO_ERROR;- nfkd = icu::Normalizer2::getNFKDInstance(errcode);++ if(locale == "ko") {+ nfkd = icu::Normalizer2::getNFKDInstance(errcode);+ }++ if(locale == "th") {+ nfkc = icu::Normalizer2::getNFKCInstance(errcode);+ }cd = iconv_open("ASCII//TRANSLIT", "UTF-8");@@ -119,6 +126,16 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_indeauto raw_text = unicode_text.tempSubStringBetween(start_pos, end_pos);transliterator->transliterate(raw_text);token = raw_text.toUTF8String(word);+ } else if(locale == "th") {+ UErrorCode errcode = U_ZERO_ERROR;+ icu::UnicodeString src = unicode_text.tempSubStringBetween(start_pos, end_pos);+ icu::UnicodeString dst;+ nfkc->normalize(src, dst, errcode);+ if(!U_FAILURE(errcode)) {+ token = dst.toUTF8String(word);+ } else {+ LOG(ERROR) << "Unicode error during parsing: " << errcode;+ }} else {token = unicode_text.tempSubStringBetween(start_pos, end_pos).toUTF8String(word);}diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex e76c5d2c..4940f2b9 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -187,6 +187,39 @@ TEST_F(CollectionLocaleTest, SearchAgainstThaiText) {ASSERT_EQ("<mark>พกไฟ</mark>\nเสมอ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());}+TEST_F(CollectionLocaleTest, ThaiTextShouldBeNormalizedToNFKC) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "th"),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"น้ำมัน", "Dustin Kensrue"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ auto results = coll1->search("น้ํามัน",{"title"}, "", {}, {},+ {0}, 10, 1, FREQUENCY).get();++ ASSERT_EQ(1, results["found"].get<size_t>());+}+TEST_F(CollectionLocaleTest, SearchThaiTextPreSegmentedQuery) {Collection *coll1;diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex 1fa43f78..258a7a17 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -241,7 +241,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {ASSERT_EQ(4, tokens.size());ASSERT_EQ("จิ้งจอก", tokens[0]);ASSERT_EQ("สี", tokens[1]);- ASSERT_EQ("น้ำตาล", tokens[2]);+ ASSERT_EQ("น้ําตาล", tokens[2]);ASSERT_EQ("ด่วน", tokens[3]);tokens.clear();