Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Normalize thai text via nfkc.

Kishore Nallan6729b72b1a0a
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 35195637..36f3af90 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -36,8 +36,9 @@ private:
int32_t utf8_start_index = 0;
char* normalized_text = nullptr;
- // non-deletable singleton
- const icu::Normalizer2* nfkd;
+ // non-deletable singletons
+ const icu::Normalizer2* nfkd = nullptr;
+ const icu::Normalizer2* nfkc = nullptr;
icu::Transliterator* transliterator = nullptr;
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 3c9cb3d0..41aab66e 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -16,7 +16,14 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const
}
UErrorCode errcode = U_ZERO_ERROR;
- nfkd = icu::Normalizer2::getNFKDInstance(errcode);
+
+ if(locale == "ko") {
+ nfkd = icu::Normalizer2::getNFKDInstance(errcode);
+ }
+
+ if(locale == "th") {
+ nfkc = icu::Normalizer2::getNFKCInstance(errcode);
+ }
cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
@@ -119,6 +126,16 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
auto raw_text = unicode_text.tempSubStringBetween(start_pos, end_pos);
transliterator->transliterate(raw_text);
token = raw_text.toUTF8String(word);
+ } else if(locale == "th") {
+ UErrorCode errcode = U_ZERO_ERROR;
+ icu::UnicodeString src = unicode_text.tempSubStringBetween(start_pos, end_pos);
+ icu::UnicodeString dst;
+ nfkc->normalize(src, dst, errcode);
+ if(!U_FAILURE(errcode)) {
+ token = dst.toUTF8String(word);
+ } else {
+ LOG(ERROR) << "Unicode error during parsing: " << errcode;
+ }
} else {
token = unicode_text.tempSubStringBetween(start_pos, end_pos).toUTF8String(word);
}
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index e76c5d2c..4940f2b9 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -187,6 +187,39 @@ TEST_F(CollectionLocaleTest, SearchAgainstThaiText) {
ASSERT_EQ("<mark>พกไฟ</mark>\nเสมอ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
}
+TEST_F(CollectionLocaleTest, ThaiTextShouldBeNormalizedToNFKC) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "th"),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"น้ำมัน", "Dustin Kensrue"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ auto results = coll1->search("น้ํามัน",{"title"}, "", {}, {},
+ {0}, 10, 1, FREQUENCY).get();
+
+ ASSERT_EQ(1, results["found"].get<size_t>());
+}
+
TEST_F(CollectionLocaleTest, SearchThaiTextPreSegmentedQuery) {
Collection *coll1;
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index 1fa43f78..258a7a17 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -241,7 +241,7 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
ASSERT_EQ(4, tokens.size());
ASSERT_EQ("จิ้งจอก", tokens[0]);
ASSERT_EQ("สี", tokens[1]);
- ASSERT_EQ("น้ำตาล", tokens[2]);
+ ASSERT_EQ("น้ําตาล", tokens[2]);
ASSERT_EQ("ด่วน", tokens[3]);
tokens.clear();