Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Remove general punctuation for th locale.

Kishore Nallanc69ff12d3d3e
diff --git a/include/tokenizer.h b/include/tokenizer.h
index e6726ad4..2b133efa 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -84,6 +84,8 @@ public:
return (c & ~0x7f) == 0;
}
+ static bool belongs_to_general_punctuation_unicode_block(UChar c);
+
void decr_token_counter();
bool should_skip_char(char c);
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 958abcfd..dee76827 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -2,6 +2,7 @@
#include <algorithm>
#include <string_utils.h>
#include "tokenizer.h"
+#include <unicode/uchar.h>
Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const std::string& locale,
const std::vector<char>& symbols_to_index,
@@ -102,6 +103,11 @@ void Tokenizer::init(const std::string& input) {
}
}
+bool Tokenizer::belongs_to_general_punctuation_unicode_block(UChar c) {
+ UBlockCode blockCode = ublock_getCode(c);
+ return blockCode == UBLOCK_GENERAL_PUNCTUATION;
+}
+
bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_index, size_t& end_index) {
if(no_op) {
if(i == text.size()) {
@@ -142,7 +148,14 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
icu::UnicodeString dst;
nfkc->normalize(src, dst, errcode);
if(!U_FAILURE(errcode)) {
- dst.toUTF8String(word);
+ icu::UnicodeString transformedString;
+ for (int32_t t = 0; t < dst.length(); t++) {
+ if (!belongs_to_general_punctuation_unicode_block(dst[t])) {
+ transformedString += dst[t];
+ }
+ }
+
+ transformedString.toUTF8String(word);
} else {
LOG(ERROR) << "Unicode error during parsing: " << errcode;
}
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index c8c55e01..2a5f0ff8 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -345,6 +345,21 @@ TEST(TokenizerTest, ShouldTokenizeLocaleTextWithEnglishText) {
ASSERT_EQ("math", ttokens[8]);
}
+TEST(TokenizerTest, ShouldRemoveGenericPunctuationFromThaiText) {
+ std::string tstr = "f’’b";
+ std::vector<std::string> ttokens;
+ Tokenizer(tstr, true, false, "th").tokenize(ttokens);
+ ASSERT_EQ(2, ttokens.size());
+ ASSERT_EQ("f", ttokens[0]);
+ ASSERT_EQ("b", ttokens[1]);
+
+ ttokens.clear();
+ tstr = "Lay’s";
+ Tokenizer(tstr, true, false, "th").tokenize(ttokens);
+ ASSERT_EQ(1, ttokens.size());
+ ASSERT_EQ("lays", ttokens[0]);
+}
+
TEST(TokenizerTest, ShouldTokenizeLocaleTextWithSwedishText) {
std::string tstr = "södra";
std::vector<std::string> ttokens;