Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Remove general punctuation for th locale.
Kishore Nallanc69ff12d3d3e
diff --git a/include/tokenizer.h b/include/tokenizer.hindex e6726ad4..2b133efa 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -84,6 +84,8 @@ public:return (c & ~0x7f) == 0;}+ static bool belongs_to_general_punctuation_unicode_block(UChar c);+void decr_token_counter();bool should_skip_char(char c);diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 958abcfd..dee76827 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -2,6 +2,7 @@#include <algorithm>#include <string_utils.h>#include "tokenizer.h"+#include <unicode/uchar.h>Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const std::string& locale,const std::vector<char>& symbols_to_index,@@ -102,6 +103,11 @@ void Tokenizer::init(const std::string& input) {}}+bool Tokenizer::belongs_to_general_punctuation_unicode_block(UChar c) {+ UBlockCode blockCode = ublock_getCode(c);+ return blockCode == UBLOCK_GENERAL_PUNCTUATION;+}+bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_index, size_t& end_index) {if(no_op) {if(i == text.size()) {@@ -142,7 +148,14 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_indeicu::UnicodeString dst;nfkc->normalize(src, dst, errcode);if(!U_FAILURE(errcode)) {- dst.toUTF8String(word);+ icu::UnicodeString transformedString;+ for (int32_t t = 0; t < dst.length(); t++) {+ if (!belongs_to_general_punctuation_unicode_block(dst[t])) {+ transformedString += dst[t];+ }+ }++ transformedString.toUTF8String(word);} else {LOG(ERROR) << "Unicode error during parsing: " << errcode;}diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex c8c55e01..2a5f0ff8 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -345,6 +345,21 @@ TEST(TokenizerTest, ShouldTokenizeLocaleTextWithEnglishText) {ASSERT_EQ("math", ttokens[8]);}+TEST(TokenizerTest, ShouldRemoveGenericPunctuationFromThaiText) {+ std::string tstr = "f’’b";+ std::vector<std::string> ttokens;+ Tokenizer(tstr, true, false, "th").tokenize(ttokens);+ ASSERT_EQ(2, ttokens.size());+ ASSERT_EQ("f", ttokens[0]);+ ASSERT_EQ("b", ttokens[1]);++ ttokens.clear();+ tstr = "Lay’s";+ Tokenizer(tstr, true, false, "th").tokenize(ttokens);+ ASSERT_EQ(1, ttokens.size());+ ASSERT_EQ("lays", ttokens[0]);+}+TEST(TokenizerTest, ShouldTokenizeLocaleTextWithSwedishText) {std::string tstr = "södra";std::vector<std::string> ttokens;