Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch β‘ π β¨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Imporve prefix searching on ko locale.
Kishore Nallan56d3a26cc578
diff --git a/include/string_utils.h b/include/string_utils.hindex 552a2b6b..6aaf399f 100644--- a/include/string_utils.h+++ b/include/string_utils.h@@ -8,16 +8,16 @@#include <random>#include <map>#include "wyhash_v5.h"+#include <unicode/normalizer2.h>struct StringUtils {- StringUtils() {+ // non-deletable singleton+ const icu::Normalizer2* nfkd;- }-- ~StringUtils() {+ StringUtils();- }+ ~StringUtils();// Adapted from: http://stackoverflow.com/a/236180/131050static size_t split(const std::string& s, std::vector<std::string> & result, const std::string& delim,@@ -281,6 +281,8 @@ struct StringUtils {return hash != std::numeric_limits<uint64_t>::max() ? hash : (std::numeric_limits<uint64_t>::max()-1);}+ std::string unicode_nfkd(const std::string& text);+static std::string randstring(size_t length);static std::string hmac(const std::string& key, const std::string& msg);diff --git a/include/tokenizer.h b/include/tokenizer.hindex 9d75e0e3..8bf62803 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -4,6 +4,7 @@#include <vector>#include <iconv.h>#include <unicode/brkiter.h>+#include <unicode/normalizer2.h>#include "japanese_localizer.h"#include "logger.h"@@ -33,6 +34,9 @@ private:int32_t utf8_start_index = 0;char* normalized_text = nullptr;+ // non-deletable singleton+ const icu::Normalizer2* nfkd;+inline size_t get_stream_mode(char c) {return (std::isalnum(c) || index_symbols[uint8_t(c)] == 1) ? INDEX : ((c == ' ' || c == '\n') ? SEPARATE : SKIP@@ -48,35 +52,7 @@ public:explicit Tokenizer(const std::string& input,bool normalize=true, bool no_op=false,const std::string& locale = "",- const std::vector<char>& symbols_to_index = {}):- i(0), normalize(normalize),- no_op(no_op), locale(locale) {-- if(locale == "ja") {- normalized_text = JapaneseLocalizer::get_instance().normalize(input);- text = normalized_text;- } else {- text = input;- }-- cd = iconv_open("ASCII//TRANSLIT", "UTF-8");-- if(!locale.empty() && locale != "en") {- UErrorCode status = U_ZERO_ERROR;- const icu::Locale& icu_locale = icu::Locale(locale.c_str());- bi = icu::BreakIterator::createWordInstance(icu_locale, status);-- unicode_text = icu::UnicodeString::fromUTF8(text);- bi->setText(unicode_text);-- position = bi->first();- prev_position = -1;- }-- for(char c: symbols_to_index) {- index_symbols[uint8_t(c)] = 1;- }- }+ const std::vector<char>& symbols_to_index = {});~Tokenizer() {iconv_close(cd);diff --git a/src/collection.cpp b/src/collection.cppindex 2e443153..186353ef 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1505,9 +1505,14 @@ void Collection::highlight_result(const field &search_field,}}- const std::string& text = (search_field.type == field_types::STRING) ? document[search_field.name] : document[search_field.name][match_index.index];+ std::string text = (search_field.type == field_types::STRING) ? document[search_field.name] :+ document[search_field.name][match_index.index];Tokenizer tokenizer(text, true, false, search_field.locale);+ if(search_field.locale == "ko") {+ text = string_utils.unicode_nfkd(text);+ }+// need an ordered map here to ensure that it is ordered by the key (start offset)std::map<size_t, size_t> token_offsets;diff --git a/src/string_utils.cpp b/src/string_utils.cppindex 617b1b00..e0c3ec16 100644--- a/src/string_utils.cpp+++ b/src/string_utils.cpp@@ -7,6 +7,16 @@#include <map>#include "logger.h"++StringUtils::StringUtils() {+ UErrorCode errcode = U_ZERO_ERROR;+ nfkd = icu::Normalizer2::getNFKDInstance(errcode);+}++StringUtils::~StringUtils() {++}+std::string lower_and_no_special_chars(const std::string & str) {std::stringstream ss;@@ -186,6 +196,22 @@ std::string StringUtils::float_to_str(float value) {return os.str();}+std::string StringUtils::unicode_nfkd(const std::string& text) {+ UErrorCode errcode = U_ZERO_ERROR;+ icu::UnicodeString src = icu::UnicodeString::fromUTF8(text);+ icu::UnicodeString dst;+ nfkd->normalize(src, dst, errcode);++ if(!U_FAILURE(errcode)) {+ std::string output;+ dst.toUTF8String(output);+ return output;+ } else {+ LOG(ERROR) << "Unicode error during parsing: " << errcode;+ return text;+ }+}+/*size_t StringUtils::unicode_length(const std::string& bytes) {std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;return utf8conv.from_bytes(bytes).size();diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 3ac192d0..8b553405 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -2,6 +2,39 @@#include <algorithm>#include "tokenizer.h"+Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const std::string& locale,+ const std::vector<char>& symbols_to_index):+ i(0), normalize(normalize), no_op(no_op), locale(locale) {++ if(locale == "ja") {+ normalized_text = JapaneseLocalizer::get_instance().normalize(input);+ text = normalized_text;+ } else {+ text = input;+ }++ cd = iconv_open("ASCII//TRANSLIT", "UTF-8");++ if(!locale.empty() && locale != "en") {+ UErrorCode status = U_ZERO_ERROR;+ const icu::Locale& icu_locale = icu::Locale(locale.c_str());+ bi = icu::BreakIterator::createWordInstance(icu_locale, status);++ unicode_text = icu::UnicodeString::fromUTF8(text);+ bi->setText(unicode_text);++ position = bi->first();+ prev_position = -1;+ }++ for(char c: symbols_to_index) {+ index_symbols[uint8_t(c)] = 1;+ }++ UErrorCode errcode = U_ZERO_ERROR;+ nfkd = icu::Normalizer2::getNFKDInstance(errcode);+}+bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_index, size_t& end_index) {if(no_op) {if(i == text.size()) {@@ -23,10 +56,25 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_indeif(prev_position != -1) {std::string word;size_t length = position - prev_position;- token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);-//LOG(INFO) << "token: " << token;+ if(locale == "ko") {+ UErrorCode errcode = U_ZERO_ERROR;+ icu::UnicodeString src = unicode_text.tempSubString(prev_position, length);+ icu::UnicodeString dst;+ nfkd->normalize(src, dst, errcode);++ if(!U_FAILURE(errcode)) {+ std::string output;+ dst.toUTF8String(output);+ token = output;+ } else {+ LOG(ERROR) << "Unicode error during parsing: " << errcode;+ }+ } else {+ token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);+ }+if(!token.empty()) {if (!std::isalnum(token[0]) && is_ascii_char(token[i])) {// ignore ascii symbolsdiff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex efb26943..47a68bae 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -131,7 +131,7 @@ TEST_F(CollectionLocaleTest, SearchAgainstThaiTextExactMatch) {TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {Collection *coll1;- std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "th"),+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "ko"),field("artist", field_types::STRING, false),field("points", field_types::INT32, false),};@@ -162,7 +162,7 @@ TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());- ASSERT_EQ("μλ μνμ² λ999<mark>κ·Ήμ₯ν</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("αα ‘α«αα §αΌαα ³α«αα ‘αα ₯α―αα ©999<mark>αα ³α¨αα ‘αΌαα ‘α«</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());results = coll1->search("μ°μ ",{"title", "artist"}, "", {}, {}, {0}, 10, 1, FREQUENCY).get();@@ -170,7 +170,108 @@ TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {ASSERT_EQ(1, results["found"].get<size_t>());ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());- ASSERT_EQ("κ²½μΉμ§Β·<mark>μ°μ </mark>Β·ν곑", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("αα §αΌαα ³αΌαα ΅Β·<mark>αα ‘α«αα ‘α¨</mark>Β·αα §αΈαα ©α¨", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+}++TEST_F(CollectionLocaleTest, KoreanTextPrefixConsonant) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "ko"),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"μμΈνΉλ³μ μ±λΆκ΅¬", "Wrong Result"},+ {"μμΈνΉλ³μ μ€κ΅¬ μ΄λ", "Wrong Result"},+ {"μμΈνΉλ³μ κ΄μ ꡬ", "Expected Result"},+ {"μμΈνΉλ³μ μ©μ°κ΅¬ μ©μ°λ", "Wrong Result"},+ {"μμΈνΉλ³μ λλ문ꡬ μ΄λ¬Έλ", "Wrong Result"},+ {"μμΈνΉλ³μ μλ문ꡬ νμ λ", "Wrong Result"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ std::vector<sort_by> sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("points", "DESC") };++ // To ensure that NFKD works, we will test for both ᄀ (Hangul Choseong Kiyeok)+ auto results = coll1->search("μμΈνΉλ³μ α",+ {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, true).get();++ ASSERT_EQ(6, results["found"].get<size_t>());+ ASSERT_EQ(6, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());++ // and ㄱ (Hangul Letter Kiyeok)+ results = coll1->search("μμΈνΉλ³μ γ±",+ {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, true).get();++ ASSERT_EQ(6, results["found"].get<size_t>());+ ASSERT_EQ(6, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());++ // search for full word+ results = coll1->search("μμΈνΉλ³μ κ΄",+ {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, true).get();++ ASSERT_EQ(6, results["found"].get<size_t>());+ ASSERT_EQ(6, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());+}++TEST_F(CollectionLocaleTest, KoreanTextPrefixVowel) {+ Collection *coll1;++ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "ko"),+ field("artist", field_types::STRING, false),+ field("points", field_types::INT32, false),};++ coll1 = collectionManager.get_collection("coll1").get();+ if(coll1 == nullptr) {+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();+ }++ std::vector<std::vector<std::string>> records = {+ {"μμΈνΉλ³μ κ°μꡬ 곡νλ", "Wrong Result"},+ {"μμΈνΉλ³μ κ΄μ ꡬ", "Wrong Result"},+ {"μμΈνΉλ³μ κ°λꡬ κ³ λλ", "Expected Result"},+ {"μμΈνΉλ³μ κ΄μ ꡬ κ΄μ μ°λλ€κΈΈ", "Wrong Result"},+ {"μμΈνΉλ³μ κ΄μ ꡬ κ΄μ λ‘", "Wrong Result"},+ {"μμΈνΉλ³μ κ΄μ ꡬ κ³Όμ²λλ‘", "Wrong Result"},+ };++ for(size_t i=0; i<records.size(); i++) {+ nlohmann::json doc;++ doc["id"] = std::to_string(i);+ doc["title"] = records[i][0];+ doc["artist"] = records[i][1];+ doc["points"] = i;++ ASSERT_TRUE(coll1->add(doc.dump()).ok());+ }++ std::vector<sort_by> sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("points", "DESC") };++ auto results = coll1->search("μμΈνΉλ³μ κ³ ",+ {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, true).get();++ ASSERT_EQ(6, results["found"].get<size_t>());+ ASSERT_EQ(6, results["hits"].size());+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());}TEST_F(CollectionLocaleTest, SearchAgainstKoreanTextContainingEnglishChars) {diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cppindex bc94d793..e53811ba 100644--- a/test/tokenizer_test.cpp+++ b/test/tokenizer_test.cpp@@ -193,16 +193,16 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {tokens.clear();Tokenizer("κ²½μΉμ§Β·μ°μ Β·ν곑", false, false, "ko").tokenize(tokens);ASSERT_EQ(3, tokens.size());- ASSERT_EQ("κ²½μΉμ§", tokens[0]);- ASSERT_EQ("μ°μ ", tokens[1]);- ASSERT_EQ("ν곑", tokens[2]);+ ASSERT_EQ("αα §αΌαα ³αΌαα ΅", tokens[0]);+ ASSERT_EQ("αα ‘α«αα ‘α¨", tokens[1]);+ ASSERT_EQ("αα §αΈαα ©α¨", tokens[2]);tokens.clear();Tokenizer("μλ μνμ² λ999κ·Ήμ₯ν", false, false, "ko").tokenize(tokens);ASSERT_EQ(3, tokens.size());- ASSERT_EQ("μλ μνμ² λ", tokens[0]);+ ASSERT_EQ("αα ‘α«αα §αΌαα ³α«αα ‘αα ₯α―αα ©", tokens[0]);ASSERT_EQ("999", tokens[1]);- ASSERT_EQ("κ·Ήμ₯ν", tokens[2]);+ ASSERT_EQ("αα ³α¨αα ‘αΌαα ‘α«", tokens[2]);// japanesetokens.clear();