Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚑ πŸ” ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Imporve prefix searching on ko locale.

Kishore Nallan56d3a26cc578
diff --git a/include/string_utils.h b/include/string_utils.h
index 552a2b6b..6aaf399f 100644
--- a/include/string_utils.h
+++ b/include/string_utils.h
@@ -8,16 +8,16 @@
#include <random>
#include <map>
#include "wyhash_v5.h"
+#include <unicode/normalizer2.h>
struct StringUtils {
- StringUtils() {
+ // non-deletable singleton
+ const icu::Normalizer2* nfkd;
- }
-
- ~StringUtils() {
+ StringUtils();
- }
+ ~StringUtils();
// Adapted from: http://stackoverflow.com/a/236180/131050
static size_t split(const std::string& s, std::vector<std::string> & result, const std::string& delim,
@@ -281,6 +281,8 @@ struct StringUtils {
return hash != std::numeric_limits<uint64_t>::max() ? hash : (std::numeric_limits<uint64_t>::max()-1);
}
+ std::string unicode_nfkd(const std::string& text);
+
static std::string randstring(size_t length);
static std::string hmac(const std::string& key, const std::string& msg);
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 9d75e0e3..8bf62803 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -4,6 +4,7 @@
#include <vector>
#include <iconv.h>
#include <unicode/brkiter.h>
+#include <unicode/normalizer2.h>
#include "japanese_localizer.h"
#include "logger.h"
@@ -33,6 +34,9 @@ private:
int32_t utf8_start_index = 0;
char* normalized_text = nullptr;
+ // non-deletable singleton
+ const icu::Normalizer2* nfkd;
+
inline size_t get_stream_mode(char c) {
return (std::isalnum(c) || index_symbols[uint8_t(c)] == 1) ? INDEX : (
(c == ' ' || c == '\n') ? SEPARATE : SKIP
@@ -48,35 +52,7 @@ public:
explicit Tokenizer(const std::string& input,
bool normalize=true, bool no_op=false,
const std::string& locale = "",
- const std::vector<char>& symbols_to_index = {}):
- i(0), normalize(normalize),
- no_op(no_op), locale(locale) {
-
- if(locale == "ja") {
- normalized_text = JapaneseLocalizer::get_instance().normalize(input);
- text = normalized_text;
- } else {
- text = input;
- }
-
- cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
-
- if(!locale.empty() && locale != "en") {
- UErrorCode status = U_ZERO_ERROR;
- const icu::Locale& icu_locale = icu::Locale(locale.c_str());
- bi = icu::BreakIterator::createWordInstance(icu_locale, status);
-
- unicode_text = icu::UnicodeString::fromUTF8(text);
- bi->setText(unicode_text);
-
- position = bi->first();
- prev_position = -1;
- }
-
- for(char c: symbols_to_index) {
- index_symbols[uint8_t(c)] = 1;
- }
- }
+ const std::vector<char>& symbols_to_index = {});
~Tokenizer() {
iconv_close(cd);
diff --git a/src/collection.cpp b/src/collection.cpp
index 2e443153..186353ef 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1505,9 +1505,14 @@ void Collection::highlight_result(const field &search_field,
}
}
- const std::string& text = (search_field.type == field_types::STRING) ? document[search_field.name] : document[search_field.name][match_index.index];
+ std::string text = (search_field.type == field_types::STRING) ? document[search_field.name] :
+ document[search_field.name][match_index.index];
Tokenizer tokenizer(text, true, false, search_field.locale);
+ if(search_field.locale == "ko") {
+ text = string_utils.unicode_nfkd(text);
+ }
+
// need an ordered map here to ensure that it is ordered by the key (start offset)
std::map<size_t, size_t> token_offsets;
diff --git a/src/string_utils.cpp b/src/string_utils.cpp
index 617b1b00..e0c3ec16 100644
--- a/src/string_utils.cpp
+++ b/src/string_utils.cpp
@@ -7,6 +7,16 @@
#include <map>
#include "logger.h"
+
+StringUtils::StringUtils() {
+ UErrorCode errcode = U_ZERO_ERROR;
+ nfkd = icu::Normalizer2::getNFKDInstance(errcode);
+}
+
+StringUtils::~StringUtils() {
+
+}
+
std::string lower_and_no_special_chars(const std::string & str) {
std::stringstream ss;
@@ -186,6 +196,22 @@ std::string StringUtils::float_to_str(float value) {
return os.str();
}
+std::string StringUtils::unicode_nfkd(const std::string& text) {
+ UErrorCode errcode = U_ZERO_ERROR;
+ icu::UnicodeString src = icu::UnicodeString::fromUTF8(text);
+ icu::UnicodeString dst;
+ nfkd->normalize(src, dst, errcode);
+
+ if(!U_FAILURE(errcode)) {
+ std::string output;
+ dst.toUTF8String(output);
+ return output;
+ } else {
+ LOG(ERROR) << "Unicode error during parsing: " << errcode;
+ return text;
+ }
+}
+
/*size_t StringUtils::unicode_length(const std::string& bytes) {
std::wstring_convert<std::codecvt_utf8<char32_t>, char32_t> utf8conv;
return utf8conv.from_bytes(bytes).size();
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 3ac192d0..8b553405 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -2,6 +2,39 @@
#include <algorithm>
#include "tokenizer.h"
+Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const std::string& locale,
+ const std::vector<char>& symbols_to_index):
+ i(0), normalize(normalize), no_op(no_op), locale(locale) {
+
+ if(locale == "ja") {
+ normalized_text = JapaneseLocalizer::get_instance().normalize(input);
+ text = normalized_text;
+ } else {
+ text = input;
+ }
+
+ cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
+
+ if(!locale.empty() && locale != "en") {
+ UErrorCode status = U_ZERO_ERROR;
+ const icu::Locale& icu_locale = icu::Locale(locale.c_str());
+ bi = icu::BreakIterator::createWordInstance(icu_locale, status);
+
+ unicode_text = icu::UnicodeString::fromUTF8(text);
+ bi->setText(unicode_text);
+
+ position = bi->first();
+ prev_position = -1;
+ }
+
+ for(char c: symbols_to_index) {
+ index_symbols[uint8_t(c)] = 1;
+ }
+
+ UErrorCode errcode = U_ZERO_ERROR;
+ nfkd = icu::Normalizer2::getNFKDInstance(errcode);
+}
+
bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_index, size_t& end_index) {
if(no_op) {
if(i == text.size()) {
@@ -23,10 +56,25 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
if(prev_position != -1) {
std::string word;
size_t length = position - prev_position;
- token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);
-
//LOG(INFO) << "token: " << token;
+ if(locale == "ko") {
+ UErrorCode errcode = U_ZERO_ERROR;
+ icu::UnicodeString src = unicode_text.tempSubString(prev_position, length);
+ icu::UnicodeString dst;
+ nfkd->normalize(src, dst, errcode);
+
+ if(!U_FAILURE(errcode)) {
+ std::string output;
+ dst.toUTF8String(output);
+ token = output;
+ } else {
+ LOG(ERROR) << "Unicode error during parsing: " << errcode;
+ }
+ } else {
+ token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);
+ }
+
if(!token.empty()) {
if (!std::isalnum(token[0]) && is_ascii_char(token[i])) {
// ignore ascii symbols
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index efb26943..47a68bae 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -131,7 +131,7 @@ TEST_F(CollectionLocaleTest, SearchAgainstThaiTextExactMatch) {
TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {
Collection *coll1;
- std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "th"),
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "ko"),
field("artist", field_types::STRING, false),
field("points", field_types::INT32, false),};
@@ -162,7 +162,7 @@ TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
- ASSERT_EQ("μ•ˆλ…•μ€ν•˜μ² λ„999<mark>κ·Ήμž₯판</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("α„‹α…‘α†«α„‚α…§α†Όα„‹α…³α†«α„’α…‘α„Žα…₯ᆯ도999<mark>α„€α…³α†¨α„Œα…‘α†Όα„‘α…‘α†«</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
results = coll1->search("μ‚°μ•…",
{"title", "artist"}, "", {}, {}, {0}, 10, 1, FREQUENCY).get();
@@ -170,7 +170,108 @@ TEST_F(CollectionLocaleTest, SearchAgainstKoreanText) {
ASSERT_EQ(1, results["found"].get<size_t>());
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
- ASSERT_EQ("κ²½μŠΉμ§€Β·<mark>μ‚°μ•…</mark>Β·ν˜‘κ³‘", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("α„€α…§α†Όα„‰α…³α†Όα„Œα…΅Β·<mark>ᄉᅑᆫᄋᅑᆨ</mark>·협곡", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+}
+
+TEST_F(CollectionLocaleTest, KoreanTextPrefixConsonant) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "ko"),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"μ„œμšΈνŠΉλ³„μ‹œ 성뢁ꡬ", "Wrong Result"},
+ {"μ„œμšΈνŠΉλ³„μ‹œ 쀑ꡬ μ΄ˆλ™", "Wrong Result"},
+ {"μ„œμšΈνŠΉλ³„μ‹œ 관악ꡬ", "Expected Result"},
+ {"μ„œμšΈνŠΉλ³„μ‹œ μš©μ‚°κ΅¬ μš©μ‚°λ™", "Wrong Result"},
+ {"μ„œμšΈνŠΉλ³„μ‹œ λ™λŒ€λ¬Έκ΅¬ 이문동", "Wrong Result"},
+ {"μ„œμšΈνŠΉλ³„μ‹œ μ„œλŒ€λ¬Έκ΅¬ ν˜„μ €λ™", "Wrong Result"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ std::vector<sort_by> sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("points", "DESC") };
+
+ // To ensure that NFKD works, we will test for both &#4352; (Hangul Choseong Kiyeok)
+ auto results = coll1->search("μ„œμšΈνŠΉλ³„μ‹œ α„€",
+ {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, true).get();
+
+ ASSERT_EQ(6, results["found"].get<size_t>());
+ ASSERT_EQ(6, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // and &#12593; (Hangul Letter Kiyeok)
+ results = coll1->search("μ„œμšΈνŠΉλ³„μ‹œ γ„±",
+ {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, true).get();
+
+ ASSERT_EQ(6, results["found"].get<size_t>());
+ ASSERT_EQ(6, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+
+ // search for full word
+ results = coll1->search("μ„œμšΈνŠΉλ³„μ‹œ κ΄€",
+ {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, true).get();
+
+ ASSERT_EQ(6, results["found"].get<size_t>());
+ ASSERT_EQ(6, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
+}
+
+TEST_F(CollectionLocaleTest, KoreanTextPrefixVowel) {
+ Collection *coll1;
+
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, DEFAULT_GEO_RESOLUTION, "ko"),
+ field("artist", field_types::STRING, false),
+ field("points", field_types::INT32, false),};
+
+ coll1 = collectionManager.get_collection("coll1").get();
+ if(coll1 == nullptr) {
+ coll1 = collectionManager.create_collection("coll1", 1, fields, "points").get();
+ }
+
+ std::vector<std::vector<std::string>> records = {
+ {"μ„œμšΈνŠΉλ³„μ‹œ κ°•μ„œκ΅¬ 곡항동", "Wrong Result"},
+ {"μ„œμšΈνŠΉλ³„μ‹œ 관악ꡬ", "Wrong Result"},
+ {"μ„œμšΈνŠΉλ³„μ‹œ 강동ꡬ 고덕동", "Expected Result"},
+ {"μ„œμšΈνŠΉλ³„μ‹œ 관악ꡬ κ΄€μ•…μ‚°λ‚˜λ“€κΈΈ", "Wrong Result"},
+ {"μ„œμšΈνŠΉλ³„μ‹œ 관악ꡬ κ΄€μ•…λ‘œ", "Wrong Result"},
+ {"μ„œμšΈνŠΉλ³„μ‹œ 관악ꡬ κ³Όμ²œλŒ€λ‘œ", "Wrong Result"},
+ };
+
+ for(size_t i=0; i<records.size(); i++) {
+ nlohmann::json doc;
+
+ doc["id"] = std::to_string(i);
+ doc["title"] = records[i][0];
+ doc["artist"] = records[i][1];
+ doc["points"] = i;
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+ }
+
+ std::vector<sort_by> sort_fields = { sort_by(sort_field_const::text_match, "DESC"), sort_by("points", "DESC") };
+
+ auto results = coll1->search("μ„œμšΈνŠΉλ³„μ‹œ κ³ ",
+ {"title"}, "", {}, sort_fields, {0}, 10, 1, FREQUENCY, true).get();
+
+ ASSERT_EQ(6, results["found"].get<size_t>());
+ ASSERT_EQ(6, results["hits"].size());
+ ASSERT_EQ("2", results["hits"][0]["document"]["id"].get<std::string>());
}
TEST_F(CollectionLocaleTest, SearchAgainstKoreanTextContainingEnglishChars) {
diff --git a/test/tokenizer_test.cpp b/test/tokenizer_test.cpp
index bc94d793..e53811ba 100644
--- a/test/tokenizer_test.cpp
+++ b/test/tokenizer_test.cpp
@@ -193,16 +193,16 @@ TEST(TokenizerTest, ShouldTokenizeLocaleText) {
tokens.clear();
Tokenizer("κ²½μŠΉμ§€Β·μ‚°μ•…Β·ν˜‘κ³‘", false, false, "ko").tokenize(tokens);
ASSERT_EQ(3, tokens.size());
- ASSERT_EQ("κ²½μŠΉμ§€", tokens[0]);
- ASSERT_EQ("μ‚°μ•…", tokens[1]);
- ASSERT_EQ("ν˜‘κ³‘", tokens[2]);
+ ASSERT_EQ("α„€α…§α†Όα„‰α…³α†Όα„Œα…΅", tokens[0]);
+ ASSERT_EQ("ᄉᅑᆫᄋᅑᆨ", tokens[1]);
+ ASSERT_EQ("협곡", tokens[2]);
tokens.clear();
Tokenizer("μ•ˆλ…•μ€ν•˜μ² λ„999κ·Ήμž₯판", false, false, "ko").tokenize(tokens);
ASSERT_EQ(3, tokens.size());
- ASSERT_EQ("μ•ˆλ…•μ€ν•˜μ² λ„", tokens[0]);
+ ASSERT_EQ("α„‹α…‘α†«α„‚α…§α†Όα„‹α…³α†«α„’α…‘α„Žα…₯ᆯ도", tokens[0]);
ASSERT_EQ("999", tokens[1]);
- ASSERT_EQ("κ·Ήμž₯판", tokens[2]);
+ ASSERT_EQ("α„€α…³α†¨α„Œα…‘α†Όα„‘α…‘α†«", tokens[2]);
// japanese
tokens.clear();