Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Improve Cyrillic support.

Kishore Nallanbd6fd1c03e11
diff --git a/include/field.h b/include/field.h
index 690148f9..0ad27629 100644
--- a/include/field.h
+++ b/include/field.h
@@ -305,9 +305,8 @@ struct field {
field_json[fields::name].get<std::string>() + std::string("` should be a string."));
}
- if(!field_json[fields::locale].get<std::string>().empty() && field_json[fields::locale] != "en" &&
- field_json[fields::locale] != "ja" && field_json[fields::locale] != "ko" &&
- field_json[fields::locale] != "zh" && field_json[fields::locale] != "th") {
+ if(!field_json[fields::locale].get<std::string>().empty() &&
+ field_json[fields::locale].get<std::string>().size() != 2) {
return Option<bool>(400, std::string("The `locale` value of the field `") +
field_json[fields::name].get<std::string>() + std::string("` is not valid."));
}
diff --git a/include/tokenizer.h b/include/tokenizer.h
index ba9e101f..0e4ff41f 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -73,4 +73,6 @@ public:
void tokenize(std::vector<std::string>& tokens);
void tokenize(std::string& token);
+
+ static bool is_cyrillic(const std::string& locale);
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index 8e306221..d350f3f2 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1651,7 +1651,12 @@ void Collection::highlight_result(const field &search_field,
text = document[search_field.name][match_index.index];
}
- Tokenizer tokenizer(text, true, false, search_field.locale, symbols_to_index, token_separators);
+ bool is_cyrillic = Tokenizer::is_cyrillic(search_field.locale);
+ bool normalise = is_cyrillic ? false : true;
+ Tokenizer tokenizer(text, normalise, false, search_field.locale, symbols_to_index, token_separators);
+
+ // word tokenizer is a secondary tokenizer used for specific languages that requires transliteration
+ Tokenizer word_tokenizer("", true, false, search_field.locale, symbols_to_index, token_separators);
if(search_field.locale == "ko") {
text = string_utils.unicode_nfkd(text);
@@ -1676,6 +1681,10 @@ void Collection::highlight_result(const field &search_field,
bool found_first_match = false;
while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {
+ if(is_cyrillic) {
+ word_tokenizer.tokenize(raw_token);
+ }
+
if(!found_first_match) {
if(snippet_start_window.size() == highlight_affix_num_tokens + 1) {
snippet_start_window.pop_front();
diff --git a/src/index.cpp b/src/index.cpp
index 555b37f6..6a1f2f76 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2840,8 +2840,8 @@ void Index::search_field(const uint8_t & field_id,
size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;
- if(the_field.locale != "" && the_field.locale != "en") {
- // disable fuzzy trie traversal for non-english locales
+ if(the_field.locale != "" && the_field.locale != "en" && !Tokenizer::is_cyrillic(the_field.locale)) {
+ // disable fuzzy trie traversal for certain non-english locales
max_cost = 0;
}
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 745e47e8..6be4def4 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -29,6 +29,12 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const
else if(locale == "ja") {
normalized_text = JapaneseLocalizer::get_instance().normalize(input);
text = normalized_text;
+ } else if(is_cyrillic(locale)) {
+ // init transliterator but will only transliterate during tokenization
+ UErrorCode translit_status = U_ZERO_ERROR;
+ transliterator = icu::Transliterator::createInstance("Any-Latin; Latin-ASCII",
+ UTRANS_FORWARD, translit_status);
+ text = input;
} else {
text = input;
}
@@ -89,14 +95,16 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
nfkd->normalize(src, dst, errcode);
if(!U_FAILURE(errcode)) {
- std::string output;
- dst.toUTF8String(output);
- token = output;
+ token = dst.toUTF8String(word);
} else {
LOG(ERROR) << "Unicode error during parsing: " << errcode;
}
+ } else if(normalize && is_cyrillic(locale)) {
+ auto raw_text = unicode_text.tempSubString(prev_position, length);
+ transliterator->transliterate(raw_text);
+ token = raw_text.toUTF8String(word);
} else {
- token = unicode_text.toLower().tempSubString(prev_position, length).toUTF8String(word);
+ token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);
}
if(!token.empty()) {
@@ -197,7 +205,7 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
//printf("[%s]\n", inbuf);
errno = 0;
- iconv(cd, &inptr, &insize, &outptr, &outsize);
+ iconv(cd, &inptr, &insize, &outptr, &outsize); // this can be handled by ICU via "Latin-ASCII"
if(errno == EILSEQ) {
// symbol cannot be represented as ASCII, so write the original symbol
@@ -252,3 +260,8 @@ bool Tokenizer::next(std::string &token, size_t &token_index) {
size_t start_index, end_index;
return next(token, token_index, start_index, end_index);
}
+
+bool Tokenizer::is_cyrillic(const std::string& locale) {
+ return locale == "el" ||
+ locale == "ru" || locale == "sr" || locale == "uk" || locale == "be";
+}
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index f54759bc..31bbf2d8 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -470,3 +470,59 @@ TEST_F(CollectionLocaleTest, SearchAgainstKoreanTextContainingEnglishChars) {
ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("개혁 등의 영향으로 <mark>11</mark>%나 위축됐다", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
}
+
+
+TEST_F(CollectionLocaleTest, SearchCyrillicText) {
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "sr"),};
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc;
+ doc["title"] = "Test Тест";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ doc["title"] = "TEST ТЕСТ";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("тест", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ("<mark>TEST</mark> ТЕСТ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("<mark>Test</mark> Тест", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+
+ // with typo
+
+ results = coll1->search("тетст", {"title"}, "", {}, {}, {1}, 10, 1, FREQUENCY, {false}).get();
+
+ ASSERT_EQ(2, results["hits"].size());
+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
+
+ ASSERT_EQ("<mark>TEST</mark> ТЕСТ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("<mark>Test</mark> Тест", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
+
+TEST_F(CollectionLocaleTest, SearchGreekText) {
+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "el"),};
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc;
+ doc["title"] = "Εμφάνιση κάθε μέρα.";
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("Εμφάν", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("<mark>Εμφάνιση</mark> κάθε μέρα.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ // with typo
+
+ results = coll1->search("Εμφάιση", {"title"}, "", {}, {}, {1}, 10, 1, FREQUENCY, {false}).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("<mark>Εμφάνιση</mark> κάθε μέρα.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}
diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cpp
index 1758a540..0e3eabd3 100644
--- a/test/collection_specific_test.cpp
+++ b/test/collection_specific_test.cpp
@@ -1542,29 +1542,6 @@ TEST_F(CollectionSpecificTest, UpdateOfTwoDocsWithSameIdWithinSameBatch) {
collectionManager.drop_collection("coll1");
}
-TEST_F(CollectionSpecificTest, CyrillicText) {
- // when the first document containing a token already cannot fit compact posting list
-
- std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "sr"),};
-
- Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
-
- nlohmann::json doc;
- doc["title"] = "Test Тест";
- ASSERT_TRUE(coll1->add(doc.dump()).ok());
-
- doc["title"] = "TEST ТЕСТ";
- ASSERT_TRUE(coll1->add(doc.dump()).ok());
-
- auto results = coll1->search("тест", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();
-
- ASSERT_EQ(2, results["hits"].size());
- ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
- ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
-
- collectionManager.drop_collection("coll1");
-}
-
TEST_F(CollectionSpecificTest, UpsertOfTwoDocsWithSameIdWithinSameBatch) {
std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),
field("points", field_types::INT32, false, true),};