Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Improve Cyrillic support.
Kishore Nallanbd6fd1c03e11
diff --git a/include/field.h b/include/field.hindex 690148f9..0ad27629 100644--- a/include/field.h+++ b/include/field.h@@ -305,9 +305,8 @@ struct field {field_json[fields::name].get<std::string>() + std::string("` should be a string."));}- if(!field_json[fields::locale].get<std::string>().empty() && field_json[fields::locale] != "en" &&- field_json[fields::locale] != "ja" && field_json[fields::locale] != "ko" &&- field_json[fields::locale] != "zh" && field_json[fields::locale] != "th") {+ if(!field_json[fields::locale].get<std::string>().empty() &&+ field_json[fields::locale].get<std::string>().size() != 2) {return Option<bool>(400, std::string("The `locale` value of the field `") +field_json[fields::name].get<std::string>() + std::string("` is not valid."));}diff --git a/include/tokenizer.h b/include/tokenizer.hindex ba9e101f..0e4ff41f 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -73,4 +73,6 @@ public:void tokenize(std::vector<std::string>& tokens);void tokenize(std::string& token);++ static bool is_cyrillic(const std::string& locale);};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex 8e306221..d350f3f2 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1651,7 +1651,12 @@ void Collection::highlight_result(const field &search_field,text = document[search_field.name][match_index.index];}- Tokenizer tokenizer(text, true, false, search_field.locale, symbols_to_index, token_separators);+ bool is_cyrillic = Tokenizer::is_cyrillic(search_field.locale);+ bool normalise = is_cyrillic ? false : true;+ Tokenizer tokenizer(text, normalise, false, search_field.locale, symbols_to_index, token_separators);++ // word tokenizer is a secondary tokenizer used for specific languages that requires transliteration+ Tokenizer word_tokenizer("", true, false, search_field.locale, symbols_to_index, token_separators);if(search_field.locale == "ko") {text = string_utils.unicode_nfkd(text);@@ -1676,6 +1681,10 @@ void Collection::highlight_result(const field &search_field,bool found_first_match = false;while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {+ if(is_cyrillic) {+ word_tokenizer.tokenize(raw_token);+ }+if(!found_first_match) {if(snippet_start_window.size() == highlight_affix_num_tokens + 1) {snippet_start_window.pop_front();diff --git a/src/index.cpp b/src/index.cppindex 555b37f6..6a1f2f76 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2840,8 +2840,8 @@ void Index::search_field(const uint8_t & field_id,size_t max_cost = (num_typos < 0 || num_typos > 2) ? 2 : num_typos;- if(the_field.locale != "" && the_field.locale != "en") {- // disable fuzzy trie traversal for non-english locales+ if(the_field.locale != "" && the_field.locale != "en" && !Tokenizer::is_cyrillic(the_field.locale)) {+ // disable fuzzy trie traversal for certain non-english localesmax_cost = 0;}diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 745e47e8..6be4def4 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -29,6 +29,12 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, constelse if(locale == "ja") {normalized_text = JapaneseLocalizer::get_instance().normalize(input);text = normalized_text;+ } else if(is_cyrillic(locale)) {+ // init transliterator but will only transliterate during tokenization+ UErrorCode translit_status = U_ZERO_ERROR;+ transliterator = icu::Transliterator::createInstance("Any-Latin; Latin-ASCII",+ UTRANS_FORWARD, translit_status);+ text = input;} else {text = input;}@@ -89,14 +95,16 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_indenfkd->normalize(src, dst, errcode);if(!U_FAILURE(errcode)) {- std::string output;- dst.toUTF8String(output);- token = output;+ token = dst.toUTF8String(word);} else {LOG(ERROR) << "Unicode error during parsing: " << errcode;}+ } else if(normalize && is_cyrillic(locale)) {+ auto raw_text = unicode_text.tempSubString(prev_position, length);+ transliterator->transliterate(raw_text);+ token = raw_text.toUTF8String(word);} else {- token = unicode_text.toLower().tempSubString(prev_position, length).toUTF8String(word);+ token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);}if(!token.empty()) {@@ -197,7 +205,7 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde//printf("[%s]\n", inbuf);errno = 0;- iconv(cd, &inptr, &insize, &outptr, &outsize);+ iconv(cd, &inptr, &insize, &outptr, &outsize); // this can be handled by ICU via "Latin-ASCII"if(errno == EILSEQ) {// symbol cannot be represented as ASCII, so write the original symbol@@ -252,3 +260,8 @@ bool Tokenizer::next(std::string &token, size_t &token_index) {size_t start_index, end_index;return next(token, token_index, start_index, end_index);}++bool Tokenizer::is_cyrillic(const std::string& locale) {+ return locale == "el" ||+ locale == "ru" || locale == "sr" || locale == "uk" || locale == "be";+}diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex f54759bc..31bbf2d8 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -470,3 +470,59 @@ TEST_F(CollectionLocaleTest, SearchAgainstKoreanTextContainingEnglishChars) {ASSERT_EQ("0", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("개혁 등의 영향으로 <mark>11</mark>%나 위축됐다", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());}+++TEST_F(CollectionLocaleTest, SearchCyrillicText) {+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "sr"),};+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc;+ doc["title"] = "Test Тест";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ doc["title"] = "TEST ТЕСТ";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("тест", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ ASSERT_EQ("<mark>TEST</mark> ТЕСТ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("<mark>Test</mark> Тест", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());++ // with typo++ results = coll1->search("тетст", {"title"}, "", {}, {}, {1}, 10, 1, FREQUENCY, {false}).get();++ ASSERT_EQ(2, results["hits"].size());+ ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());+ ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());++ ASSERT_EQ("<mark>TEST</mark> ТЕСТ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("<mark>Test</mark> Тест", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());++ collectionManager.drop_collection("coll1");+}++TEST_F(CollectionLocaleTest, SearchGreekText) {+ std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "el"),};+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc;+ doc["title"] = "Εμφάνιση κάθε μέρα.";+ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("Εμφάν", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("<mark>Εμφάνιση</mark> κάθε μέρα.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ // with typo++ results = coll1->search("Εμφάιση", {"title"}, "", {}, {}, {1}, 10, 1, FREQUENCY, {false}).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("<mark>Εμφάνιση</mark> κάθε μέρα.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ collectionManager.drop_collection("coll1");+}diff --git a/test/collection_specific_test.cpp b/test/collection_specific_test.cppindex 1758a540..0e3eabd3 100644--- a/test/collection_specific_test.cpp+++ b/test/collection_specific_test.cpp@@ -1542,29 +1542,6 @@ TEST_F(CollectionSpecificTest, UpdateOfTwoDocsWithSameIdWithinSameBatch) {collectionManager.drop_collection("coll1");}-TEST_F(CollectionSpecificTest, CyrillicText) {- // when the first document containing a token already cannot fit compact posting list-- std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "sr"),};-- Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();-- nlohmann::json doc;- doc["title"] = "Test Тест";- ASSERT_TRUE(coll1->add(doc.dump()).ok());-- doc["title"] = "TEST ТЕСТ";- ASSERT_TRUE(coll1->add(doc.dump()).ok());-- auto results = coll1->search("тест", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {false}).get();-- ASSERT_EQ(2, results["hits"].size());- ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());- ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());-- collectionManager.drop_collection("coll1");-}-TEST_F(CollectionSpecificTest, UpsertOfTwoDocsWithSameIdWithinSameBatch) {std::vector<field> fields = {field("last_chance", field_types::BOOL, false, true),field("points", field_types::INT32, false, true),};