Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix facet highlighting for Cyrillic text.
Kishore Nallana9c15ec0f755
diff --git a/include/tokenizer.h b/include/tokenizer.hindex 0e4ff41f..8a767f21 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -31,8 +31,8 @@ private:std::string locale;icu::BreakIterator* bi = nullptr;icu::UnicodeString unicode_text;- int32_t position = 0;- int32_t prev_position = -1;+ int32_t start_pos = 0;+ int32_t end_pos = 0;int32_t utf8_start_index = 0;char* normalized_text = nullptr;@@ -66,6 +66,8 @@ public:delete transliterator;}+ void init(const std::string& input);+bool next(std::string& token, size_t& token_index, size_t& start_index, size_t& end_index);bool next(std::string& token, size_t& token_index);diff --git a/src/collection.cpp b/src/collection.cppindex d350f3f2..ee67b1ed 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1000,6 +1000,31 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const soverride_kv_index++;}+ std::string facet_query_last_token;+ size_t facet_query_num_tokens = 0; // used to identify drop token scenario++ if(!facet_query.query.empty()) {+ // identify facet hash tokens++ for(const auto& the_facet: facets) {+ if(the_facet.field_name == facet_query.field_name) {+ //the_facet.hash_tokens+ break;+ }+ }++ auto fq_field = search_schema.at(facet_query.field_name);+ bool is_cyrillic = Tokenizer::is_cyrillic(fq_field.locale);+ bool normalise = is_cyrillic ? false : true;++ std::vector<std::string> facet_query_tokens;+ Tokenizer(facet_query.query, normalise, !fq_field.is_string(), fq_field.locale,+ symbols_to_index, token_separators).tokenize(facet_query_tokens);++ facet_query_num_tokens = facet_query_tokens.size();+ facet_query_last_token = facet_query_tokens.empty() ? "" : facet_query_tokens.back();+ }+const long start_result_index = (page - 1) * per_page;// `end_result_index` could be -1 when max_hits is 0@@ -1186,14 +1211,6 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const sstd::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,facet_hash_counts.end(), Collection::facet_count_compare);-- std::vector<std::string> facet_query_tokens;- if(the_field.locale.empty() || the_field.locale == "en") {- StringUtils::split(facet_query.query, facet_query_tokens, " ");- } else {- Tokenizer(facet_query.query, true, !the_field.is_string()).tokenize(facet_query_tokens);- }-std::vector<facet_value_t> facet_values;for(size_t fi = 0; fi < max_facets; fi++) {@@ -1235,11 +1252,18 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s}const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();- const std::string& last_q_token = facet_query_tokens.empty() ? "" : facet_query_tokens.back();// 2 passes: first identify tokens that need to be highlighted and then construct highlighted text- Tokenizer tokenizer(value, true, !the_field.is_string());+ bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);+ bool normalise = is_cyrillic ? false : true;++ Tokenizer tokenizer(value, normalise, !the_field.is_string(), the_field.locale, symbols_to_index, token_separators);++ // secondary tokenizer used for specific languages that requires transliteration+ // we use 2 tokenizers so that the original text offsets are available for highlighting+ Tokenizer word_tokenizer("", true, false, the_field.locale, symbols_to_index, token_separators);+std::string raw_token;size_t raw_token_index = 0, tok_start = 0, tok_end = 0;@@ -1248,6 +1272,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const ssize_t prefix_token_start_index = 0;while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {+ if(is_cyrillic) {+ word_tokenizer.tokenize(raw_token);+ }+auto token_pos_it = ftoken_pos.find(raw_token);if(token_pos_it != ftoken_pos.end()) {token_offsets[tok_start] = tok_end;@@ -1261,16 +1289,23 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const ssize_t i = 0;std::stringstream highlightedss;+ // loop until end index, accumulate token and complete highlightingwhile(i < value.size()) {if(offset_it != token_offsets.end()) {if (i == offset_it->first) {highlightedss << highlight_start_tag;- // loop until end index, accumulate token and complete highlighting- size_t token_len = (i == prefix_token_start_index) ?- std::min(last_full_q_token.size(), last_q_token.size()) :+ // do prefix highlighting for non-dropped last token+ size_t token_len = (i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) ?+ facet_query_last_token.size() :(offset_it->second - i + 1);+ if(i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) {+ token_len = std::min((offset_it->second - i + 1), facet_query_last_token.size());+ } else {+ token_len = (offset_it->second - i + 1);+ }+for(size_t j = 0; j < token_len; j++) {highlightedss << value[i + j];}diff --git a/src/index.cpp b/src/index.cppindex 1d61061a..3c3cf09a 100644--- a/src/index.cpp+++ b/src/index.cpp@@ -2541,7 +2541,8 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&//LOG(INFO) << "facet_query.query: " << facet_query.query;std::vector<std::string> query_tokens;- Tokenizer(facet_query.query, true, !facet_field.is_string()).tokenize(query_tokens);+ Tokenizer(facet_query.query, true, !facet_field.is_string(),+ facet_field.locale, symbols_to_index, token_separators).tokenize(query_tokens);std::vector<token_t> search_tokens, qtokens;diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 6be4def4..3a1626f6 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -7,6 +7,24 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, constconst std::vector<char>& separators):i(0), normalize(normalize), no_op(no_op), locale(locale) {+ for(char c: symbols_to_index) {+ index_symbols[uint8_t(c)] = 1;+ }++ for(char c: separators) {+ separator_symbols[uint8_t(c)] = 1;+ }++ UErrorCode errcode = U_ZERO_ERROR;+ nfkd = icu::Normalizer2::getNFKDInstance(errcode);++ cd = iconv_open("ASCII//TRANSLIT", "UTF-8");++ init(input);+}+++void Tokenizer::init(const std::string& input) {if(locale == "zh") {UErrorCode translit_status = U_ZERO_ERROR;transliterator = icu::Transliterator::createInstance("Traditional-Simplified",@@ -39,8 +57,6 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, consttext = input;}- cd = iconv_open("ASCII//TRANSLIT", "UTF-8");-if(!locale.empty() && locale != "en") {UErrorCode status = U_ZERO_ERROR;const icu::Locale& icu_locale = icu::Locale(locale.c_str());@@ -49,20 +65,10 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, constunicode_text = icu::UnicodeString::fromUTF8(text);bi->setText(unicode_text);- position = bi->first();- prev_position = -1;- }-- for(char c: symbols_to_index) {- index_symbols[uint8_t(c)] = 1;- }-- for(char c: separators) {- separator_symbols[uint8_t(c)] = 1;+ start_pos = bi->first();+ end_pos = bi->next();+ utf8_start_index = 0;}-- UErrorCode errcode = U_ZERO_ERROR;- nfkd = icu::Normalizer2::getNFKDInstance(errcode);}bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_index, size_t& end_index) {@@ -79,62 +85,59 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde}if(!locale.empty() && locale != "en") {- while (position != icu::BreakIterator::DONE) {- //LOG(INFO) << "Position: " << position;+ while (end_pos != icu::BreakIterator::DONE) {+ //LOG(INFO) << "Position: " << start_pos;bool found_token = false;- if(prev_position != -1) {- std::string word;- size_t length = position - prev_position;- //LOG(INFO) << "token: " << token;-- if(locale == "ko") {- UErrorCode errcode = U_ZERO_ERROR;- icu::UnicodeString src = unicode_text.tempSubString(prev_position, length);- icu::UnicodeString dst;- nfkd->normalize(src, dst, errcode);-- if(!U_FAILURE(errcode)) {- token = dst.toUTF8String(word);- } else {- LOG(ERROR) << "Unicode error during parsing: " << errcode;- }- } else if(normalize && is_cyrillic(locale)) {- auto raw_text = unicode_text.tempSubString(prev_position, length);- transliterator->transliterate(raw_text);- token = raw_text.toUTF8String(word);+ std::string word;+ //LOG(INFO) << "token: " << token;++ if(locale == "ko") {+ UErrorCode errcode = U_ZERO_ERROR;+ icu::UnicodeString src = unicode_text.tempSubStringBetween(start_pos, end_pos);+ icu::UnicodeString dst;+ nfkd->normalize(src, dst, errcode);++ if(!U_FAILURE(errcode)) {+ token = dst.toUTF8String(word);} else {- token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);+ LOG(ERROR) << "Unicode error during parsing: " << errcode;}+ } else if(normalize && is_cyrillic(locale)) {+ auto raw_text = unicode_text.tempSubStringBetween(start_pos, end_pos);+ transliterator->transliterate(raw_text);+ token = raw_text.toUTF8String(word);+ } else {+ token = unicode_text.tempSubStringBetween(start_pos, end_pos).toUTF8String(word);+ }- if(!token.empty()) {- if (!std::isalnum(token[0]) && is_ascii_char(token[i])) {- // ignore ascii symbols- found_token = false;- } else if(locale == "ko" && token == "·") {- found_token = false;- } else if(locale == "zh" && (token == "," || token == "─" || token == "。")) {- found_token = false;- } else {-- if(std::isalnum(token[0]) && is_ascii_char(token[0])) {- // normalize an ascii string- std::transform(token.begin(), token.end(), token.begin(),- [](unsigned char c){ return std::tolower(c); });- }-- found_token = true;- token_index = token_counter++;+ if(!token.empty()) {+ if (!std::isalnum(token[0]) && is_ascii_char(token[i])) {+ // ignore ascii symbols+ found_token = false;+ } else if(locale == "ko" && token == "·") {+ found_token = false;+ } else if(locale == "zh" && (token == "," || token == "─" || token == "。")) {+ found_token = false;+ } else {++ if(std::isalnum(token[0]) && is_ascii_char(token[0])) {+ // normalize an ascii string+ std::transform(token.begin(), token.end(), token.begin(),+ [](unsigned char c){ return std::tolower(c); });}- start_index = utf8_start_index;- end_index = utf8_start_index + token.size() - 1;- utf8_start_index = end_index + 1;+ found_token = true;+ token_index = token_counter++;}++ start_index = utf8_start_index;+ end_index = utf8_start_index + token.size() - 1;+ utf8_start_index = end_index + 1;}- prev_position = position;- position = bi->next();+ start_pos = end_pos;+ end_pos = bi->next();if(found_token) {return true;@@ -252,12 +255,13 @@ void Tokenizer::tokenize(std::vector<std::string> &tokens) {}void Tokenizer::tokenize(std::string& token) {- size_t token_index;+ size_t token_index = 0;+ init(token);next(token, token_index);}bool Tokenizer::next(std::string &token, size_t &token_index) {- size_t start_index, end_index;+ size_t start_index = 0, end_index = 0;return next(token, token_index, start_index, end_index);}diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cppindex c0b63ea7..6e5f6612 100644--- a/test/collection_faceting_test.cpp+++ b/test/collection_faceting_test.cpp@@ -716,7 +716,7 @@ TEST_F(CollectionFacetingTest, FacetQueryOnStringWithColon) {auto results = res_op.get();ASSERT_STREQ("foo:bar", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());- ASSERT_STREQ("<mark>foo:ba</mark>r", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());+ ASSERT_STREQ("<mark>foo:b</mark>ar", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());results = coll1->search("*", {}, "", {"title"}, sort_fields, {0}, 10, 1,token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex 05affa37..707b0d2c 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -489,8 +489,8 @@ TEST_F(CollectionLocaleTest, SearchCyrillicText) {ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());- ASSERT_EQ("<mark>TEST</mark> ТЕСТ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());- ASSERT_EQ("<mark>Test</mark> Тест", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("<mark>TEST</mark> <mark>ТЕСТ</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("<mark>Test</mark> <mark>Тест</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());// with typo@@ -500,8 +500,8 @@ TEST_F(CollectionLocaleTest, SearchCyrillicText) {ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());- ASSERT_EQ("<mark>TEST</mark> ТЕСТ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());- ASSERT_EQ("<mark>Test</mark> Тест", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("<mark>TEST</mark> <mark>ТЕСТ</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ ASSERT_EQ("<mark>Test</mark> <mark>Тест</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());collectionManager.drop_collection("coll1");}@@ -538,8 +538,8 @@ TEST_F(CollectionLocaleTest, SearchCyrillicTextWithDropTokens) {collectionManager.drop_collection("coll1");}-TEST_F(CollectionLocaleTest, SearchGreekText) {- std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "el"),};+TEST_F(CollectionLocaleTest, SearchAndFacetSearchForGreekText) {+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, "el"),};Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();nlohmann::json doc;@@ -556,5 +556,29 @@ TEST_F(CollectionLocaleTest, SearchGreekText) {ASSERT_EQ(1, results["hits"].size());ASSERT_EQ("<mark>Εμφάνιση</mark> κάθε μέρα.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ // facet search with prefix++ results = coll1->search("*", {"title"}, "", {"title"}, {}, {1}, 10, 1, FREQUENCY, {false},+ Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "title: Εμφάν").get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("<mark>Εμφάν</mark>ιση κάθε μέρα.", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>());++ // facet search with prefix typo++ results = coll1->search("*", {"title"}, "", {"title"}, {}, {1}, 10, 1, FREQUENCY, {false},+ Index::DROP_TOKENS_THRESHOLD,+ spp::sparse_hash_set<std::string>(),+ spp::sparse_hash_set<std::string>(), 10, "title: Εμφάνση").get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ(1, results["facet_counts"].size());+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());+ ASSERT_EQ("<mark>Εμφάνισ</mark>η κάθε μέρα.", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>());+collectionManager.drop_collection("coll1");}