Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix facet highlighting for Cyrillic text.

Kishore Nallana9c15ec0f755
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 0e4ff41f..8a767f21 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -31,8 +31,8 @@ private:
std::string locale;
icu::BreakIterator* bi = nullptr;
icu::UnicodeString unicode_text;
- int32_t position = 0;
- int32_t prev_position = -1;
+ int32_t start_pos = 0;
+ int32_t end_pos = 0;
int32_t utf8_start_index = 0;
char* normalized_text = nullptr;
@@ -66,6 +66,8 @@ public:
delete transliterator;
}
+ void init(const std::string& input);
+
bool next(std::string& token, size_t& token_index, size_t& start_index, size_t& end_index);
bool next(std::string& token, size_t& token_index);
diff --git a/src/collection.cpp b/src/collection.cpp
index d350f3f2..ee67b1ed 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1000,6 +1000,31 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
override_kv_index++;
}
+ std::string facet_query_last_token;
+ size_t facet_query_num_tokens = 0; // used to identify drop token scenario
+
+ if(!facet_query.query.empty()) {
+ // identify facet hash tokens
+
+ for(const auto& the_facet: facets) {
+ if(the_facet.field_name == facet_query.field_name) {
+ //the_facet.hash_tokens
+ break;
+ }
+ }
+
+ auto fq_field = search_schema.at(facet_query.field_name);
+ bool is_cyrillic = Tokenizer::is_cyrillic(fq_field.locale);
+ bool normalise = is_cyrillic ? false : true;
+
+ std::vector<std::string> facet_query_tokens;
+ Tokenizer(facet_query.query, normalise, !fq_field.is_string(), fq_field.locale,
+ symbols_to_index, token_separators).tokenize(facet_query_tokens);
+
+ facet_query_num_tokens = facet_query_tokens.size();
+ facet_query_last_token = facet_query_tokens.empty() ? "" : facet_query_tokens.back();
+ }
+
const long start_result_index = (page - 1) * per_page;
// `end_result_index` could be -1 when max_hits is 0
@@ -1186,14 +1211,6 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
std::nth_element(facet_hash_counts.begin(), facet_hash_counts.begin() + max_facets,
facet_hash_counts.end(), Collection::facet_count_compare);
-
- std::vector<std::string> facet_query_tokens;
- if(the_field.locale.empty() || the_field.locale == "en") {
- StringUtils::split(facet_query.query, facet_query_tokens, " ");
- } else {
- Tokenizer(facet_query.query, true, !the_field.is_string()).tokenize(facet_query_tokens);
- }
-
std::vector<facet_value_t> facet_values;
for(size_t fi = 0; fi < max_facets; fi++) {
@@ -1235,11 +1252,18 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
}
const std::string& last_full_q_token = ftokens.empty() ? "" : ftokens.back();
- const std::string& last_q_token = facet_query_tokens.empty() ? "" : facet_query_tokens.back();
// 2 passes: first identify tokens that need to be highlighted and then construct highlighted text
- Tokenizer tokenizer(value, true, !the_field.is_string());
+ bool is_cyrillic = Tokenizer::is_cyrillic(the_field.locale);
+ bool normalise = is_cyrillic ? false : true;
+
+ Tokenizer tokenizer(value, normalise, !the_field.is_string(), the_field.locale, symbols_to_index, token_separators);
+
+ // secondary tokenizer used for specific languages that requires transliteration
+ // we use 2 tokenizers so that the original text offsets are available for highlighting
+ Tokenizer word_tokenizer("", true, false, the_field.locale, symbols_to_index, token_separators);
+
std::string raw_token;
size_t raw_token_index = 0, tok_start = 0, tok_end = 0;
@@ -1248,6 +1272,10 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
size_t prefix_token_start_index = 0;
while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {
+ if(is_cyrillic) {
+ word_tokenizer.tokenize(raw_token);
+ }
+
auto token_pos_it = ftoken_pos.find(raw_token);
if(token_pos_it != ftoken_pos.end()) {
token_offsets[tok_start] = tok_end;
@@ -1261,16 +1289,23 @@ Option<nlohmann::json> Collection::search(const std::string & raw_query, const s
size_t i = 0;
std::stringstream highlightedss;
+ // loop until end index, accumulate token and complete highlighting
while(i < value.size()) {
if(offset_it != token_offsets.end()) {
if (i == offset_it->first) {
highlightedss << highlight_start_tag;
- // loop until end index, accumulate token and complete highlighting
- size_t token_len = (i == prefix_token_start_index) ?
- std::min(last_full_q_token.size(), last_q_token.size()) :
+ // do prefix highlighting for non-dropped last token
+ size_t token_len = (i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) ?
+ facet_query_last_token.size() :
(offset_it->second - i + 1);
+ if(i == prefix_token_start_index && token_offsets.size() == facet_query_num_tokens) {
+ token_len = std::min((offset_it->second - i + 1), facet_query_last_token.size());
+ } else {
+ token_len = (offset_it->second - i + 1);
+ }
+
for(size_t j = 0; j < token_len; j++) {
highlightedss << value[i + j];
}
diff --git a/src/index.cpp b/src/index.cpp
index 1d61061a..3c3cf09a 100644
--- a/src/index.cpp
+++ b/src/index.cpp
@@ -2541,7 +2541,8 @@ void Index::compute_facet_infos(const std::vector<facet>& facets, facet_query_t&
//LOG(INFO) << "facet_query.query: " << facet_query.query;
std::vector<std::string> query_tokens;
- Tokenizer(facet_query.query, true, !facet_field.is_string()).tokenize(query_tokens);
+ Tokenizer(facet_query.query, true, !facet_field.is_string(),
+ facet_field.locale, symbols_to_index, token_separators).tokenize(query_tokens);
std::vector<token_t> search_tokens, qtokens;
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 6be4def4..3a1626f6 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -7,6 +7,24 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const
const std::vector<char>& separators):
i(0), normalize(normalize), no_op(no_op), locale(locale) {
+ for(char c: symbols_to_index) {
+ index_symbols[uint8_t(c)] = 1;
+ }
+
+ for(char c: separators) {
+ separator_symbols[uint8_t(c)] = 1;
+ }
+
+ UErrorCode errcode = U_ZERO_ERROR;
+ nfkd = icu::Normalizer2::getNFKDInstance(errcode);
+
+ cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
+
+ init(input);
+}
+
+
+void Tokenizer::init(const std::string& input) {
if(locale == "zh") {
UErrorCode translit_status = U_ZERO_ERROR;
transliterator = icu::Transliterator::createInstance("Traditional-Simplified",
@@ -39,8 +57,6 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const
text = input;
}
- cd = iconv_open("ASCII//TRANSLIT", "UTF-8");
-
if(!locale.empty() && locale != "en") {
UErrorCode status = U_ZERO_ERROR;
const icu::Locale& icu_locale = icu::Locale(locale.c_str());
@@ -49,20 +65,10 @@ Tokenizer::Tokenizer(const std::string& input, bool normalize, bool no_op, const
unicode_text = icu::UnicodeString::fromUTF8(text);
bi->setText(unicode_text);
- position = bi->first();
- prev_position = -1;
- }
-
- for(char c: symbols_to_index) {
- index_symbols[uint8_t(c)] = 1;
- }
-
- for(char c: separators) {
- separator_symbols[uint8_t(c)] = 1;
+ start_pos = bi->first();
+ end_pos = bi->next();
+ utf8_start_index = 0;
}
-
- UErrorCode errcode = U_ZERO_ERROR;
- nfkd = icu::Normalizer2::getNFKDInstance(errcode);
}
bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_index, size_t& end_index) {
@@ -79,62 +85,59 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
}
if(!locale.empty() && locale != "en") {
- while (position != icu::BreakIterator::DONE) {
- //LOG(INFO) << "Position: " << position;
+ while (end_pos != icu::BreakIterator::DONE) {
+ //LOG(INFO) << "Position: " << start_pos;
bool found_token = false;
- if(prev_position != -1) {
- std::string word;
- size_t length = position - prev_position;
- //LOG(INFO) << "token: " << token;
-
- if(locale == "ko") {
- UErrorCode errcode = U_ZERO_ERROR;
- icu::UnicodeString src = unicode_text.tempSubString(prev_position, length);
- icu::UnicodeString dst;
- nfkd->normalize(src, dst, errcode);
-
- if(!U_FAILURE(errcode)) {
- token = dst.toUTF8String(word);
- } else {
- LOG(ERROR) << "Unicode error during parsing: " << errcode;
- }
- } else if(normalize && is_cyrillic(locale)) {
- auto raw_text = unicode_text.tempSubString(prev_position, length);
- transliterator->transliterate(raw_text);
- token = raw_text.toUTF8String(word);
+ std::string word;
+ //LOG(INFO) << "token: " << token;
+
+ if(locale == "ko") {
+ UErrorCode errcode = U_ZERO_ERROR;
+ icu::UnicodeString src = unicode_text.tempSubStringBetween(start_pos, end_pos);
+ icu::UnicodeString dst;
+ nfkd->normalize(src, dst, errcode);
+
+ if(!U_FAILURE(errcode)) {
+ token = dst.toUTF8String(word);
} else {
- token = unicode_text.tempSubString(prev_position, length).toUTF8String(word);
+ LOG(ERROR) << "Unicode error during parsing: " << errcode;
}
+ } else if(normalize && is_cyrillic(locale)) {
+ auto raw_text = unicode_text.tempSubStringBetween(start_pos, end_pos);
+ transliterator->transliterate(raw_text);
+ token = raw_text.toUTF8String(word);
+ } else {
+ token = unicode_text.tempSubStringBetween(start_pos, end_pos).toUTF8String(word);
+ }
- if(!token.empty()) {
- if (!std::isalnum(token[0]) && is_ascii_char(token[i])) {
- // ignore ascii symbols
- found_token = false;
- } else if(locale == "ko" && token == "·") {
- found_token = false;
- } else if(locale == "zh" && (token == "," || token == "─" || token == "。")) {
- found_token = false;
- } else {
-
- if(std::isalnum(token[0]) && is_ascii_char(token[0])) {
- // normalize an ascii string
- std::transform(token.begin(), token.end(), token.begin(),
- [](unsigned char c){ return std::tolower(c); });
- }
-
- found_token = true;
- token_index = token_counter++;
+ if(!token.empty()) {
+ if (!std::isalnum(token[0]) && is_ascii_char(token[i])) {
+ // ignore ascii symbols
+ found_token = false;
+ } else if(locale == "ko" && token == "·") {
+ found_token = false;
+ } else if(locale == "zh" && (token == "," || token == "─" || token == "。")) {
+ found_token = false;
+ } else {
+
+ if(std::isalnum(token[0]) && is_ascii_char(token[0])) {
+ // normalize an ascii string
+ std::transform(token.begin(), token.end(), token.begin(),
+ [](unsigned char c){ return std::tolower(c); });
}
- start_index = utf8_start_index;
- end_index = utf8_start_index + token.size() - 1;
- utf8_start_index = end_index + 1;
+ found_token = true;
+ token_index = token_counter++;
}
+
+ start_index = utf8_start_index;
+ end_index = utf8_start_index + token.size() - 1;
+ utf8_start_index = end_index + 1;
}
- prev_position = position;
- position = bi->next();
+ start_pos = end_pos;
+ end_pos = bi->next();
if(found_token) {
return true;
@@ -252,12 +255,13 @@ void Tokenizer::tokenize(std::vector<std::string> &tokens) {
}
void Tokenizer::tokenize(std::string& token) {
- size_t token_index;
+ size_t token_index = 0;
+ init(token);
next(token, token_index);
}
bool Tokenizer::next(std::string &token, size_t &token_index) {
- size_t start_index, end_index;
+ size_t start_index = 0, end_index = 0;
return next(token, token_index, start_index, end_index);
}
diff --git a/test/collection_faceting_test.cpp b/test/collection_faceting_test.cpp
index c0b63ea7..6e5f6612 100644
--- a/test/collection_faceting_test.cpp
+++ b/test/collection_faceting_test.cpp
@@ -716,7 +716,7 @@ TEST_F(CollectionFacetingTest, FacetQueryOnStringWithColon) {
auto results = res_op.get();
ASSERT_STREQ("foo:bar", results["facet_counts"][0]["counts"][0]["value"].get<std::string>().c_str());
- ASSERT_STREQ("<mark>foo:ba</mark>r", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
+ ASSERT_STREQ("<mark>foo:b</mark>ar", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>().c_str());
results = coll1->search("*", {}, "", {"title"}, sort_fields, {0}, 10, 1,
token_ordering::FREQUENCY, {true}, 10, spp::sparse_hash_set<std::string>(),
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index 05affa37..707b0d2c 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -489,8 +489,8 @@ TEST_F(CollectionLocaleTest, SearchCyrillicText) {
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
- ASSERT_EQ("<mark>TEST</mark> ТЕСТ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
- ASSERT_EQ("<mark>Test</mark> Тест", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("<mark>TEST</mark> <mark>ТЕСТ</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("<mark>Test</mark> <mark>Тест</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
// with typo
@@ -500,8 +500,8 @@ TEST_F(CollectionLocaleTest, SearchCyrillicText) {
ASSERT_EQ("1", results["hits"][0]["document"]["id"].get<std::string>());
ASSERT_EQ("0", results["hits"][1]["document"]["id"].get<std::string>());
- ASSERT_EQ("<mark>TEST</mark> ТЕСТ", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
- ASSERT_EQ("<mark>Test</mark> Тест", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("<mark>TEST</mark> <mark>ТЕСТ</mark>", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ ASSERT_EQ("<mark>Test</mark> <mark>Тест</mark>", results["hits"][1]["highlights"][0]["snippet"].get<std::string>());
collectionManager.drop_collection("coll1");
}
@@ -538,8 +538,8 @@ TEST_F(CollectionLocaleTest, SearchCyrillicTextWithDropTokens) {
collectionManager.drop_collection("coll1");
}
-TEST_F(CollectionLocaleTest, SearchGreekText) {
- std::vector<field> fields = {field("title", field_types::STRING, false, false, true, "el"),};
+TEST_F(CollectionLocaleTest, SearchAndFacetSearchForGreekText) {
+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, "el"),};
Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
nlohmann::json doc;
@@ -556,5 +556,29 @@ TEST_F(CollectionLocaleTest, SearchGreekText) {
ASSERT_EQ(1, results["hits"].size());
ASSERT_EQ("<mark>Εμφάνιση</mark> κάθε μέρα.", results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ // facet search with prefix
+
+ results = coll1->search("*", {"title"}, "", {"title"}, {}, {1}, 10, 1, FREQUENCY, {false},
+ Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "title: Εμφάν").get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("<mark>Εμφάν</mark>ιση κάθε μέρα.", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>());
+
+ // facet search with prefix typo
+
+ results = coll1->search("*", {"title"}, "", {"title"}, {}, {1}, 10, 1, FREQUENCY, {false},
+ Index::DROP_TOKENS_THRESHOLD,
+ spp::sparse_hash_set<std::string>(),
+ spp::sparse_hash_set<std::string>(), 10, "title: Εμφάνση").get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ(1, results["facet_counts"].size());
+ ASSERT_EQ(1, results["facet_counts"][0]["counts"].size());
+ ASSERT_EQ("<mark>Εμφάνισ</mark>η κάθε μέρα.", results["facet_counts"][0]["counts"][0]["highlighted"].get<std::string>());
+
collectionManager.drop_collection("coll1");
}