Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix highlighting on cyrillic text with special chars.

Kishore Nallan2b2b4c5aecec
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 3a1626f6..09b7727b 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -112,13 +112,16 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde
}
if(!token.empty()) {
- if (!std::isalnum(token[0]) && is_ascii_char(token[i])) {
+ if (!std::isalnum(token[0]) && is_ascii_char(token[0])) {
// ignore ascii symbols
found_token = false;
+ token_counter++;
} else if(locale == "ko" && token == "·") {
found_token = false;
+ token_counter++;
} else if(locale == "zh" && (token == "," || token == "─" || token == "。")) {
found_token = false;
+ token_counter++;
} else {
if(std::isalnum(token[0]) && is_ascii_char(token[0])) {
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index 707b0d2c..7aa8ee51 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -582,3 +582,23 @@ TEST_F(CollectionLocaleTest, SearchAndFacetSearchForGreekText) {
collectionManager.drop_collection("coll1");
}
+
+TEST_F(CollectionLocaleTest, SearchOnCyrillicTextWithSpecialCharacters) {
+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, "ru"),};
+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();
+
+ nlohmann::json doc;
+ doc["title"] = "«Сирый», «несчастный», «никчёмный» — принятое "
+ "особ, сейчас, впрочем, оттенок скромности. Посыл, "
+ "среди которых отсутствие мобильного страшное.";
+
+ ASSERT_TRUE(coll1->add(doc.dump()).ok());
+
+ auto results = coll1->search("отсутствие", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}).get();
+
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("скромности. Посыл, среди которых <mark>отсутствие</mark> мобильного страшное",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
+ collectionManager.drop_collection("coll1");
+}