Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix highlighting on cyrillic text with special chars.
Kishore Nallan2b2b4c5aecec
diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 3a1626f6..09b7727b 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -112,13 +112,16 @@ bool Tokenizer::next(std::string &token, size_t& token_index, size_t& start_inde}if(!token.empty()) {- if (!std::isalnum(token[0]) && is_ascii_char(token[i])) {+ if (!std::isalnum(token[0]) && is_ascii_char(token[0])) {// ignore ascii symbolsfound_token = false;+ token_counter++;} else if(locale == "ko" && token == "·") {found_token = false;+ token_counter++;} else if(locale == "zh" && (token == "," || token == "─" || token == "。")) {found_token = false;+ token_counter++;} else {if(std::isalnum(token[0]) && is_ascii_char(token[0])) {diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex 707b0d2c..7aa8ee51 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -582,3 +582,23 @@ TEST_F(CollectionLocaleTest, SearchAndFacetSearchForGreekText) {collectionManager.drop_collection("coll1");}++TEST_F(CollectionLocaleTest, SearchOnCyrillicTextWithSpecialCharacters) {+ std::vector<field> fields = {field("title", field_types::STRING, true, false, true, "ru"),};+ Collection* coll1 = collectionManager.create_collection("coll1", 1, fields).get();++ nlohmann::json doc;+ doc["title"] = "«Сирый», «несчастный», «никчёмный» — принятое "+ "особ, сейчас, впрочем, оттенок скромности. Посыл, "+ "среди которых отсутствие мобильного страшное.";++ ASSERT_TRUE(coll1->add(doc.dump()).ok());++ auto results = coll1->search("отсутствие", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}).get();++ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("скромности. Посыл, среди которых <mark>отсутствие</mark> мобильного страшное",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>());++ collectionManager.drop_collection("coll1");+}