Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences
C++ algoliadatastoreelasticsearchenterprise-searchfacetingfull-text-searchfuzzy-searchgeosearchin-memoryinstantsearchmerchandisingpineconesearchsearch-enginesemantic-searchsimilarity-searchsite-searchsynonymstypo-tolerancevector-search
Fix highlight surround num tokens in Cyrillic.
Kishore Nallanc3a85be42f1a
diff --git a/include/tokenizer.h b/include/tokenizer.hindex 8a767f21..4ba43f88 100644--- a/include/tokenizer.h+++ b/include/tokenizer.h@@ -74,7 +74,7 @@ public:void tokenize(std::vector<std::string>& tokens);- void tokenize(std::string& token);+ bool tokenize(std::string& token);static bool is_cyrillic(const std::string& locale);};\ No newline at end of filediff --git a/src/collection.cpp b/src/collection.cppindex 1d38de7f..62599bd3 100644--- a/src/collection.cpp+++ b/src/collection.cpp@@ -1770,7 +1770,10 @@ void Collection::highlight_result(const field &search_field,while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {if(is_cyrillic) {- word_tokenizer.tokenize(raw_token);+ bool found_token = word_tokenizer.tokenize(raw_token);+ if(!found_token) {+ continue;+ }}if(!found_first_match) {diff --git a/src/tokenizer.cpp b/src/tokenizer.cppindex 09b7727b..c291ecf9 100644--- a/src/tokenizer.cpp+++ b/src/tokenizer.cpp@@ -257,10 +257,10 @@ void Tokenizer::tokenize(std::vector<std::string> &tokens) {}}-void Tokenizer::tokenize(std::string& token) {+bool Tokenizer::tokenize(std::string& token) {size_t token_index = 0;init(token);- next(token, token_index);+ return next(token, token_index);}bool Tokenizer::next(std::string &token, size_t &token_index) {diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cppindex 45d3cff6..ae30fb59 100644--- a/test/collection_locale_test.cpp+++ b/test/collection_locale_test.cpp@@ -600,6 +600,11 @@ TEST_F(CollectionLocaleTest, SearchOnCyrillicTextWithSpecialCharacters) {ASSERT_EQ("скромности. Посыл, среди которых <mark>отсутствие</mark> мобильного страшное",results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+ results = coll1->search("принятое", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}).get();+ ASSERT_EQ(1, results["hits"].size());+ ASSERT_EQ("Сирый», «несчастный», «никчёмный» — <mark>принятое</mark> особ, сейчас, впрочем, оттенок скромности. Посыл, среди которых отсутствие мобильного страшное.",+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>());+results = coll1->search("*", {}, "", {"title"}, {}, {0}, 0, 1, FREQUENCY, {true}, 10,spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(),10, "title: отсутствие").get();