Skip to content

Open Source alternative to Algolia + Pinecone and an Easier-to-Use alternative to ElasticSearch ⚡ 🔍 ✨ Fast, typo tolerant, in-memory fuzzy Search Engine for building delightful search experiences

Fix highlight surround num tokens in Cyrillic.

Kishore Nallanc3a85be42f1a
diff --git a/include/tokenizer.h b/include/tokenizer.h
index 8a767f21..4ba43f88 100644
--- a/include/tokenizer.h
+++ b/include/tokenizer.h
@@ -74,7 +74,7 @@ public:
void tokenize(std::vector<std::string>& tokens);
- void tokenize(std::string& token);
+ bool tokenize(std::string& token);
static bool is_cyrillic(const std::string& locale);
};
\ No newline at end of file
diff --git a/src/collection.cpp b/src/collection.cpp
index 1d38de7f..62599bd3 100644
--- a/src/collection.cpp
+++ b/src/collection.cpp
@@ -1770,7 +1770,10 @@ void Collection::highlight_result(const field &search_field,
while(tokenizer.next(raw_token, raw_token_index, tok_start, tok_end)) {
if(is_cyrillic) {
- word_tokenizer.tokenize(raw_token);
+ bool found_token = word_tokenizer.tokenize(raw_token);
+ if(!found_token) {
+ continue;
+ }
}
if(!found_first_match) {
diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp
index 09b7727b..c291ecf9 100644
--- a/src/tokenizer.cpp
+++ b/src/tokenizer.cpp
@@ -257,10 +257,10 @@ void Tokenizer::tokenize(std::vector<std::string> &tokens) {
}
}
-void Tokenizer::tokenize(std::string& token) {
+bool Tokenizer::tokenize(std::string& token) {
size_t token_index = 0;
init(token);
- next(token, token_index);
+ return next(token, token_index);
}
bool Tokenizer::next(std::string &token, size_t &token_index) {
diff --git a/test/collection_locale_test.cpp b/test/collection_locale_test.cpp
index 45d3cff6..ae30fb59 100644
--- a/test/collection_locale_test.cpp
+++ b/test/collection_locale_test.cpp
@@ -600,6 +600,11 @@ TEST_F(CollectionLocaleTest, SearchOnCyrillicTextWithSpecialCharacters) {
ASSERT_EQ("скромности. Посыл, среди которых <mark>отсутствие</mark> мобильного страшное",
results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+ results = coll1->search("принятое", {"title"}, "", {}, {}, {0}, 10, 1, FREQUENCY, {true}).get();
+ ASSERT_EQ(1, results["hits"].size());
+ ASSERT_EQ("Сирый», «несчастный», «никчёмный» — <mark>принятое</mark> особ, сейчас, впрочем, оттенок скромности. Посыл, среди которых отсутствие мобильного страшное.",
+ results["hits"][0]["highlights"][0]["snippet"].get<std::string>());
+
results = coll1->search("*", {}, "", {"title"}, {}, {0}, 0, 1, FREQUENCY, {true}, 10,
spp::sparse_hash_set<std::string>(), spp::sparse_hash_set<std::string>(),
10, "title: отсутствие").get();